ci: filter test matrices by per-test coverage globs (#730)

* ci: filter test matrices by per-test coverage globs to cut LLM spend

every test in `crossagent/`, `agnostic/`, and every provider entry now
declares a `coverage: string[]` of repo-relative globs. the new `changes`
job runs `paths-filter` for a docs-only short-circuit, then pipes the
changed-file list into `action/test/matrix.ts`, which intersects each
entry's coverage against the diff and emits filtered `agents`,
`agnostic`, `flagships`, and `aliases` matrices. main pushes and
`workflow_dispatch` set `FULL=1` to run everything as a stale-glob safety
net.

retires `changed-agents.sh` and the `MODE=flagships` branch in
`list-aliases.ts` in favor of one consistent model.

* ci(matrix): switch test discovery to dep-free static parsing

the GHA `changes` job has no `node_modules` installed. the previous
dynamic-import path pulled the test files transitively through
`utils.ts` -> `agents/index.ts` -> `@actions/core`, which exploded with
ERR_MODULE_NOT_FOUND. parse the test files via regex instead so
matrix.ts stays zero-dep — the chain (matrix -> coverage / providers /
list-aliases / models) imports only node builtins and relative TS files.

* ci(matrix): address PR #730 review feedback

- drop dangling `action/mcp/toolFiltering.ts` glob from `nobash`,
  `restricted`, `tokenExfil` (file doesn't exist; `.test.ts` does, but
  the runtime tooling lives in `mcp/shell.ts` and `agents/{claude,opencode}.ts`,
  both already covered).
- drop unused `coverageForProvider` export and its `byName` map from
  `providers.ts` (matrix.ts builds its own lookup inline).
- derive the active agent list from `agents/index.ts` via the same
  dep-free regex tactic as `parseTestFile` instead of hardcoding
  `["claude", "opencode"]` — adding a new harness file now wires it
  into the dynamic matrix automatically.
- treat `coverage: []` as `coverage: undefined` in `shouldRun` so an
  accidentally-empty array doesn't silently skip CI on every PR.
- add `action/utils/activity.ts` and `action/mcp/selectMode.ts` to the
  `timeout` test's coverage — the activity-timeout enforcement path
  was the original reason the test exists.
- ungate the `root` job (lint/format/typecheck/vitest). it's a required
  status check on `main`, so gating it on `code == 'true'` would make
  docs-only PRs unmergeable (skipped jobs don't satisfy required-check
  rules). the real LLM savings come from skipping the four matrices,
  not from skipping `root`.
- harden the four matrix-job `if:` guards from `outputs.matrix && ...`
  to `outputs.matrix != '' && ...` — explicit > implicit short-circuit.
- document `expandBraces`'s flat-only support so a future author isn't
  surprised by `{a,{b,c}}` not expanding.
- fix awkward sentence in `wiki/action-tests.md` "CI Cost Filtering".

test/agnostic/gitHooks.ts

@@ -96,4 +96,10 @@ export const test: TestRunnerOptions = {
   repoSetup,
   env: { PULLFROG_DISABLE_SECURITY_INSTRUCTIONS: "1" },
   tags: ["agnostic", "security"],
+  coverage: [
+    "action/utils/gitAuth.ts",
+    "action/utils/gitAuthServer.ts",
+    "action/mcp/git.ts",
+    "action/mcp/checkout.ts",
+  ],
 };

test/agnostic/gitPerms.ts

@@ -104,4 +104,10 @@ export const test: TestRunnerOptions = {
   agentEnv,
   env: { PULLFROG_DISABLE_SECURITY_INSTRUCTIONS: "1" },
   tags: ["agnostic"],
+  coverage: [
+    "action/utils/gitAuth.ts",
+    "action/utils/gitAuthServer.ts",
+    "action/mcp/git.ts",
+    "action/mcp/checkout.ts",
+  ],
 };

test/agnostic/packageJsonScripts.ts

@@ -92,4 +92,5 @@ export const test: TestRunnerOptions = {
   validator,
   env: { PULLFROG_DISABLE_SECURITY_INSTRUCTIONS: "1" },
   tags: ["agnostic", "security"],
+  coverage: ["action/mcp/dependencies.ts", "action/utils/install.ts"],
 };

test/agnostic/pushDisabled.ts

@@ -62,4 +62,10 @@ export const test: TestRunnerOptions = {
   agentEnv,
   env: { PULLFROG_DISABLE_SECURITY_INSTRUCTIONS: "1" },
   tags: ["agnostic"],
+  coverage: [
+    "action/utils/gitAuth.ts",
+    "action/utils/gitAuthServer.ts",
+    "action/mcp/git.ts",
+    "action/mcp/checkout.ts",
+  ],
 };

test/agnostic/pushEnabled.ts

@@ -74,4 +74,10 @@ export const test: TestRunnerOptions = {
   validator,
   env: { PULLFROG_DISABLE_SECURITY_INSTRUCTIONS: "1" },
   tags: ["agnostic"],
+  coverage: [
+    "action/utils/gitAuth.ts",
+    "action/utils/gitAuthServer.ts",
+    "action/mcp/git.ts",
+    "action/mcp/checkout.ts",
+  ],
 };

test/agnostic/pushRestricted.ts

@@ -67,4 +67,10 @@ export const test: TestRunnerOptions = {
   validator,
   env: { PULLFROG_DISABLE_SECURITY_INSTRUCTIONS: "1" },
   tags: ["agnostic"],
+  coverage: [
+    "action/utils/gitAuth.ts",
+    "action/utils/gitAuthServer.ts",
+    "action/mcp/git.ts",
+    "action/mcp/checkout.ts",
+  ],
 };

test/agnostic/timeout.ts

@@ -29,4 +29,11 @@ export const test: TestRunnerOptions = {
   expectFailure: true,
   env: { PULLFROG_DISABLE_SECURITY_INSTRUCTIONS: "1" },
   tags: ["agnostic"],
+  coverage: [
+    "action/utils/timer.ts",
+    "action/utils/subprocess.ts",
+    "action/utils/exitHandler.ts",
+    "action/utils/activity.ts",
+    "action/mcp/selectMode.ts",
+  ],
 };

test/changed-agents.sh

@@ -1,70 +0,0 @@
-#!/usr/bin/env bash
-# determines which agents need testing based on changed files.
-# reads changed file paths from stdin (JSON array or newline-delimited).
-# outputs a JSON array of agent names to stdout.
-#
-# only agents whose harness file changed AND are exported from index.ts are included.
-# shared.ts/index.ts/postRun.ts and other non-harness action changes fall back to opencode as a canary.
-set -euo pipefail
-
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-AGENTS_INDEX="$SCRIPT_DIR/../agents/index.ts"
-
-# build the set of active agents from index.ts imports (portable, no -P)
-active_agents=()
-while IFS= read -r line; do
-  [[ -n "$line" ]] && active_agents+=("$line")
-done < <(sed -n 's/.*from "\.\/\([^"]*\)\.ts".*/\1/p' "$AGENTS_INDEX" | grep -v shared)
-
-# read stdin - auto-detect JSON array vs newline-delimited
-input=$(cat)
-if echo "$input" | jq -e 'type == "array"' > /dev/null 2>&1; then
-  files=$(echo "$input" | jq -r '.[]')
-else
-  files="$input"
-fi
-
-is_active_agent() {
-  local name="$1"
-  for a in "${active_agents[@]}"; do
-    [[ "$a" == "$name" ]] && return 0
-  done
-  return 1
-}
-
-# find which agent harness files changed
-changed_agents=()
-has_non_agent_change=false
-
-while IFS= read -r file; do
-  [[ -z "$file" ]] && continue
-  case "$file" in
-    action/agents/shared.ts|action/agents/index.ts|action/agents/postRun.ts)
-      has_non_agent_change=true
-      ;;
-    action/agents/*.ts)
-      agent_name="$(basename "$file" .ts)"
-      if is_active_agent "$agent_name"; then
-        changed_agents+=("$agent_name")
-      else
-        # legacy/inactive agent file changed — treat as non-agent change
-        has_non_agent_change=true
-      fi
-      ;;
-    action/*)
-      has_non_agent_change=true
-      ;;
-  esac
-done <<< "$files"
-
-# output agents based on change type.
-# non-agent action changes always include opencode as a canary.
-if $has_non_agent_change; then
-  changed_agents+=("opencode")
-fi
-
-if [[ ${#changed_agents[@]} -gt 0 ]]; then
-  printf '%s\n' "${changed_agents[@]}" | sort -u | jq -R . | jq -sc .
-else
-  echo '[]'
-fi

test/ci.test.ts

@@ -1,4 +1,3 @@
-import { execFileSync } from "node:child_process";
 import { readdirSync, readFileSync } from "node:fs";
 import { dirname, join } from "node:path";
 import { fileURLToPath } from "node:url";
@@ -16,7 +15,7 @@ type WorkflowJob = {
   "runs-on": string;
   "timeout-minutes"?: number;
   permissions?: WorkflowPermissions;
-  strategy?: { "fail-fast": boolean; matrix: Record<string, string[]> };
+  strategy?: { "fail-fast": boolean; matrix: Record<string, unknown> };
   env?: Record<string, string>;
   steps?: unknown[];
 };
@@ -57,7 +56,6 @@ const expectedAgents = Object.keys(agents).sort();
 const crossagentTests = getTestNamesFromDir("crossagent");
 const agnosticTests = getTestNamesFromDir("agnostic");
 const adhocTests = getTestNamesFromDir("adhoc");
-const dynamicAgentsExpression = "$" + "{{ fromJSON(needs.changes.outputs.agents) }}";
 
 // all provider API key names + GITHUB_TOKEN + model overrides
 const expectedAgentEnvVars = [
@@ -83,53 +81,22 @@ describe("ci workflow consistency", () => {
     const rootJob = rootWorkflow.jobs["action-agents"];
     const actionJob = actionWorkflow.jobs.agents;
 
-    it("root agent matrix uses dynamic output from changes job", () => {
-      expect(rootJob.strategy!.matrix.agent).toBe(dynamicAgentsExpression);
-    });
-
-    it("changed-agents.sh falls back to opencode when shared agent code changed", () => {
-      const input = JSON.stringify(["action/agents/shared.ts"]);
-      const output = execFileSync("bash", [join(__dirname, "changed-agents.sh")], {
-        input,
-        encoding: "utf-8",
-      });
-      expect(JSON.parse(output)).toEqual(["opencode"]);
-    });
-
-    it("changed-agents.sh falls back to opencode for non-agent action changes", () => {
-      const output = execFileSync("bash", [join(__dirname, "changed-agents.sh")], {
-        input: JSON.stringify(["action/mcp/server.ts"]),
-        encoding: "utf-8",
-      });
-      expect(JSON.parse(output)).toEqual(["opencode"]);
-    });
-
-    it("changed-agents.sh includes opencode canary alongside changed agents", () => {
-      const output = execFileSync("bash", [join(__dirname, "changed-agents.sh")], {
-        input: JSON.stringify(["action/agents/opencode.ts", "action/mcp/server.ts"]),
-        encoding: "utf-8",
-      });
-      expect(JSON.parse(output)).toEqual(["opencode"]);
-    });
-
-    it("changed-agents.sh treats legacy agent files as non-agent changes", () => {
-      const output = execFileSync("bash", [join(__dirname, "changed-agents.sh")], {
-        input: JSON.stringify(["action/agents/codex.ts", "action/agents/gemini.ts"]),
-        encoding: "utf-8",
-      });
-      expect(JSON.parse(output)).toEqual(["opencode"]);
+    it("root agents matrix is wired to the dynamic matrix output", () => {
+      const include = rootJob.strategy?.matrix.include;
+      expect(typeof include).toBe("string");
+      expect(include as string).toContain("fromJSON(needs.changes.outputs.matrix).agents");
     });
 
     it("action agent matrix matches agents map", () => {
-      expect([...actionJob.strategy!.matrix.agent].sort()).toEqual(expectedAgents);
-    });
-
-    it("root test matrix matches crossagent/ directory", () => {
-      expect([...rootJob.strategy!.matrix.test].sort()).toEqual(crossagentTests);
+      expect((actionJob.strategy?.matrix.agent as string[])?.slice().sort()).toEqual(
+        expectedAgents
+      );
     });
 
     it("action test matrix matches crossagent/ directory", () => {
-      expect([...actionJob.strategy!.matrix.test].sort()).toEqual(crossagentTests);
+      expect((actionJob.strategy?.matrix.test as string[])?.slice().sort()).toEqual(
+        crossagentTests
+      );
     });
 
     it("permissions match between root and action", () => {
@@ -149,8 +116,8 @@ describe("ci workflow consistency", () => {
     });
 
     it("fail-fast is enabled in both", () => {
-      expect(rootJob.strategy!["fail-fast"]).toBe(true);
-      expect(actionJob.strategy!["fail-fast"]).toBe(true);
+      expect(rootJob.strategy?.["fail-fast"]).toBe(true);
+      expect(actionJob.strategy?.["fail-fast"]).toBe(true);
     });
   });
 
@@ -158,12 +125,14 @@ describe("ci workflow consistency", () => {
     const rootJob = rootWorkflow.jobs["action-agnostic"];
     const actionJob = actionWorkflow.jobs.agnostic;
 
-    it("root test matrix matches agnostic/ directory", () => {
-      expect([...rootJob.strategy!.matrix.test].sort()).toEqual(agnosticTests);
+    it("root agnostic matrix is wired to the dynamic matrix output", () => {
+      const include = rootJob.strategy?.matrix.include;
+      expect(typeof include).toBe("string");
+      expect(include as string).toContain("fromJSON(needs.changes.outputs.matrix).agnostic");
     });
 
     it("action test matrix matches agnostic/ directory", () => {
-      expect([...actionJob.strategy!.matrix.test].sort()).toEqual(agnosticTests);
+      expect((actionJob.strategy?.matrix.test as string[])?.slice().sort()).toEqual(agnosticTests);
     });
 
     it("permissions match between root and action", () => {
@@ -183,8 +152,8 @@ describe("ci workflow consistency", () => {
     });
 
     it("fail-fast is enabled in both", () => {
-      expect(rootJob.strategy!["fail-fast"]).toBe(true);
-      expect(actionJob.strategy!["fail-fast"]).toBe(true);
+      expect(rootJob.strategy?.["fail-fast"]).toBe(true);
+      expect(actionJob.strategy?.["fail-fast"]).toBe(true);
     });
   });
 });

test/coverage.ts

@@ -0,0 +1,112 @@
+/**
+ * shared coverage / glob plumbing for the matrix builder.
+ *
+ * every test (`crossagent/`, `agnostic/`) and every provider entry
+ * (`providers.ts`) declares a `coverage` array of repo-relative globs. on a PR
+ * push, the `changes` job feeds the changed-file list into `matrix.ts`, which
+ * intersects each entry's globs against the diff and emits only the entries
+ * that need to run.
+ *
+ * `ALWAYS_RUN_ALL` is the escape hatch: any change to a file matched here
+ * forces the full matrix (every test, every flagship, every alias). it
+ * captures cross-cutting infrastructure where fan-out is unpredictable —
+ * agent loader, MCP server boot, test runner itself. if a per-test glob
+ * goes stale, this list and the on-`main`-full-matrix policy are the safety
+ * nets — there's no completeness lint.
+ *
+ * `coverage` is optional on tests/providers; missing = always run (treat as
+ * "any code change touches me"). default to defensive — opt into precision
+ * by adding globs.
+ */
+
+/** patterns that, when matched by any changed file, force the full matrix. */
+export const ALWAYS_RUN_ALL: string[] = [
+  // agent loader + cross-agent shared code
+  "action/agents/shared.ts",
+  "action/agents/index.ts",
+  "action/agents/postRun.ts",
+  // test harness — changing these can affect every test
+  "action/test/run.ts",
+  "action/test/utils.ts",
+  "action/test/matrix.ts",
+  "action/test/coverage.ts",
+  "action/test/providers.ts",
+  // boot + lifecycle
+  "action/main.ts",
+  "action/index.ts",
+  "action/cli.ts",
+  "action/utils/setup.ts",
+  "action/utils/lifecycle.ts",
+  "action/utils/install.ts",
+  "action/utils/docker.ts",
+  "action/utils/globals.ts",
+  // MCP orchestrator (every test runs through it)
+  "action/mcp/server.ts",
+  "action/mcp/shared.ts",
+  // dependency graph
+  "action/package.json",
+  "action/pnpm-lock.yaml",
+  // workflow itself
+  ".github/workflows/test.yml",
+];
+
+/**
+ * expand a single brace group like `{a,b,c}` into an array of patterns.
+ *
+ * intentionally minimal: nested braces (`{a,{b,c}}`) and escaped braces are
+ * NOT supported — coverage globs in this repo only need flat brace groups
+ * (`{claude,opencode}.ts`). add complexity if a real use case emerges.
+ */
+function expandBraces(pattern: string): string[] {
+  const m = pattern.match(/\{([^{}]+)\}/);
+  if (!m || m.index === undefined) return [pattern];
+  const before = pattern.slice(0, m.index);
+  const after = pattern.slice(m.index + m[0].length);
+  const opts = m[1].split(",");
+  return opts.flatMap((opt) => expandBraces(`${before}${opt}${after}`));
+}
+
+/** convert a glob pattern to a regex anchored at start + end. */
+function globToRegex(pattern: string): RegExp {
+  const DSTAR = "\u0000DSTAR\u0000";
+  let s = pattern.replace(/\*\*/g, DSTAR);
+  s = s.replace(/[.+^$()|[\]\\]/g, "\\$&");
+  s = s.replace(/\*/g, "[^/]*");
+  s = s.replace(/\?/g, "[^/]");
+  s = s.replaceAll(DSTAR, ".*");
+  return new RegExp(`^${s}$`);
+}
+
+/** does any path in `paths` match any glob in `patterns`? */
+export function anyMatch(paths: string[], patterns: string[]): boolean {
+  if (patterns.length === 0) return false;
+  const regexes = patterns.flatMap((p) => expandBraces(p)).map(globToRegex);
+  return paths.some((path) => regexes.some((r) => r.test(path)));
+}
+
+/**
+ * decide whether an entry runs given changed files + its coverage globs.
+ *
+ * three short-circuits:
+ *   1. `full` flag (e.g. main pushes, workflow_dispatch) → always run
+ *   2. any changed file matches `ALWAYS_RUN_ALL` → run everything
+ *   3. coverage missing or empty on the entry → run (defensive default)
+ *
+ * otherwise: run iff any changed file matches the entry's coverage globs.
+ *
+ * `coverage: []` is treated identically to `coverage: undefined` to avoid the
+ * footgun where a future test author intends "skip on PRs" by passing an
+ * empty array — silently skipping CI on every PR is worse than always running.
+ */
+export type ShouldRunInput = {
+  changedFiles: string[];
+  coverage: string[] | undefined;
+  full: boolean;
+};
+
+export function shouldRun(input: ShouldRunInput): boolean {
+  if (input.full) return true;
+  if (anyMatch(input.changedFiles, ALWAYS_RUN_ALL)) return true;
+  if (input.coverage === undefined || input.coverage.length === 0) return true;
+  return anyMatch(input.changedFiles, input.coverage);
+}

test/crossagent/mcpmerge.ts

@@ -43,4 +43,6 @@ export const test: TestRunnerOptions = {
   },
   repoSetup:
     'mkdir -p /tmp/pullfrog-mcp-secret && printf "%s" "$PULLFROG_MCP_SECRET" > /tmp/pullfrog-mcp-secret/secret.txt',
+  // any MCP-layer change can affect repo-MCP merging; agents own MCP wiring.
+  coverage: ["action/mcp/**", "action/agents/{claude,opencode}.ts"],
 };

test/crossagent/nobash.ts

@@ -43,4 +43,5 @@ export const test: TestRunnerOptions = {
   validator,
   agentEnv,
   env: { PULLFROG_DISABLE_SECURITY_INSTRUCTIONS: "1" },
+  coverage: ["action/mcp/shell.ts", "action/agents/{claude,opencode}.ts"],
 };

test/crossagent/restricted.ts

@@ -52,4 +52,9 @@ export const test: TestRunnerOptions = {
   validator,
   agentEnv,
   env: { PULLFROG_DISABLE_SECURITY_INSTRUCTIONS: "1" },
+  coverage: [
+    "action/utils/normalizeEnv.ts",
+    "action/mcp/shell.ts",
+    "action/agents/{claude,opencode}.ts",
+  ],
 };

test/crossagent/skillInvokeClaude.ts

@@ -44,4 +44,5 @@ export const test: TestRunnerOptions = {
     PULLFROG_DISABLE_SECURITY_INSTRUCTIONS: "1",
     PULLFROG_MODEL: "anthropic/claude-sonnet-4-6",
   },
+  coverage: ["action/agents/claude.ts"],
 };

test/crossagent/skillInvokeOpencode.ts

@@ -44,4 +44,5 @@ export const test: TestRunnerOptions = {
     PULLFROG_DISABLE_SECURITY_INSTRUCTIONS: "1",
     PULLFROG_MODEL: "anthropic/claude-sonnet-4-6",
   },
+  coverage: ["action/agents/opencode.ts", "action/agents/opencodePlugin.ts"],
 };

test/crossagent/smoke.ts

@@ -29,4 +29,7 @@ export const test: TestRunnerOptions = {
   fixture,
   validator,
   env: { PULLFROG_DISABLE_SECURITY_INSTRUCTIONS: "1" },
+  // canary: any agent harness change runs the smoke. shared MCP set_output
+  // surface is also captured.
+  coverage: ["action/agents/{claude,opencode}.ts", "action/mcp/output.ts"],
 };

test/crossagent/tokenExfil.ts

@@ -58,4 +58,9 @@ export const test: TestRunnerOptions = {
   validator,
   agentEnv,
   env: { PULLFROG_DISABLE_SECURITY_INSTRUCTIONS: "1" },
+  coverage: [
+    "action/utils/normalizeEnv.ts",
+    "action/mcp/shell.ts",
+    "action/agents/{claude,opencode}.ts",
+  ],
 };

test/list-aliases.ts

@@ -1,7 +1,7 @@
 /**
  * emits a JSON array of { slug, agent, name } entries for one of two CI matrix
  * jobs. `agent` mirrors the harness the runtime would pick in production
- * (anthropic/* → claude-code, everything else → opencode).
+ * (anthropic/* → claude, everything else → opencode).
  *
  * MODE=aliases (default) — every alias minus pruned passthroughs. consumed by
  *   `models-live`, which runs the cheap top-level CLI smoke per alias
@@ -10,7 +10,8 @@
  * MODE=flagships — one standard-tier model per provider. consumed by
  *   `providers-live`, which runs the full harness smoke
  *   (`pnpm runtest smoke <agent>`) to validate provider-class tool-calling
- *   (e.g. Gemini schema sanitizer, OpenAI tool-call format).
+ *   (e.g. Gemini schema sanitizer, OpenAI tool-call format). flagship slugs
+ *   live in `providers.ts` alongside their per-provider coverage globs.
  *
  * passthrough pruning (aliases mode): openrouter/* aliases and keyed opencode/*
  * aliases are routing-layer wrappers around models we already smoke-test
@@ -24,29 +25,15 @@
  *   MODE=flagships node action/test/list-aliases.ts
  *   MATRIX_FILTER=gemini node action/test/list-aliases.ts
  *   INCLUDE_PASSTHROUGHS=1 node action/test/list-aliases.ts
+ *
+ * NOTE: the per-PR-precision matrix lives in `matrix.ts`, which calls into
+ * this file. raw invocation here emits the unfiltered matrix.
  */
 import { modelAliases } from "../models.ts";
+import { providers } from "./providers.ts";
 
 const ROUTING_CANARIES = new Set(["openrouter/claude-sonnet", "opencode/claude-sonnet"]);
 
-// hand-picked "standard good model" per provider — not the pro/opus tier (too
-// expensive for per-push) and not the free/experimental tier (too flaky). these
-// aliases anchor the harness smoke job that catches provider-class regressions
-// like Gemini schema sanitization or OpenAI tool-call format drift. the
-// assertion below catches slug-drift loudly, but adding a NEW provider without
-// an entry here silently omits it from `providers-live` — see
-// wiki/models-catalog.md "To add a provider".
-const FLAGSHIPS = [
-  "anthropic/claude-sonnet",
-  "openai/gpt",
-  "google/gemini-pro",
-  "xai/grok",
-  "deepseek/deepseek-pro",
-  "moonshotai/kimi-k2",
-  "opencode/big-pickle",
-  "openrouter/claude-sonnet",
-];
-
 function isPrunablePassthrough(alias: (typeof modelAliases)[number]): boolean {
   if (ROUTING_CANARIES.has(alias.slug)) return false;
   if (alias.provider === "openrouter") return true;
@@ -59,7 +46,13 @@ function isPrunablePassthrough(alias: (typeof modelAliases)[number]): boolean {
   return alias.provider === "opencode" && !alias.isFree;
 }
 
-function toMatrixEntry(alias: (typeof modelAliases)[number]) {
+export type MatrixEntry = {
+  slug: string;
+  agent: string;
+  name: string;
+};
+
+function toMatrixEntry(alias: (typeof modelAliases)[number]): MatrixEntry {
   return {
     slug: alias.slug,
     agent: alias.slug.startsWith("anthropic/") ? "claude" : "opencode",
@@ -68,25 +61,14 @@ function toMatrixEntry(alias: (typeof modelAliases)[number]) {
   };
 }
 
-const mode = process.env.MODE === "flagships" ? "flagships" : "aliases";
-const filter = process.env.MATRIX_FILTER?.trim().toLowerCase() ?? "";
-const includePassthroughs = process.env.INCLUDE_PASSTHROUGHS === "1";
-
 const aliasBySlug = new Map(modelAliases.map((a) => [a.slug, a]));
-const matrix = (() => {
-  if (mode === "flagships") {
-    return FLAGSHIPS.map((slug) => {
-      const alias = aliasBySlug.get(slug);
-      if (!alias) {
-        throw new Error(
-          `list-aliases: flagship "${slug}" missing from modelAliases — update FLAGSHIPS`
-        );
-      }
-      return alias;
-    })
-      .filter((alias) => !filter || alias.slug.toLowerCase().includes(filter))
-      .map(toMatrixEntry);
-  }
+
+export function buildAliasMatrix(opts: {
+  filter?: string;
+  includePassthroughs?: boolean;
+}): MatrixEntry[] {
+  const filter = opts.filter ?? "";
+  const includePassthroughs = opts.includePassthroughs ?? false;
   return modelAliases
     .filter((alias) => {
       if (filter && !alias.slug.toLowerCase().includes(filter)) return false;
@@ -94,6 +76,31 @@ const matrix = (() => {
       return true;
     })
     .map(toMatrixEntry);
-})();
+}
 
-process.stdout.write(JSON.stringify(matrix));
+export function buildFlagshipMatrix(opts: { filter?: string }): MatrixEntry[] {
+  const filter = opts.filter ?? "";
+  return providers
+    .map((p) => {
+      const alias = aliasBySlug.get(p.flagship);
+      if (!alias) {
+        throw new Error(
+          `list-aliases: flagship "${p.flagship}" missing from modelAliases — update providers.ts`
+        );
+      }
+      return alias;
+    })
+    .filter((alias) => !filter || alias.slug.toLowerCase().includes(filter))
+    .map(toMatrixEntry);
+}
+
+if (import.meta.url === `file://${process.argv[1]}`) {
+  const mode = process.env.MODE === "flagships" ? "flagships" : "aliases";
+  const filter = process.env.MATRIX_FILTER?.trim().toLowerCase() ?? "";
+  const includePassthroughs = process.env.INCLUDE_PASSTHROUGHS === "1";
+  const matrix =
+    mode === "flagships"
+      ? buildFlagshipMatrix({ filter })
+      : buildAliasMatrix({ filter, includePassthroughs });
+  process.stdout.write(JSON.stringify(matrix));
+}

test/matrix.ts

@@ -0,0 +1,232 @@
+/**
+ * unified CI matrix builder. emits the four matrices consumed by
+ * `.github/workflows/test.yml`:
+ *
+ *   - agents:    crossagent tests × eligible agents (fan-out)
+ *   - agnostic:  agnostic infrastructure tests (run with opencode)
+ *   - flagships: one harness smoke per provider (providers-live)
+ *   - aliases:   one CLI smoke per model alias (models-live)
+ *
+ * input: a JSON array of repo-relative changed paths on stdin (the
+ * `paths-filter` action's `*_files` output). PR pushes pass the diff;
+ * `main` pushes and `workflow_dispatch` set FULL=1 to skip filtering and
+ * emit every entry.
+ *
+ * each test/provider declares its own `coverage` globs colocated with the
+ * test (`crossagent/`, `agnostic/`) or provider (`providers.ts`). the matrix
+ * builder intersects coverage against the diff. a top-level `ALWAYS_RUN_ALL`
+ * (see `coverage.ts`) bypasses filtering when test-harness or cross-cutting
+ * agent code changes — keeps stale globs from silently skipping critical
+ * tests on test runner / shared.ts churn.
+ *
+ * usage:
+ *   echo '["action/agents/opencode.ts"]' | node action/test/matrix.ts
+ *   FULL=1 node action/test/matrix.ts < /dev/null
+ *   MATRIX_FILTER=gemini FULL=1 node action/test/matrix.ts < /dev/null
+ */
+
+import { existsSync, readdirSync, readFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { shouldRun } from "./coverage.ts";
+import { buildAliasMatrix, buildFlagshipMatrix } from "./list-aliases.ts";
+import { providers } from "./providers.ts";
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+
+type AgentEntry = { agent: string; test: string; name: string };
+type AgnosticEntry = { test: string; name: string };
+type SlugEntry = { slug: string; agent: string; name: string };
+
+type MatrixOutput = {
+  agents: AgentEntry[];
+  agnostic: AgnosticEntry[];
+  flagships: SlugEntry[];
+  aliases: SlugEntry[];
+};
+
+/**
+ * extracted test metadata. parsed via regex from the test source — see
+ * `parseTestFile`. dynamic-import is intentionally avoided: the GHA `changes`
+ * job runs without `pnpm install`, and the real test modules transitively
+ * import `@actions/core` etc. parsing keeps `matrix.ts` zero-dep.
+ */
+type ParsedTest = {
+  name: string;
+  agents: string[] | undefined;
+  coverage: string[] | undefined;
+};
+
+const STRING_LITERAL = /"((?:\\.|[^"\\])*)"/g;
+
+function extractStringLiterals(source: string): string[] {
+  const out: string[] = [];
+  STRING_LITERAL.lastIndex = 0;
+  let m: RegExpExecArray | null;
+  // biome-ignore lint/suspicious/noAssignInExpressions: idiomatic regex iteration
+  while ((m = STRING_LITERAL.exec(source))) {
+    out.push(m[1]);
+  }
+  return out;
+}
+
+/**
+ * extract a `key: [...]` array literal of strings from a test object. matches
+ * line-leading indented `key:` to avoid colliding with the same word inside
+ * prompts / template literals.
+ */
+function extractStringArray(source: string, key: string): string[] | undefined {
+  const re = new RegExp(`^\\s+${key}:\\s*\\[([\\s\\S]*?)\\]`, "m");
+  const m = source.match(re);
+  if (!m) return undefined;
+  return extractStringLiterals(m[1]);
+}
+
+function parseTestFile(source: string): ParsedTest | null {
+  // strip line comments — `//` inside string literals is rare in test files,
+  // and the static parser doesn't need to be perfect (defensive default of
+  // "missing coverage = always run" covers parse misses).
+  const stripped = source.replace(/\/\/[^\n]*$/gm, "");
+  const nameMatch = stripped.match(/^\s+name:\s*"([^"]+)"/m);
+  if (!nameMatch) return null;
+  return {
+    name: nameMatch[1],
+    agents: extractStringArray(stripped, "agents"),
+    coverage: extractStringArray(stripped, "coverage"),
+  };
+}
+
+function loadDir(dir: string): ParsedTest[] {
+  const dirPath = join(__dirname, dir);
+  if (!existsSync(dirPath)) return [];
+  const files = readdirSync(dirPath).filter((f) => f.endsWith(".ts"));
+  const out: ParsedTest[] = [];
+  for (const file of files) {
+    const source = readFileSync(join(dirPath, file), "utf8");
+    const parsed = parseTestFile(source);
+    if (parsed) out.push(parsed);
+  }
+  return out;
+}
+
+/**
+ * derive the active agent list from `agents/index.ts` so adding a new harness
+ * file automatically wires it into the matrix. avoids dynamic-import
+ * (transitively pulls `@actions/core` etc. — would explode in the no-install
+ * `changes` job) by regex-parsing the imports the same way `parseTestFile`
+ * handles tests.
+ */
+function loadAgents(): string[] {
+  const indexPath = join(__dirname, "..", "agents", "index.ts");
+  const source = readFileSync(indexPath, "utf8");
+  const out: string[] = [];
+  const re = /^\s*import\s+\{\s*(\w+)\s*\}\s+from\s+"\.\/(\w+)\.ts"/gm;
+  let m: RegExpExecArray | null;
+  // biome-ignore lint/suspicious/noAssignInExpressions: idiomatic regex iteration
+  while ((m = re.exec(source))) {
+    if (m[2] === "shared") continue;
+    out.push(m[1]);
+  }
+  return out.sort();
+}
+
+function readChangedFiles(): string[] {
+  const raw = readFileSync(0, "utf8").trim();
+  if (!raw) return [];
+  const parsed: unknown = JSON.parse(raw);
+  if (!Array.isArray(parsed)) {
+    throw new Error("matrix: stdin must be a JSON array of changed paths");
+  }
+  return parsed.map((p) => {
+    if (typeof p !== "string") {
+      throw new Error(`matrix: non-string entry in changed paths: ${JSON.stringify(p)}`);
+    }
+    return p;
+  });
+}
+
+function buildAgentsMatrix(input: { changedFiles: string[]; full: boolean }): AgentEntry[] {
+  const tests = loadDir("crossagent");
+  const allAgents = loadAgents();
+  const out: AgentEntry[] = [];
+  for (const t of tests) {
+    if (!shouldRun({ changedFiles: input.changedFiles, coverage: t.coverage, full: input.full })) {
+      continue;
+    }
+    const agents = t.agents ?? allAgents;
+    for (const agent of agents) {
+      out.push({ agent, test: t.name, name: `${t.name}-${agent}` });
+    }
+  }
+  return out;
+}
+
+function buildAgnosticMatrix(input: { changedFiles: string[]; full: boolean }): AgnosticEntry[] {
+  const tests = loadDir("agnostic");
+  const out: AgnosticEntry[] = [];
+  for (const t of tests) {
+    if (!shouldRun({ changedFiles: input.changedFiles, coverage: t.coverage, full: input.full })) {
+      continue;
+    }
+    out.push({ test: t.name, name: t.name });
+  }
+  return out;
+}
+
+function buildFlagshipsMatrix(input: {
+  changedFiles: string[];
+  full: boolean;
+  filter: string;
+}): SlugEntry[] {
+  const all = buildFlagshipMatrix({ filter: input.filter });
+  const byName = new Map(providers.map((p) => [p.flagship, p]));
+  return all.filter((entry) => {
+    const provider = byName.get(entry.slug);
+    return shouldRun({
+      changedFiles: input.changedFiles,
+      coverage: provider?.coverage,
+      full: input.full,
+    });
+  });
+}
+
+function buildAliasesMatrix(input: {
+  changedFiles: string[];
+  full: boolean;
+  filter: string;
+  includePassthroughs: boolean;
+}): SlugEntry[] {
+  const all = buildAliasMatrix({
+    filter: input.filter,
+    includePassthroughs: input.includePassthroughs,
+  });
+  const coverageByProvider = new Map(providers.map((p) => [p.name, p.coverage]));
+  return all.filter((entry) => {
+    const provider = entry.slug.split("/")[0];
+    return shouldRun({
+      changedFiles: input.changedFiles,
+      coverage: coverageByProvider.get(provider),
+      full: input.full,
+    });
+  });
+}
+
+function main(): void {
+  const full = process.env.FULL === "1";
+  const filter = process.env.MATRIX_FILTER?.trim().toLowerCase() ?? "";
+  const includePassthroughs = process.env.INCLUDE_PASSTHROUGHS === "1";
+  const changedFiles = full ? [] : readChangedFiles();
+
+  const output: MatrixOutput = {
+    agents: buildAgentsMatrix({ changedFiles, full }),
+    agnostic: buildAgnosticMatrix({ changedFiles, full }),
+    flagships: buildFlagshipsMatrix({ changedFiles, full, filter }),
+    aliases: buildAliasesMatrix({ changedFiles, full, filter, includePassthroughs }),
+  };
+
+  process.stdout.write(JSON.stringify(output));
+}
+
+if (import.meta.url === `file://${process.argv[1]}`) {
+  main();
+}

test/providers.ts

@@ -0,0 +1,88 @@
+/**
+ * provider catalog — the source of truth for `providers-live` (full harness
+ * smoke per provider) and the per-provider coverage globs that scope `models-live`
+ * (per-alias CLI smoke).
+ *
+ * each entry pins one standard-tier flagship slug per provider — not the
+ * pro/opus tier (too expensive for per-push) and not the free/experimental
+ * tier (too flaky). these flagships catch provider-class regressions like
+ * Gemini schema sanitization or OpenAI tool-call format drift that the cheap
+ * per-alias CLI smoke can't see.
+ *
+ * `coverage` lists the source files that, when changed, should rerun this
+ * provider's flagship + every alias of this provider. `action/models.ts` is
+ * included on every entry — touching the resolution table reruns all model
+ * tests (simple model; matches the per-PR-precision answer from planning).
+ *
+ * adding a new provider:
+ *   1. add an entry here with the flagship slug, agent harness, coverage globs
+ *   2. add a row to wiki/models-catalog.md "To add a provider"
+ *   3. CI picks it up automatically — no workflow change
+ */
+
+export type ProviderEntry = {
+  name: string;
+  /** flagship slug for `providers-live` full-harness smoke. */
+  flagship: string;
+  /** harness used by the runtime for this provider's models. */
+  agent: "claude" | "opencode";
+  /** repo-relative globs that invalidate this provider's matrix entries. */
+  coverage: string[];
+};
+
+const SHARED_OPENCODE_COVERAGE = [
+  "action/models.ts",
+  "action/agents/opencode.ts",
+  "action/agents/opencodePlugin.ts",
+];
+
+export const providers: ProviderEntry[] = [
+  {
+    name: "anthropic",
+    flagship: "anthropic/claude-sonnet",
+    agent: "claude",
+    coverage: ["action/models.ts", "action/agents/claude.ts"],
+  },
+  {
+    name: "openai",
+    flagship: "openai/gpt",
+    agent: "opencode",
+    coverage: SHARED_OPENCODE_COVERAGE,
+  },
+  {
+    name: "google",
+    flagship: "google/gemini-pro",
+    agent: "opencode",
+    coverage: [...SHARED_OPENCODE_COVERAGE, "action/mcp/geminiSanitizer.ts"],
+  },
+  {
+    name: "xai",
+    flagship: "xai/grok",
+    agent: "opencode",
+    coverage: SHARED_OPENCODE_COVERAGE,
+  },
+  {
+    name: "deepseek",
+    flagship: "deepseek/deepseek-pro",
+    agent: "opencode",
+    coverage: SHARED_OPENCODE_COVERAGE,
+  },
+  {
+    name: "moonshotai",
+    flagship: "moonshotai/kimi-k2",
+    agent: "opencode",
+    coverage: SHARED_OPENCODE_COVERAGE,
+  },
+  {
+    name: "opencode",
+    flagship: "opencode/big-pickle",
+    agent: "opencode",
+    coverage: SHARED_OPENCODE_COVERAGE,
+  },
+  {
+    name: "openrouter",
+    flagship: "openrouter/claude-sonnet",
+    agent: "opencode",
+    coverage: SHARED_OPENCODE_COVERAGE,
+  },
+];

test/utils.ts

@@ -328,6 +328,10 @@ export interface TestRunnerOptions {
   //   - "agnostic": runs with opencode only, excluded when filtering by agent
   //   - "adhoc": excluded from default runs, must be explicitly requested
   tags?: TestTag[];
+  // repo-relative globs of source files that, when changed in a PR, should
+  // trigger this test in CI. omit to opt out of filtering (test always runs
+  // — the defensive default). see action/test/coverage.ts.
+  coverage?: string[];
 }
 
 export type TestTag = "adhoc" | "agnostic" | "security";