ci: filter test matrices by per-test coverage globs (#730)

* ci: filter test matrices by per-test coverage globs to cut LLM spend every test in `crossagent/`, `agnostic/`, and every provider entry now declares a `coverage: string[]` of repo-relative globs. the new `changes` job runs `paths-filter` for a docs-only short-circuit, then pipes the changed-file list into `action/test/matrix.ts`, which intersects each entry's coverage against the diff and emits filtered `agents`, `agnostic`, `flagships`, and `aliases` matrices. main pushes and `workflow_dispatch` set `FULL=1` to run everything as a stale-glob safety net. retires `changed-agents.sh` and the `MODE=flagships` branch in `list-aliases.ts` in favor of one consistent model. * ci(matrix): switch test discovery to dep-free static parsing the GHA `changes` job has no `node_modules` installed. the previous dynamic-import path pulled the test files transitively through `utils.ts` -> `agents/index.ts` -> `@actions/core`, which exploded with ERR_MODULE_NOT_FOUND. parse the test files via regex instead so matrix.ts stays zero-dep — the chain (matrix -> coverage / providers / list-aliases / models) imports only node builtins and relative TS files. * ci(matrix): address PR #730 review feedback - drop dangling `action/mcp/toolFiltering.ts` glob from `nobash`, `restricted`, `tokenExfil` (file doesn't exist; `.test.ts` does, but the runtime tooling lives in `mcp/shell.ts` and `agents/{claude,opencode}.ts`, both already covered). - drop unused `coverageForProvider` export and its `byName` map from `providers.ts` (matrix.ts builds its own lookup inline). - derive the active agent list from `agents/index.ts` via the same dep-free regex tactic as `parseTestFile` instead of hardcoding `["claude", "opencode"]` — adding a new harness file now wires it into the dynamic matrix automatically. - treat `coverage: []` as `coverage: undefined` in `shouldRun` so an accidentally-empty array doesn't silently skip CI on every PR. - add `action/utils/activity.ts` and `action/mcp/selectMode.ts` to the `timeout` test's coverage — the activity-timeout enforcement path was the original reason the test exists. - ungate the `root` job (lint/format/typecheck/vitest). it's a required status check on `main`, so gating it on `code == 'true'` would make docs-only PRs unmergeable (skipped jobs don't satisfy required-check rules). the real LLM savings come from skipping the four matrices, not from skipping `root`. - harden the four matrix-job `if:` guards from `outputs.matrix && ...` to `outputs.matrix != '' && ...` — explicit > implicit short-circuit. - document `expandBraces`'s flat-only support so a future author isn't surprised by `{a,{b,c}}` not expanding. - fix awkward sentence in `wiki/action-tests.md` "CI Cost Filtering".
2026-05-14 03:55:33 +00:00
parent 4ad649ebb9
commit 1f4c3031be
21 changed files with 560 additions and 162 deletions
@@ -0,0 +1,232 @@
+/**
+ * unified CI matrix builder. emits the four matrices consumed by
+ * `.github/workflows/test.yml`:
+ *
+ *   - agents:    crossagent tests × eligible agents (fan-out)
+ *   - agnostic:  agnostic infrastructure tests (run with opencode)
+ *   - flagships: one harness smoke per provider (providers-live)
+ *   - aliases:   one CLI smoke per model alias (models-live)
+ *
+ * input: a JSON array of repo-relative changed paths on stdin (the
+ * `paths-filter` action's `*_files` output). PR pushes pass the diff;
+ * `main` pushes and `workflow_dispatch` set FULL=1 to skip filtering and
+ * emit every entry.
+ *
+ * each test/provider declares its own `coverage` globs colocated with the
+ * test (`crossagent/`, `agnostic/`) or provider (`providers.ts`). the matrix
+ * builder intersects coverage against the diff. a top-level `ALWAYS_RUN_ALL`
+ * (see `coverage.ts`) bypasses filtering when test-harness or cross-cutting
+ * agent code changes — keeps stale globs from silently skipping critical
+ * tests on test runner / shared.ts churn.
+ *
+ * usage:
+ *   echo '["action/agents/opencode.ts"]' | node action/test/matrix.ts
+ *   FULL=1 node action/test/matrix.ts < /dev/null
+ *   MATRIX_FILTER=gemini FULL=1 node action/test/matrix.ts < /dev/null
+ */
+
+import { existsSync, readdirSync, readFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { shouldRun } from "./coverage.ts";
+import { buildAliasMatrix, buildFlagshipMatrix } from "./list-aliases.ts";
+import { providers } from "./providers.ts";
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+
+type AgentEntry = { agent: string; test: string; name: string };
+type AgnosticEntry = { test: string; name: string };
+type SlugEntry = { slug: string; agent: string; name: string };
+
+type MatrixOutput = {
+  agents: AgentEntry[];
+  agnostic: AgnosticEntry[];
+  flagships: SlugEntry[];
+  aliases: SlugEntry[];
+};
+
+/**
+ * extracted test metadata. parsed via regex from the test source — see
+ * `parseTestFile`. dynamic-import is intentionally avoided: the GHA `changes`
+ * job runs without `pnpm install`, and the real test modules transitively
+ * import `@actions/core` etc. parsing keeps `matrix.ts` zero-dep.
+ */
+type ParsedTest = {
+  name: string;
+  agents: string[] | undefined;
+  coverage: string[] | undefined;
+};
+
+const STRING_LITERAL = /"((?:\\.|[^"\\])*)"/g;
+
+function extractStringLiterals(source: string): string[] {
+  const out: string[] = [];
+  STRING_LITERAL.lastIndex = 0;
+  let m: RegExpExecArray | null;
+  // biome-ignore lint/suspicious/noAssignInExpressions: idiomatic regex iteration
+  while ((m = STRING_LITERAL.exec(source))) {
+    out.push(m[1]);
+  }
+  return out;
+}
+
+/**
+ * extract a `key: [...]` array literal of strings from a test object. matches
+ * line-leading indented `key:` to avoid colliding with the same word inside
+ * prompts / template literals.
+ */
+function extractStringArray(source: string, key: string): string[] | undefined {
+  const re = new RegExp(`^\\s+${key}:\\s*\\[([\\s\\S]*?)\\]`, "m");
+  const m = source.match(re);
+  if (!m) return undefined;
+  return extractStringLiterals(m[1]);
+}
+
+function parseTestFile(source: string): ParsedTest | null {
+  // strip line comments — `//` inside string literals is rare in test files,
+  // and the static parser doesn't need to be perfect (defensive default of
+  // "missing coverage = always run" covers parse misses).
+  const stripped = source.replace(/\/\/[^\n]*$/gm, "");
+  const nameMatch = stripped.match(/^\s+name:\s*"([^"]+)"/m);
+  if (!nameMatch) return null;
+  return {
+    name: nameMatch[1],
+    agents: extractStringArray(stripped, "agents"),
+    coverage: extractStringArray(stripped, "coverage"),
+  };
+}
+
+function loadDir(dir: string): ParsedTest[] {
+  const dirPath = join(__dirname, dir);
+  if (!existsSync(dirPath)) return [];
+  const files = readdirSync(dirPath).filter((f) => f.endsWith(".ts"));
+  const out: ParsedTest[] = [];
+  for (const file of files) {
+    const source = readFileSync(join(dirPath, file), "utf8");
+    const parsed = parseTestFile(source);
+    if (parsed) out.push(parsed);
+  }
+  return out;
+}
+
+/**
+ * derive the active agent list from `agents/index.ts` so adding a new harness
+ * file automatically wires it into the matrix. avoids dynamic-import
+ * (transitively pulls `@actions/core` etc. — would explode in the no-install
+ * `changes` job) by regex-parsing the imports the same way `parseTestFile`
+ * handles tests.
+ */
+function loadAgents(): string[] {
+  const indexPath = join(__dirname, "..", "agents", "index.ts");
+  const source = readFileSync(indexPath, "utf8");
+  const out: string[] = [];
+  const re = /^\s*import\s+\{\s*(\w+)\s*\}\s+from\s+"\.\/(\w+)\.ts"/gm;
+  let m: RegExpExecArray | null;
+  // biome-ignore lint/suspicious/noAssignInExpressions: idiomatic regex iteration
+  while ((m = re.exec(source))) {
+    if (m[2] === "shared") continue;
+    out.push(m[1]);
+  }
+  return out.sort();
+}
+
+function readChangedFiles(): string[] {
+  const raw = readFileSync(0, "utf8").trim();
+  if (!raw) return [];
+  const parsed: unknown = JSON.parse(raw);
+  if (!Array.isArray(parsed)) {
+    throw new Error("matrix: stdin must be a JSON array of changed paths");
+  }
+  return parsed.map((p) => {
+    if (typeof p !== "string") {
+      throw new Error(`matrix: non-string entry in changed paths: ${JSON.stringify(p)}`);
+    }
+    return p;
+  });
+}
+
+function buildAgentsMatrix(input: { changedFiles: string[]; full: boolean }): AgentEntry[] {
+  const tests = loadDir("crossagent");
+  const allAgents = loadAgents();
+  const out: AgentEntry[] = [];
+  for (const t of tests) {
+    if (!shouldRun({ changedFiles: input.changedFiles, coverage: t.coverage, full: input.full })) {
+      continue;
+    }
+    const agents = t.agents ?? allAgents;
+    for (const agent of agents) {
+      out.push({ agent, test: t.name, name: `${t.name}-${agent}` });
+    }
+  }
+  return out;
+}
+
+function buildAgnosticMatrix(input: { changedFiles: string[]; full: boolean }): AgnosticEntry[] {
+  const tests = loadDir("agnostic");
+  const out: AgnosticEntry[] = [];
+  for (const t of tests) {
+    if (!shouldRun({ changedFiles: input.changedFiles, coverage: t.coverage, full: input.full })) {
+      continue;
+    }
+    out.push({ test: t.name, name: t.name });
+  }
+  return out;
+}
+
+function buildFlagshipsMatrix(input: {
+  changedFiles: string[];
+  full: boolean;
+  filter: string;
+}): SlugEntry[] {
+  const all = buildFlagshipMatrix({ filter: input.filter });
+  const byName = new Map(providers.map((p) => [p.flagship, p]));
+  return all.filter((entry) => {
+    const provider = byName.get(entry.slug);
+    return shouldRun({
+      changedFiles: input.changedFiles,
+      coverage: provider?.coverage,
+      full: input.full,
+    });
+  });
+}
+
+function buildAliasesMatrix(input: {
+  changedFiles: string[];
+  full: boolean;
+  filter: string;
+  includePassthroughs: boolean;
+}): SlugEntry[] {
+  const all = buildAliasMatrix({
+    filter: input.filter,
+    includePassthroughs: input.includePassthroughs,
+  });
+  const coverageByProvider = new Map(providers.map((p) => [p.name, p.coverage]));
+  return all.filter((entry) => {
+    const provider = entry.slug.split("/")[0];
+    return shouldRun({
+      changedFiles: input.changedFiles,
+      coverage: coverageByProvider.get(provider),
+      full: input.full,
+    });
+  });
+}
+
+function main(): void {
+  const full = process.env.FULL === "1";
+  const filter = process.env.MATRIX_FILTER?.trim().toLowerCase() ?? "";
+  const includePassthroughs = process.env.INCLUDE_PASSTHROUGHS === "1";
+  const changedFiles = full ? [] : readChangedFiles();
+
+  const output: MatrixOutput = {
+    agents: buildAgentsMatrix({ changedFiles, full }),
+    agnostic: buildAgnosticMatrix({ changedFiles, full }),
+    flagships: buildFlagshipsMatrix({ changedFiles, full, filter }),
+    aliases: buildAliasesMatrix({ changedFiles, full, filter, includePassthroughs }),
+  };
+
+  process.stdout.write(JSON.stringify(output));
+}
+
+if (import.meta.url === `file://${process.argv[1]}`) {
+  main();
+}