diff --git a/test/agnostic/gitHooks.ts b/test/agnostic/gitHooks.ts index 477cdb3..1ba5df4 100644 --- a/test/agnostic/gitHooks.ts +++ b/test/agnostic/gitHooks.ts @@ -96,4 +96,10 @@ export const test: TestRunnerOptions = { repoSetup, env: { PULLFROG_DISABLE_SECURITY_INSTRUCTIONS: "1" }, tags: ["agnostic", "security"], + coverage: [ + "action/utils/gitAuth.ts", + "action/utils/gitAuthServer.ts", + "action/mcp/git.ts", + "action/mcp/checkout.ts", + ], }; diff --git a/test/agnostic/gitPerms.ts b/test/agnostic/gitPerms.ts index 328268c..ec2f07c 100644 --- a/test/agnostic/gitPerms.ts +++ b/test/agnostic/gitPerms.ts @@ -104,4 +104,10 @@ export const test: TestRunnerOptions = { agentEnv, env: { PULLFROG_DISABLE_SECURITY_INSTRUCTIONS: "1" }, tags: ["agnostic"], + coverage: [ + "action/utils/gitAuth.ts", + "action/utils/gitAuthServer.ts", + "action/mcp/git.ts", + "action/mcp/checkout.ts", + ], }; diff --git a/test/agnostic/packageJsonScripts.ts b/test/agnostic/packageJsonScripts.ts index f14e1d7..517c982 100644 --- a/test/agnostic/packageJsonScripts.ts +++ b/test/agnostic/packageJsonScripts.ts @@ -92,4 +92,5 @@ export const test: TestRunnerOptions = { validator, env: { PULLFROG_DISABLE_SECURITY_INSTRUCTIONS: "1" }, tags: ["agnostic", "security"], + coverage: ["action/mcp/dependencies.ts", "action/utils/install.ts"], }; diff --git a/test/agnostic/pushDisabled.ts b/test/agnostic/pushDisabled.ts index 4f1ccb0..b3b7c6c 100644 --- a/test/agnostic/pushDisabled.ts +++ b/test/agnostic/pushDisabled.ts @@ -62,4 +62,10 @@ export const test: TestRunnerOptions = { agentEnv, env: { PULLFROG_DISABLE_SECURITY_INSTRUCTIONS: "1" }, tags: ["agnostic"], + coverage: [ + "action/utils/gitAuth.ts", + "action/utils/gitAuthServer.ts", + "action/mcp/git.ts", + "action/mcp/checkout.ts", + ], }; diff --git a/test/agnostic/pushEnabled.ts b/test/agnostic/pushEnabled.ts index 68d68b6..d5c36f3 100644 --- a/test/agnostic/pushEnabled.ts +++ b/test/agnostic/pushEnabled.ts @@ -74,4 +74,10 @@ export const test: TestRunnerOptions = { validator, env: { PULLFROG_DISABLE_SECURITY_INSTRUCTIONS: "1" }, tags: ["agnostic"], + coverage: [ + "action/utils/gitAuth.ts", + "action/utils/gitAuthServer.ts", + "action/mcp/git.ts", + "action/mcp/checkout.ts", + ], }; diff --git a/test/agnostic/pushRestricted.ts b/test/agnostic/pushRestricted.ts index 1de5950..17011fb 100644 --- a/test/agnostic/pushRestricted.ts +++ b/test/agnostic/pushRestricted.ts @@ -67,4 +67,10 @@ export const test: TestRunnerOptions = { validator, env: { PULLFROG_DISABLE_SECURITY_INSTRUCTIONS: "1" }, tags: ["agnostic"], + coverage: [ + "action/utils/gitAuth.ts", + "action/utils/gitAuthServer.ts", + "action/mcp/git.ts", + "action/mcp/checkout.ts", + ], }; diff --git a/test/agnostic/timeout.ts b/test/agnostic/timeout.ts index c5bf641..b8b2a62 100644 --- a/test/agnostic/timeout.ts +++ b/test/agnostic/timeout.ts @@ -29,4 +29,11 @@ export const test: TestRunnerOptions = { expectFailure: true, env: { PULLFROG_DISABLE_SECURITY_INSTRUCTIONS: "1" }, tags: ["agnostic"], + coverage: [ + "action/utils/timer.ts", + "action/utils/subprocess.ts", + "action/utils/exitHandler.ts", + "action/utils/activity.ts", + "action/mcp/selectMode.ts", + ], }; diff --git a/test/changed-agents.sh b/test/changed-agents.sh deleted file mode 100755 index 4cf6537..0000000 --- a/test/changed-agents.sh +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/env bash -# determines which agents need testing based on changed files. -# reads changed file paths from stdin (JSON array or newline-delimited). -# outputs a JSON array of agent names to stdout. -# -# only agents whose harness file changed AND are exported from index.ts are included. -# shared.ts/index.ts/postRun.ts and other non-harness action changes fall back to opencode as a canary. -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -AGENTS_INDEX="$SCRIPT_DIR/../agents/index.ts" - -# build the set of active agents from index.ts imports (portable, no -P) -active_agents=() -while IFS= read -r line; do - [[ -n "$line" ]] && active_agents+=("$line") -done < <(sed -n 's/.*from "\.\/\([^"]*\)\.ts".*/\1/p' "$AGENTS_INDEX" | grep -v shared) - -# read stdin - auto-detect JSON array vs newline-delimited -input=$(cat) -if echo "$input" | jq -e 'type == "array"' > /dev/null 2>&1; then - files=$(echo "$input" | jq -r '.[]') -else - files="$input" -fi - -is_active_agent() { - local name="$1" - for a in "${active_agents[@]}"; do - [[ "$a" == "$name" ]] && return 0 - done - return 1 -} - -# find which agent harness files changed -changed_agents=() -has_non_agent_change=false - -while IFS= read -r file; do - [[ -z "$file" ]] && continue - case "$file" in - action/agents/shared.ts|action/agents/index.ts|action/agents/postRun.ts) - has_non_agent_change=true - ;; - action/agents/*.ts) - agent_name="$(basename "$file" .ts)" - if is_active_agent "$agent_name"; then - changed_agents+=("$agent_name") - else - # legacy/inactive agent file changed — treat as non-agent change - has_non_agent_change=true - fi - ;; - action/*) - has_non_agent_change=true - ;; - esac -done <<< "$files" - -# output agents based on change type. -# non-agent action changes always include opencode as a canary. -if $has_non_agent_change; then - changed_agents+=("opencode") -fi - -if [[ ${#changed_agents[@]} -gt 0 ]]; then - printf '%s\n' "${changed_agents[@]}" | sort -u | jq -R . | jq -sc . -else - echo '[]' -fi diff --git a/test/ci.test.ts b/test/ci.test.ts index 1b9ba18..08c3f9c 100644 --- a/test/ci.test.ts +++ b/test/ci.test.ts @@ -1,4 +1,3 @@ -import { execFileSync } from "node:child_process"; import { readdirSync, readFileSync } from "node:fs"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; @@ -16,7 +15,7 @@ type WorkflowJob = { "runs-on": string; "timeout-minutes"?: number; permissions?: WorkflowPermissions; - strategy?: { "fail-fast": boolean; matrix: Record }; + strategy?: { "fail-fast": boolean; matrix: Record }; env?: Record; steps?: unknown[]; }; @@ -57,7 +56,6 @@ const expectedAgents = Object.keys(agents).sort(); const crossagentTests = getTestNamesFromDir("crossagent"); const agnosticTests = getTestNamesFromDir("agnostic"); const adhocTests = getTestNamesFromDir("adhoc"); -const dynamicAgentsExpression = "$" + "{{ fromJSON(needs.changes.outputs.agents) }}"; // all provider API key names + GITHUB_TOKEN + model overrides const expectedAgentEnvVars = [ @@ -83,53 +81,22 @@ describe("ci workflow consistency", () => { const rootJob = rootWorkflow.jobs["action-agents"]; const actionJob = actionWorkflow.jobs.agents; - it("root agent matrix uses dynamic output from changes job", () => { - expect(rootJob.strategy!.matrix.agent).toBe(dynamicAgentsExpression); - }); - - it("changed-agents.sh falls back to opencode when shared agent code changed", () => { - const input = JSON.stringify(["action/agents/shared.ts"]); - const output = execFileSync("bash", [join(__dirname, "changed-agents.sh")], { - input, - encoding: "utf-8", - }); - expect(JSON.parse(output)).toEqual(["opencode"]); - }); - - it("changed-agents.sh falls back to opencode for non-agent action changes", () => { - const output = execFileSync("bash", [join(__dirname, "changed-agents.sh")], { - input: JSON.stringify(["action/mcp/server.ts"]), - encoding: "utf-8", - }); - expect(JSON.parse(output)).toEqual(["opencode"]); - }); - - it("changed-agents.sh includes opencode canary alongside changed agents", () => { - const output = execFileSync("bash", [join(__dirname, "changed-agents.sh")], { - input: JSON.stringify(["action/agents/opencode.ts", "action/mcp/server.ts"]), - encoding: "utf-8", - }); - expect(JSON.parse(output)).toEqual(["opencode"]); - }); - - it("changed-agents.sh treats legacy agent files as non-agent changes", () => { - const output = execFileSync("bash", [join(__dirname, "changed-agents.sh")], { - input: JSON.stringify(["action/agents/codex.ts", "action/agents/gemini.ts"]), - encoding: "utf-8", - }); - expect(JSON.parse(output)).toEqual(["opencode"]); + it("root agents matrix is wired to the dynamic matrix output", () => { + const include = rootJob.strategy?.matrix.include; + expect(typeof include).toBe("string"); + expect(include as string).toContain("fromJSON(needs.changes.outputs.matrix).agents"); }); it("action agent matrix matches agents map", () => { - expect([...actionJob.strategy!.matrix.agent].sort()).toEqual(expectedAgents); - }); - - it("root test matrix matches crossagent/ directory", () => { - expect([...rootJob.strategy!.matrix.test].sort()).toEqual(crossagentTests); + expect((actionJob.strategy?.matrix.agent as string[])?.slice().sort()).toEqual( + expectedAgents + ); }); it("action test matrix matches crossagent/ directory", () => { - expect([...actionJob.strategy!.matrix.test].sort()).toEqual(crossagentTests); + expect((actionJob.strategy?.matrix.test as string[])?.slice().sort()).toEqual( + crossagentTests + ); }); it("permissions match between root and action", () => { @@ -149,8 +116,8 @@ describe("ci workflow consistency", () => { }); it("fail-fast is enabled in both", () => { - expect(rootJob.strategy!["fail-fast"]).toBe(true); - expect(actionJob.strategy!["fail-fast"]).toBe(true); + expect(rootJob.strategy?.["fail-fast"]).toBe(true); + expect(actionJob.strategy?.["fail-fast"]).toBe(true); }); }); @@ -158,12 +125,14 @@ describe("ci workflow consistency", () => { const rootJob = rootWorkflow.jobs["action-agnostic"]; const actionJob = actionWorkflow.jobs.agnostic; - it("root test matrix matches agnostic/ directory", () => { - expect([...rootJob.strategy!.matrix.test].sort()).toEqual(agnosticTests); + it("root agnostic matrix is wired to the dynamic matrix output", () => { + const include = rootJob.strategy?.matrix.include; + expect(typeof include).toBe("string"); + expect(include as string).toContain("fromJSON(needs.changes.outputs.matrix).agnostic"); }); it("action test matrix matches agnostic/ directory", () => { - expect([...actionJob.strategy!.matrix.test].sort()).toEqual(agnosticTests); + expect((actionJob.strategy?.matrix.test as string[])?.slice().sort()).toEqual(agnosticTests); }); it("permissions match between root and action", () => { @@ -183,8 +152,8 @@ describe("ci workflow consistency", () => { }); it("fail-fast is enabled in both", () => { - expect(rootJob.strategy!["fail-fast"]).toBe(true); - expect(actionJob.strategy!["fail-fast"]).toBe(true); + expect(rootJob.strategy?.["fail-fast"]).toBe(true); + expect(actionJob.strategy?.["fail-fast"]).toBe(true); }); }); }); diff --git a/test/coverage.ts b/test/coverage.ts new file mode 100644 index 0000000..32b75d4 --- /dev/null +++ b/test/coverage.ts @@ -0,0 +1,112 @@ +/** + * shared coverage / glob plumbing for the matrix builder. + * + * every test (`crossagent/`, `agnostic/`) and every provider entry + * (`providers.ts`) declares a `coverage` array of repo-relative globs. on a PR + * push, the `changes` job feeds the changed-file list into `matrix.ts`, which + * intersects each entry's globs against the diff and emits only the entries + * that need to run. + * + * `ALWAYS_RUN_ALL` is the escape hatch: any change to a file matched here + * forces the full matrix (every test, every flagship, every alias). it + * captures cross-cutting infrastructure where fan-out is unpredictable — + * agent loader, MCP server boot, test runner itself. if a per-test glob + * goes stale, this list and the on-`main`-full-matrix policy are the safety + * nets — there's no completeness lint. + * + * `coverage` is optional on tests/providers; missing = always run (treat as + * "any code change touches me"). default to defensive — opt into precision + * by adding globs. + */ + +/** patterns that, when matched by any changed file, force the full matrix. */ +export const ALWAYS_RUN_ALL: string[] = [ + // agent loader + cross-agent shared code + "action/agents/shared.ts", + "action/agents/index.ts", + "action/agents/postRun.ts", + // test harness — changing these can affect every test + "action/test/run.ts", + "action/test/utils.ts", + "action/test/matrix.ts", + "action/test/coverage.ts", + "action/test/providers.ts", + // boot + lifecycle + "action/main.ts", + "action/index.ts", + "action/cli.ts", + "action/utils/setup.ts", + "action/utils/lifecycle.ts", + "action/utils/install.ts", + "action/utils/docker.ts", + "action/utils/globals.ts", + // MCP orchestrator (every test runs through it) + "action/mcp/server.ts", + "action/mcp/shared.ts", + // dependency graph + "action/package.json", + "action/pnpm-lock.yaml", + // workflow itself + ".github/workflows/test.yml", +]; + +/** + * expand a single brace group like `{a,b,c}` into an array of patterns. + * + * intentionally minimal: nested braces (`{a,{b,c}}`) and escaped braces are + * NOT supported — coverage globs in this repo only need flat brace groups + * (`{claude,opencode}.ts`). add complexity if a real use case emerges. + */ +function expandBraces(pattern: string): string[] { + const m = pattern.match(/\{([^{}]+)\}/); + if (!m || m.index === undefined) return [pattern]; + const before = pattern.slice(0, m.index); + const after = pattern.slice(m.index + m[0].length); + const opts = m[1].split(","); + return opts.flatMap((opt) => expandBraces(`${before}${opt}${after}`)); +} + +/** convert a glob pattern to a regex anchored at start + end. */ +function globToRegex(pattern: string): RegExp { + const DSTAR = "\u0000DSTAR\u0000"; + let s = pattern.replace(/\*\*/g, DSTAR); + s = s.replace(/[.+^$()|[\]\\]/g, "\\$&"); + s = s.replace(/\*/g, "[^/]*"); + s = s.replace(/\?/g, "[^/]"); + s = s.replaceAll(DSTAR, ".*"); + return new RegExp(`^${s}$`); +} + +/** does any path in `paths` match any glob in `patterns`? */ +export function anyMatch(paths: string[], patterns: string[]): boolean { + if (patterns.length === 0) return false; + const regexes = patterns.flatMap((p) => expandBraces(p)).map(globToRegex); + return paths.some((path) => regexes.some((r) => r.test(path))); +} + +/** + * decide whether an entry runs given changed files + its coverage globs. + * + * three short-circuits: + * 1. `full` flag (e.g. main pushes, workflow_dispatch) → always run + * 2. any changed file matches `ALWAYS_RUN_ALL` → run everything + * 3. coverage missing or empty on the entry → run (defensive default) + * + * otherwise: run iff any changed file matches the entry's coverage globs. + * + * `coverage: []` is treated identically to `coverage: undefined` to avoid the + * footgun where a future test author intends "skip on PRs" by passing an + * empty array — silently skipping CI on every PR is worse than always running. + */ +export type ShouldRunInput = { + changedFiles: string[]; + coverage: string[] | undefined; + full: boolean; +}; + +export function shouldRun(input: ShouldRunInput): boolean { + if (input.full) return true; + if (anyMatch(input.changedFiles, ALWAYS_RUN_ALL)) return true; + if (input.coverage === undefined || input.coverage.length === 0) return true; + return anyMatch(input.changedFiles, input.coverage); +} diff --git a/test/crossagent/mcpmerge.ts b/test/crossagent/mcpmerge.ts index a7aad81..ded4194 100644 --- a/test/crossagent/mcpmerge.ts +++ b/test/crossagent/mcpmerge.ts @@ -43,4 +43,6 @@ export const test: TestRunnerOptions = { }, repoSetup: 'mkdir -p /tmp/pullfrog-mcp-secret && printf "%s" "$PULLFROG_MCP_SECRET" > /tmp/pullfrog-mcp-secret/secret.txt', + // any MCP-layer change can affect repo-MCP merging; agents own MCP wiring. + coverage: ["action/mcp/**", "action/agents/{claude,opencode}.ts"], }; diff --git a/test/crossagent/nobash.ts b/test/crossagent/nobash.ts index fed3a3d..7b289fc 100644 --- a/test/crossagent/nobash.ts +++ b/test/crossagent/nobash.ts @@ -43,4 +43,5 @@ export const test: TestRunnerOptions = { validator, agentEnv, env: { PULLFROG_DISABLE_SECURITY_INSTRUCTIONS: "1" }, + coverage: ["action/mcp/shell.ts", "action/agents/{claude,opencode}.ts"], }; diff --git a/test/crossagent/restricted.ts b/test/crossagent/restricted.ts index a33daff..659e760 100644 --- a/test/crossagent/restricted.ts +++ b/test/crossagent/restricted.ts @@ -52,4 +52,9 @@ export const test: TestRunnerOptions = { validator, agentEnv, env: { PULLFROG_DISABLE_SECURITY_INSTRUCTIONS: "1" }, + coverage: [ + "action/utils/normalizeEnv.ts", + "action/mcp/shell.ts", + "action/agents/{claude,opencode}.ts", + ], }; diff --git a/test/crossagent/skillInvokeClaude.ts b/test/crossagent/skillInvokeClaude.ts index 442d757..a5fb06c 100644 --- a/test/crossagent/skillInvokeClaude.ts +++ b/test/crossagent/skillInvokeClaude.ts @@ -44,4 +44,5 @@ export const test: TestRunnerOptions = { PULLFROG_DISABLE_SECURITY_INSTRUCTIONS: "1", PULLFROG_MODEL: "anthropic/claude-sonnet-4-6", }, + coverage: ["action/agents/claude.ts"], }; diff --git a/test/crossagent/skillInvokeOpencode.ts b/test/crossagent/skillInvokeOpencode.ts index 647f3bc..ac2ea2d 100644 --- a/test/crossagent/skillInvokeOpencode.ts +++ b/test/crossagent/skillInvokeOpencode.ts @@ -44,4 +44,5 @@ export const test: TestRunnerOptions = { PULLFROG_DISABLE_SECURITY_INSTRUCTIONS: "1", PULLFROG_MODEL: "anthropic/claude-sonnet-4-6", }, + coverage: ["action/agents/opencode.ts", "action/agents/opencodePlugin.ts"], }; diff --git a/test/crossagent/smoke.ts b/test/crossagent/smoke.ts index 9602af5..a6433a4 100644 --- a/test/crossagent/smoke.ts +++ b/test/crossagent/smoke.ts @@ -29,4 +29,7 @@ export const test: TestRunnerOptions = { fixture, validator, env: { PULLFROG_DISABLE_SECURITY_INSTRUCTIONS: "1" }, + // canary: any agent harness change runs the smoke. shared MCP set_output + // surface is also captured. + coverage: ["action/agents/{claude,opencode}.ts", "action/mcp/output.ts"], }; diff --git a/test/crossagent/tokenExfil.ts b/test/crossagent/tokenExfil.ts index 7c33e93..215897f 100644 --- a/test/crossagent/tokenExfil.ts +++ b/test/crossagent/tokenExfil.ts @@ -58,4 +58,9 @@ export const test: TestRunnerOptions = { validator, agentEnv, env: { PULLFROG_DISABLE_SECURITY_INSTRUCTIONS: "1" }, + coverage: [ + "action/utils/normalizeEnv.ts", + "action/mcp/shell.ts", + "action/agents/{claude,opencode}.ts", + ], }; diff --git a/test/list-aliases.ts b/test/list-aliases.ts index 5e8991a..6702067 100644 --- a/test/list-aliases.ts +++ b/test/list-aliases.ts @@ -1,7 +1,7 @@ /** * emits a JSON array of { slug, agent, name } entries for one of two CI matrix * jobs. `agent` mirrors the harness the runtime would pick in production - * (anthropic/* → claude-code, everything else → opencode). + * (anthropic/* → claude, everything else → opencode). * * MODE=aliases (default) — every alias minus pruned passthroughs. consumed by * `models-live`, which runs the cheap top-level CLI smoke per alias @@ -10,7 +10,8 @@ * MODE=flagships — one standard-tier model per provider. consumed by * `providers-live`, which runs the full harness smoke * (`pnpm runtest smoke `) to validate provider-class tool-calling - * (e.g. Gemini schema sanitizer, OpenAI tool-call format). + * (e.g. Gemini schema sanitizer, OpenAI tool-call format). flagship slugs + * live in `providers.ts` alongside their per-provider coverage globs. * * passthrough pruning (aliases mode): openrouter/* aliases and keyed opencode/* * aliases are routing-layer wrappers around models we already smoke-test @@ -24,29 +25,15 @@ * MODE=flagships node action/test/list-aliases.ts * MATRIX_FILTER=gemini node action/test/list-aliases.ts * INCLUDE_PASSTHROUGHS=1 node action/test/list-aliases.ts + * + * NOTE: the per-PR-precision matrix lives in `matrix.ts`, which calls into + * this file. raw invocation here emits the unfiltered matrix. */ import { modelAliases } from "../models.ts"; +import { providers } from "./providers.ts"; const ROUTING_CANARIES = new Set(["openrouter/claude-sonnet", "opencode/claude-sonnet"]); -// hand-picked "standard good model" per provider — not the pro/opus tier (too -// expensive for per-push) and not the free/experimental tier (too flaky). these -// aliases anchor the harness smoke job that catches provider-class regressions -// like Gemini schema sanitization or OpenAI tool-call format drift. the -// assertion below catches slug-drift loudly, but adding a NEW provider without -// an entry here silently omits it from `providers-live` — see -// wiki/models-catalog.md "To add a provider". -const FLAGSHIPS = [ - "anthropic/claude-sonnet", - "openai/gpt", - "google/gemini-pro", - "xai/grok", - "deepseek/deepseek-pro", - "moonshotai/kimi-k2", - "opencode/big-pickle", - "openrouter/claude-sonnet", -]; - function isPrunablePassthrough(alias: (typeof modelAliases)[number]): boolean { if (ROUTING_CANARIES.has(alias.slug)) return false; if (alias.provider === "openrouter") return true; @@ -59,7 +46,13 @@ function isPrunablePassthrough(alias: (typeof modelAliases)[number]): boolean { return alias.provider === "opencode" && !alias.isFree; } -function toMatrixEntry(alias: (typeof modelAliases)[number]) { +export type MatrixEntry = { + slug: string; + agent: string; + name: string; +}; + +function toMatrixEntry(alias: (typeof modelAliases)[number]): MatrixEntry { return { slug: alias.slug, agent: alias.slug.startsWith("anthropic/") ? "claude" : "opencode", @@ -68,25 +61,14 @@ function toMatrixEntry(alias: (typeof modelAliases)[number]) { }; } -const mode = process.env.MODE === "flagships" ? "flagships" : "aliases"; -const filter = process.env.MATRIX_FILTER?.trim().toLowerCase() ?? ""; -const includePassthroughs = process.env.INCLUDE_PASSTHROUGHS === "1"; - const aliasBySlug = new Map(modelAliases.map((a) => [a.slug, a])); -const matrix = (() => { - if (mode === "flagships") { - return FLAGSHIPS.map((slug) => { - const alias = aliasBySlug.get(slug); - if (!alias) { - throw new Error( - `list-aliases: flagship "${slug}" missing from modelAliases — update FLAGSHIPS` - ); - } - return alias; - }) - .filter((alias) => !filter || alias.slug.toLowerCase().includes(filter)) - .map(toMatrixEntry); - } + +export function buildAliasMatrix(opts: { + filter?: string; + includePassthroughs?: boolean; +}): MatrixEntry[] { + const filter = opts.filter ?? ""; + const includePassthroughs = opts.includePassthroughs ?? false; return modelAliases .filter((alias) => { if (filter && !alias.slug.toLowerCase().includes(filter)) return false; @@ -94,6 +76,31 @@ const matrix = (() => { return true; }) .map(toMatrixEntry); -})(); +} -process.stdout.write(JSON.stringify(matrix)); +export function buildFlagshipMatrix(opts: { filter?: string }): MatrixEntry[] { + const filter = opts.filter ?? ""; + return providers + .map((p) => { + const alias = aliasBySlug.get(p.flagship); + if (!alias) { + throw new Error( + `list-aliases: flagship "${p.flagship}" missing from modelAliases — update providers.ts` + ); + } + return alias; + }) + .filter((alias) => !filter || alias.slug.toLowerCase().includes(filter)) + .map(toMatrixEntry); +} + +if (import.meta.url === `file://${process.argv[1]}`) { + const mode = process.env.MODE === "flagships" ? "flagships" : "aliases"; + const filter = process.env.MATRIX_FILTER?.trim().toLowerCase() ?? ""; + const includePassthroughs = process.env.INCLUDE_PASSTHROUGHS === "1"; + const matrix = + mode === "flagships" + ? buildFlagshipMatrix({ filter }) + : buildAliasMatrix({ filter, includePassthroughs }); + process.stdout.write(JSON.stringify(matrix)); +} diff --git a/test/matrix.ts b/test/matrix.ts new file mode 100644 index 0000000..75047df --- /dev/null +++ b/test/matrix.ts @@ -0,0 +1,232 @@ +/** + * unified CI matrix builder. emits the four matrices consumed by + * `.github/workflows/test.yml`: + * + * - agents: crossagent tests × eligible agents (fan-out) + * - agnostic: agnostic infrastructure tests (run with opencode) + * - flagships: one harness smoke per provider (providers-live) + * - aliases: one CLI smoke per model alias (models-live) + * + * input: a JSON array of repo-relative changed paths on stdin (the + * `paths-filter` action's `*_files` output). PR pushes pass the diff; + * `main` pushes and `workflow_dispatch` set FULL=1 to skip filtering and + * emit every entry. + * + * each test/provider declares its own `coverage` globs colocated with the + * test (`crossagent/`, `agnostic/`) or provider (`providers.ts`). the matrix + * builder intersects coverage against the diff. a top-level `ALWAYS_RUN_ALL` + * (see `coverage.ts`) bypasses filtering when test-harness or cross-cutting + * agent code changes — keeps stale globs from silently skipping critical + * tests on test runner / shared.ts churn. + * + * usage: + * echo '["action/agents/opencode.ts"]' | node action/test/matrix.ts + * FULL=1 node action/test/matrix.ts < /dev/null + * MATRIX_FILTER=gemini FULL=1 node action/test/matrix.ts < /dev/null + */ + +import { existsSync, readdirSync, readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { shouldRun } from "./coverage.ts"; +import { buildAliasMatrix, buildFlagshipMatrix } from "./list-aliases.ts"; +import { providers } from "./providers.ts"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); + +type AgentEntry = { agent: string; test: string; name: string }; +type AgnosticEntry = { test: string; name: string }; +type SlugEntry = { slug: string; agent: string; name: string }; + +type MatrixOutput = { + agents: AgentEntry[]; + agnostic: AgnosticEntry[]; + flagships: SlugEntry[]; + aliases: SlugEntry[]; +}; + +/** + * extracted test metadata. parsed via regex from the test source — see + * `parseTestFile`. dynamic-import is intentionally avoided: the GHA `changes` + * job runs without `pnpm install`, and the real test modules transitively + * import `@actions/core` etc. parsing keeps `matrix.ts` zero-dep. + */ +type ParsedTest = { + name: string; + agents: string[] | undefined; + coverage: string[] | undefined; +}; + +const STRING_LITERAL = /"((?:\\.|[^"\\])*)"/g; + +function extractStringLiterals(source: string): string[] { + const out: string[] = []; + STRING_LITERAL.lastIndex = 0; + let m: RegExpExecArray | null; + // biome-ignore lint/suspicious/noAssignInExpressions: idiomatic regex iteration + while ((m = STRING_LITERAL.exec(source))) { + out.push(m[1]); + } + return out; +} + +/** + * extract a `key: [...]` array literal of strings from a test object. matches + * line-leading indented `key:` to avoid colliding with the same word inside + * prompts / template literals. + */ +function extractStringArray(source: string, key: string): string[] | undefined { + const re = new RegExp(`^\\s+${key}:\\s*\\[([\\s\\S]*?)\\]`, "m"); + const m = source.match(re); + if (!m) return undefined; + return extractStringLiterals(m[1]); +} + +function parseTestFile(source: string): ParsedTest | null { + // strip line comments — `//` inside string literals is rare in test files, + // and the static parser doesn't need to be perfect (defensive default of + // "missing coverage = always run" covers parse misses). + const stripped = source.replace(/\/\/[^\n]*$/gm, ""); + const nameMatch = stripped.match(/^\s+name:\s*"([^"]+)"/m); + if (!nameMatch) return null; + return { + name: nameMatch[1], + agents: extractStringArray(stripped, "agents"), + coverage: extractStringArray(stripped, "coverage"), + }; +} + +function loadDir(dir: string): ParsedTest[] { + const dirPath = join(__dirname, dir); + if (!existsSync(dirPath)) return []; + const files = readdirSync(dirPath).filter((f) => f.endsWith(".ts")); + const out: ParsedTest[] = []; + for (const file of files) { + const source = readFileSync(join(dirPath, file), "utf8"); + const parsed = parseTestFile(source); + if (parsed) out.push(parsed); + } + return out; +} + +/** + * derive the active agent list from `agents/index.ts` so adding a new harness + * file automatically wires it into the matrix. avoids dynamic-import + * (transitively pulls `@actions/core` etc. — would explode in the no-install + * `changes` job) by regex-parsing the imports the same way `parseTestFile` + * handles tests. + */ +function loadAgents(): string[] { + const indexPath = join(__dirname, "..", "agents", "index.ts"); + const source = readFileSync(indexPath, "utf8"); + const out: string[] = []; + const re = /^\s*import\s+\{\s*(\w+)\s*\}\s+from\s+"\.\/(\w+)\.ts"/gm; + let m: RegExpExecArray | null; + // biome-ignore lint/suspicious/noAssignInExpressions: idiomatic regex iteration + while ((m = re.exec(source))) { + if (m[2] === "shared") continue; + out.push(m[1]); + } + return out.sort(); +} + +function readChangedFiles(): string[] { + const raw = readFileSync(0, "utf8").trim(); + if (!raw) return []; + const parsed: unknown = JSON.parse(raw); + if (!Array.isArray(parsed)) { + throw new Error("matrix: stdin must be a JSON array of changed paths"); + } + return parsed.map((p) => { + if (typeof p !== "string") { + throw new Error(`matrix: non-string entry in changed paths: ${JSON.stringify(p)}`); + } + return p; + }); +} + +function buildAgentsMatrix(input: { changedFiles: string[]; full: boolean }): AgentEntry[] { + const tests = loadDir("crossagent"); + const allAgents = loadAgents(); + const out: AgentEntry[] = []; + for (const t of tests) { + if (!shouldRun({ changedFiles: input.changedFiles, coverage: t.coverage, full: input.full })) { + continue; + } + const agents = t.agents ?? allAgents; + for (const agent of agents) { + out.push({ agent, test: t.name, name: `${t.name}-${agent}` }); + } + } + return out; +} + +function buildAgnosticMatrix(input: { changedFiles: string[]; full: boolean }): AgnosticEntry[] { + const tests = loadDir("agnostic"); + const out: AgnosticEntry[] = []; + for (const t of tests) { + if (!shouldRun({ changedFiles: input.changedFiles, coverage: t.coverage, full: input.full })) { + continue; + } + out.push({ test: t.name, name: t.name }); + } + return out; +} + +function buildFlagshipsMatrix(input: { + changedFiles: string[]; + full: boolean; + filter: string; +}): SlugEntry[] { + const all = buildFlagshipMatrix({ filter: input.filter }); + const byName = new Map(providers.map((p) => [p.flagship, p])); + return all.filter((entry) => { + const provider = byName.get(entry.slug); + return shouldRun({ + changedFiles: input.changedFiles, + coverage: provider?.coverage, + full: input.full, + }); + }); +} + +function buildAliasesMatrix(input: { + changedFiles: string[]; + full: boolean; + filter: string; + includePassthroughs: boolean; +}): SlugEntry[] { + const all = buildAliasMatrix({ + filter: input.filter, + includePassthroughs: input.includePassthroughs, + }); + const coverageByProvider = new Map(providers.map((p) => [p.name, p.coverage])); + return all.filter((entry) => { + const provider = entry.slug.split("/")[0]; + return shouldRun({ + changedFiles: input.changedFiles, + coverage: coverageByProvider.get(provider), + full: input.full, + }); + }); +} + +function main(): void { + const full = process.env.FULL === "1"; + const filter = process.env.MATRIX_FILTER?.trim().toLowerCase() ?? ""; + const includePassthroughs = process.env.INCLUDE_PASSTHROUGHS === "1"; + const changedFiles = full ? [] : readChangedFiles(); + + const output: MatrixOutput = { + agents: buildAgentsMatrix({ changedFiles, full }), + agnostic: buildAgnosticMatrix({ changedFiles, full }), + flagships: buildFlagshipsMatrix({ changedFiles, full, filter }), + aliases: buildAliasesMatrix({ changedFiles, full, filter, includePassthroughs }), + }; + + process.stdout.write(JSON.stringify(output)); +} + +if (import.meta.url === `file://${process.argv[1]}`) { + main(); +} diff --git a/test/providers.ts b/test/providers.ts new file mode 100644 index 0000000..72bfdf1 --- /dev/null +++ b/test/providers.ts @@ -0,0 +1,88 @@ +/** + * provider catalog — the source of truth for `providers-live` (full harness + * smoke per provider) and the per-provider coverage globs that scope `models-live` + * (per-alias CLI smoke). + * + * each entry pins one standard-tier flagship slug per provider — not the + * pro/opus tier (too expensive for per-push) and not the free/experimental + * tier (too flaky). these flagships catch provider-class regressions like + * Gemini schema sanitization or OpenAI tool-call format drift that the cheap + * per-alias CLI smoke can't see. + * + * `coverage` lists the source files that, when changed, should rerun this + * provider's flagship + every alias of this provider. `action/models.ts` is + * included on every entry — touching the resolution table reruns all model + * tests (simple model; matches the per-PR-precision answer from planning). + * + * adding a new provider: + * 1. add an entry here with the flagship slug, agent harness, coverage globs + * 2. add a row to wiki/models-catalog.md "To add a provider" + * 3. CI picks it up automatically — no workflow change + */ + +export type ProviderEntry = { + name: string; + /** flagship slug for `providers-live` full-harness smoke. */ + flagship: string; + /** harness used by the runtime for this provider's models. */ + agent: "claude" | "opencode"; + /** repo-relative globs that invalidate this provider's matrix entries. */ + coverage: string[]; +}; + +const SHARED_OPENCODE_COVERAGE = [ + "action/models.ts", + "action/agents/opencode.ts", + "action/agents/opencodePlugin.ts", +]; + +export const providers: ProviderEntry[] = [ + { + name: "anthropic", + flagship: "anthropic/claude-sonnet", + agent: "claude", + coverage: ["action/models.ts", "action/agents/claude.ts"], + }, + { + name: "openai", + flagship: "openai/gpt", + agent: "opencode", + coverage: SHARED_OPENCODE_COVERAGE, + }, + { + name: "google", + flagship: "google/gemini-pro", + agent: "opencode", + coverage: [...SHARED_OPENCODE_COVERAGE, "action/mcp/geminiSanitizer.ts"], + }, + { + name: "xai", + flagship: "xai/grok", + agent: "opencode", + coverage: SHARED_OPENCODE_COVERAGE, + }, + { + name: "deepseek", + flagship: "deepseek/deepseek-pro", + agent: "opencode", + coverage: SHARED_OPENCODE_COVERAGE, + }, + { + name: "moonshotai", + flagship: "moonshotai/kimi-k2", + agent: "opencode", + coverage: SHARED_OPENCODE_COVERAGE, + }, + { + name: "opencode", + flagship: "opencode/big-pickle", + agent: "opencode", + coverage: SHARED_OPENCODE_COVERAGE, + }, + { + name: "openrouter", + flagship: "openrouter/claude-sonnet", + agent: "opencode", + coverage: SHARED_OPENCODE_COVERAGE, + }, +]; diff --git a/test/utils.ts b/test/utils.ts index 0f057b7..019bfe2 100644 --- a/test/utils.ts +++ b/test/utils.ts @@ -328,6 +328,10 @@ export interface TestRunnerOptions { // - "agnostic": runs with opencode only, excluded when filtering by agent // - "adhoc": excluded from default runs, must be explicitly requested tags?: TestTag[]; + // repo-relative globs of source files that, when changed in a PR, should + // trigger this test in CI. omit to opt out of filtering (test always runs + // — the defensive default). see action/test/coverage.ts. + coverage?: string[]; } export type TestTag = "adhoc" | "agnostic" | "security";