From 5e6ff676236bc1c9d7921ab0b103b7aa37b9a21e Mon Sep 17 00:00:00 2001 From: Colin McDonnell Date: Thu, 16 Apr 2026 21:10:15 +0000 Subject: [PATCH] move models.dev drift tests to main-only; add per-alias live smoke matrix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR CI kept breaking on upstream catalog drift (new model ships on models.dev, OpenRouter renames an id, etc.) — failures unrelated to the PR's contents. split the model-alias test suite so PRs only see pure-logic checks, and push the external-state drift + end-to-end coverage to main. test organization: - action/test/models.test.ts keeps pure invariants: openRouterResolve completeness and fallback-chain resolution. runs on every PR. - action/test/models-catalog.main.test.ts gets the 4 network-dependent describes (models.dev validity x2, OpenRouter API validity, latest-model snapshot). runs only on main push via a dedicated vitest config (vitest.main.config.ts + `pnpm test:catalog`). new CI jobs in .github/workflows/test.yml: - models-catalog: `pnpm test:catalog` on every main push. detects upstream catalog drift so we can react at the next convenient window. - models-live: 38-entry matrix that invokes the agent harness end-to-end against the real provider for each alias in models.ts. generated from action/test/list-aliases.ts. runs only on main push AND only when resolution-affecting files changed (action/models.ts, action/package.json, action/agents/**) — the exact shape of the opus 4.7 incident. test/run.ts: PULLFROG_MODEL now flows through from process.env so the live matrix can pin an alias per job without the per-agent default clobbering it. Made-with: Cursor --- agents/claude.ts | 4 +- package.json | 3 +- pnpm-lock.yaml | 10 +- ....snap => models-catalog.main.test.ts.snap} | 0 test/list-aliases.ts | 21 +++ test/models-catalog.main.test.ts | 145 +++++++++++++++++ test/models.test.ts | 154 ++---------------- test/run.ts | 22 ++- vitest.config.ts | 10 +- vitest.main.config.ts | 15 ++ 10 files changed, 225 insertions(+), 159 deletions(-) rename test/__snapshots__/{models.test.ts.snap => models-catalog.main.test.ts.snap} (100%) create mode 100644 test/list-aliases.ts create mode 100644 test/models-catalog.main.test.ts create mode 100644 vitest.main.config.ts diff --git a/agents/claude.ts b/agents/claude.ts index a2d0b29..bbeca94 100644 --- a/agents/claude.ts +++ b/agents/claude.ts @@ -71,8 +71,8 @@ function stripProviderPrefix(specifier: string): string { return slashIndex > 0 ? specifier.slice(slashIndex + 1) : specifier; } -// `max` effort is Opus 4.6 only — errors on other models. -// use `max` when the resolved model is Opus, `high` otherwise. +// `max` effort is supported on Opus 4.6 / 4.7; other models fall back to `high`. +// claude-code deny-lists older opus/sonnet generations from `max` at invocation time. function resolveEffort(model: string | undefined): "max" | "high" { if (model?.includes("opus")) return "max"; return "high"; diff --git a/package.json b/package.json index cdd2d67..7ba79ed 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,7 @@ ], "scripts": { "test": "vitest", + "test:catalog": "vitest run --config vitest.main.config.ts", "typecheck": "tsc --noEmit", "build": "node esbuild.config.js && tsc -p tsconfig.exports.json", "check:entrypoints": "node scripts/check-entrypoint-imports.ts", @@ -24,7 +25,7 @@ }, "devDependencies": { "@actions/core": "^1.11.1", - "@anthropic-ai/claude-code": "2.1.85", + "@anthropic-ai/claude-code": "2.1.112", "@ark/fs": "0.56.0", "@ark/util": "0.56.0", "@clack/prompts": "^1.2.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index bf4ceb7..dd95c9d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -12,8 +12,8 @@ importers: specifier: ^1.11.1 version: 1.11.1 '@anthropic-ai/claude-code': - specifier: 2.1.85 - version: 2.1.85 + specifier: 2.1.112 + version: 2.1.112 '@ark/fs': specifier: 0.56.0 version: 0.56.0 @@ -128,8 +128,8 @@ packages: '@actions/io@1.1.3': resolution: {integrity: sha512-wi9JjgKLYS7U/z8PPbco+PvTb/nRWjeoFlJ1Qer83k/3C5PHQi28hiVdeE2kHXmIL99mQFawx8qt/JPjZilJ8Q==} - '@anthropic-ai/claude-code@2.1.85': - resolution: {integrity: sha512-3/q3xTpk9EnBfQ/XsHGkOZniOgQx4sqD95CDKw1mvN1Qw5+9IZTp6ILdds02d7vOM6YuLL0G0zhqsMSAFVse4w==} + '@anthropic-ai/claude-code@2.1.112': + resolution: {integrity: sha512-9FUgJ0EOvILyhIqxFKNVliebiUjL68dwpEW3eGSSe0vkVDJ1c5qMDNWc22gW3zkD7zRAqtfQPSGv0t4vMM2DPA==} engines: {node: '>=18.0.0'} hasBin: true @@ -1960,7 +1960,7 @@ snapshots: '@actions/io@1.1.3': {} - '@anthropic-ai/claude-code@2.1.85': + '@anthropic-ai/claude-code@2.1.112': optionalDependencies: '@img/sharp-darwin-arm64': 0.34.5 '@img/sharp-darwin-x64': 0.34.5 diff --git a/test/__snapshots__/models.test.ts.snap b/test/__snapshots__/models-catalog.main.test.ts.snap similarity index 100% rename from test/__snapshots__/models.test.ts.snap rename to test/__snapshots__/models-catalog.main.test.ts.snap diff --git a/test/list-aliases.ts b/test/list-aliases.ts new file mode 100644 index 0000000..a82152c --- /dev/null +++ b/test/list-aliases.ts @@ -0,0 +1,21 @@ +/** + * emits a JSON array of { slug, agent, name } entries for the `models-live` + * matrix job. `agent` is auto-derived from the alias provider and matches the + * harness the runtime would pick in production. + * + * usage: `node action/test/list-aliases.ts` + */ +import { modelAliases } from "../models.ts"; + +function agentForSlug(slug: string): "claude" | "opencode" { + return slug.startsWith("anthropic/") ? "claude" : "opencode"; +} + +const matrix = modelAliases.map((alias) => ({ + slug: alias.slug, + agent: agentForSlug(alias.slug), + // readable display name (GHA renders slashes awkwardly in matrix job titles) + name: alias.slug.replace("/", "-"), +})); + +process.stdout.write(JSON.stringify(matrix)); diff --git a/test/models-catalog.main.test.ts b/test/models-catalog.main.test.ts new file mode 100644 index 0000000..cbbef10 --- /dev/null +++ b/test/models-catalog.main.test.ts @@ -0,0 +1,145 @@ +import { describe, expect, it } from "vitest"; +import { type ModelProvider, modelAliases, providers } from "../models.ts"; + +// ── catalog drift tests — main-only ───────────────────────────────────────────── +// +// these tests fetch models.dev and openrouter.ai to verify that every alias in +// models.ts still corresponds to a live, non-deprecated upstream model. upstream +// catalog drift (new model ships, old model deprecated, etc.) causes failures +// that are unrelated to any code change in the PR — so these run only on main. +// +// run locally with `pnpm test:catalog`. +// in CI, gated to push events on main. + +type ModelsDevModel = { + name: string; + status?: string; + release_date?: string; +}; + +type ModelsDevProvider = { + name: string; + models: Record; +}; + +type ModelsDevApi = Record; + +const api = fetch("https://models.dev/api.json").then((r) => r.json() as Promise); + +function parseResolve(resolve: string): { provider: string; modelId: string } { + const idx = resolve.indexOf("/"); + return { provider: resolve.slice(0, idx), modelId: resolve.slice(idx + 1) }; +} + +describe("models.dev validity", async () => { + const data = await api; + + for (const alias of modelAliases) { + const parsed = parseResolve(alias.resolve); + + it(`${alias.resolve} exists on models.dev`, () => { + const providerData = data[parsed.provider]; + expect(providerData, `provider "${parsed.provider}" not found on models.dev`).toBeDefined(); + const model = providerData.models[parsed.modelId]; + expect( + model, + `model "${parsed.modelId}" not found under ${parsed.provider} on models.dev` + ).toBeDefined(); + }); + + if (!alias.fallback) { + it(`${alias.resolve} is not deprecated`, () => { + const model = data[parsed.provider]?.models[parsed.modelId]; + if (!model) return; // covered by existence test above + expect(model.status, `${alias.resolve} is deprecated on models.dev`).not.toBe("deprecated"); + }); + } + } +}); + +describe("openRouterResolve models.dev validity", async () => { + const data = await api; + const seen = new Set(); + + for (const alias of modelAliases) { + if (!alias.openRouterResolve) continue; + if (seen.has(alias.openRouterResolve)) continue; + seen.add(alias.openRouterResolve); + + const parsed = parseResolve(alias.openRouterResolve); + + it(`${alias.openRouterResolve} exists on models.dev`, () => { + const providerData = data[parsed.provider]; + expect(providerData, `provider "${parsed.provider}" not found on models.dev`).toBeDefined(); + const model = providerData.models[parsed.modelId]; + expect( + model, + `model "${parsed.modelId}" not found under ${parsed.provider} on models.dev` + ).toBeDefined(); + }); + } +}); + +type OpenRouterModel = { id: string }; +type OpenRouterModelsResponse = { data: OpenRouterModel[] }; + +const openRouterApi = fetch("https://openrouter.ai/api/v1/models").then( + (r) => r.json() as Promise +); + +describe("openRouterResolve OpenRouter API validity", async () => { + const orData = await openRouterApi; + const orModelIds = new Set(orData.data.map((m) => m.id)); + const seen = new Set(); + + for (const alias of modelAliases) { + if (!alias.openRouterResolve) continue; + const orModelId = alias.openRouterResolve.slice("openrouter/".length); + if (seen.has(orModelId)) continue; + seen.add(orModelId); + + it(`${orModelId} exists on OpenRouter`, () => { + expect( + orModelIds.has(orModelId), + `model "${orModelId}" not found in OpenRouter API (/api/v1/models)` + ).toBe(true); + }); + } +}); + +describe("latest model per provider snapshot", async () => { + const data = await api; + const providerKeys = Object.keys(providers) as ModelProvider[]; + + const latestByProvider: Record = {}; + + for (const key of providerKeys) { + const providerData = data[key]; + if (!providerData) continue; + + let latest: { modelId: string; releaseDate: string } | undefined; + for (const [modelId, model] of Object.entries(providerData.models)) { + // skip non-GA models so beta/nightly churn doesn't break the snapshot + if (model.status) continue; + const rd = model.release_date; + if (!rd) continue; + // tiebreak by modelId for stable ordering when release dates match + if ( + !latest || + rd > latest.releaseDate || + (rd === latest.releaseDate && modelId > latest.modelId) + ) { + latest = { modelId, releaseDate: rd }; + } + } + if (latest) { + latestByProvider[key] = latest; + } + } + + // when this fails, a provider shipped a new model. check whether we need + // to add or update an alias in models.ts before updating the snapshot. + it("matches snapshot", () => { + expect(latestByProvider).toMatchSnapshot(); + }); +}); diff --git a/test/models.test.ts b/test/models.test.ts index cf61dea..ac0cff1 100644 --- a/test/models.test.ts +++ b/test/models.test.ts @@ -1,64 +1,11 @@ import { describe, expect, it } from "vitest"; -import { type ModelProvider, modelAliases, providers, resolveCliModel } from "../models.ts"; +import { modelAliases, resolveCliModel } from "../models.ts"; -type ModelsDevModel = { - name: string; - status?: string; - release_date?: string; -}; - -type ModelsDevProvider = { - name: string; - models: Record; -}; - -type ModelsDevApi = Record; - -const api = fetch("https://models.dev/api.json").then((r) => r.json() as Promise); - -/** split a resolve slug into the models.dev provider key and model key */ -function parseResolve(resolve: string): { provider: string; modelId: string } { - const idx = resolve.indexOf("/"); - return { provider: resolve.slice(0, idx), modelId: resolve.slice(idx + 1) }; -} - -describe("models.dev validity", async () => { - const data = await api; - - for (const alias of modelAliases) { - const parsed = parseResolve(alias.resolve); - - it(`${alias.resolve} exists on models.dev`, () => { - const providerData = data[parsed.provider]; - expect(providerData, `provider "${parsed.provider}" not found on models.dev`).toBeDefined(); - const model = providerData.models[parsed.modelId]; - expect( - model, - `model "${parsed.modelId}" not found under ${parsed.provider} on models.dev` - ).toBeDefined(); - }); - - if (!alias.fallback) { - it(`${alias.resolve} is not deprecated`, () => { - const model = data[parsed.provider]?.models[parsed.modelId]; - if (!model) return; // covered by existence test above - expect(model.status, `${alias.resolve} is deprecated on models.dev`).not.toBe("deprecated"); - }); - } - } - - for (const alias of modelAliases.filter((a) => a.fallback)) { - it(`${alias.slug} fallback chain resolves to a non-deprecated model`, () => { - const resolved = resolveCliModel(alias.slug); - expect( - resolved, - `fallback chain for "${alias.slug}" does not resolve to a non-deprecated model` - ).toBeDefined(); - }); - } -}); - -// ── openRouterResolve coverage ───────────────────────────────────────────────── +// ── pure alias-registry invariants ────────────────────────────────────────────── +// +// these tests validate our alias data structure without hitting external APIs. +// network-dependent checks (models.dev / OpenRouter catalog drift, latest-model +// snapshot) live in models-catalog.main.test.ts and run only on main. // models that have no OpenRouter equivalent and require BYOK. // add a model here ONLY when it genuinely doesn't exist on both models.dev and OpenRouter. @@ -84,91 +31,14 @@ describe("openRouterResolve completeness", () => { } }); -describe("openRouterResolve models.dev validity", async () => { - const data = await api; - const seen = new Set(); - - for (const alias of modelAliases) { - if (!alias.openRouterResolve) continue; - if (seen.has(alias.openRouterResolve)) continue; - seen.add(alias.openRouterResolve); - - const parsed = parseResolve(alias.openRouterResolve); - - it(`${alias.openRouterResolve} exists on models.dev`, () => { - const providerData = data[parsed.provider]; - expect(providerData, `provider "${parsed.provider}" not found on models.dev`).toBeDefined(); - const model = providerData.models[parsed.modelId]; +describe("fallback chain resolution", () => { + for (const alias of modelAliases.filter((a) => a.fallback)) { + it(`${alias.slug} fallback chain resolves to a non-deprecated model`, () => { + const resolved = resolveCliModel(alias.slug); expect( - model, - `model "${parsed.modelId}" not found under ${parsed.provider} on models.dev` + resolved, + `fallback chain for "${alias.slug}" does not resolve to a non-deprecated model` ).toBeDefined(); }); } }); - -type OpenRouterModel = { id: string }; -type OpenRouterModelsResponse = { data: OpenRouterModel[] }; - -const openRouterApi = fetch("https://openrouter.ai/api/v1/models").then( - (r) => r.json() as Promise -); - -describe("openRouterResolve OpenRouter API validity", async () => { - const orData = await openRouterApi; - const orModelIds = new Set(orData.data.map((m) => m.id)); - const seen = new Set(); - - for (const alias of modelAliases) { - if (!alias.openRouterResolve) continue; - const orModelId = alias.openRouterResolve.slice("openrouter/".length); - if (seen.has(orModelId)) continue; - seen.add(orModelId); - - it(`${orModelId} exists on OpenRouter`, () => { - expect( - orModelIds.has(orModelId), - `model "${orModelId}" not found in OpenRouter API (/api/v1/models)` - ).toBe(true); - }); - } -}); - -describe("latest model per provider snapshot", async () => { - const data = await api; - const providerKeys = Object.keys(providers) as ModelProvider[]; - - const latestByProvider: Record = {}; - - for (const key of providerKeys) { - const providerData = data[key]; - if (!providerData) continue; - - let latest: { modelId: string; releaseDate: string } | undefined; - for (const [modelId, model] of Object.entries(providerData.models)) { - // skip non-GA models so beta/nightly churn doesn't break the snapshot - if (model.status) continue; - const rd = model.release_date; - if (!rd) continue; - // tiebreak by modelId for stable ordering when release dates match - if ( - !latest || - rd > latest.releaseDate || - (rd === latest.releaseDate && modelId > latest.modelId) - ) { - latest = { modelId, releaseDate: rd }; - } - } - if (latest) { - latestByProvider[key] = latest; - } - } - - // when this fails, a provider shipped a new model. usually that just means - // bumping the `resolve` on an existing alias in models.ts (e.g. point - // "claude-opus" at the latest opus) rather than introducing a new alias. - // refresh the alias resolution first, then update this snapshot. - it("matches snapshot", () => { - expect(latestByProvider).toMatchSnapshot(); - }); -}); diff --git a/test/run.ts b/test/run.ts index 3f73fd1..afec1bb 100644 --- a/test/run.ts +++ b/test/run.ts @@ -299,15 +299,21 @@ async function runTestForAgent(ctx: RunContext): Promise { env.PULLFROG_AGENT = ctx.agent; // override DB model to avoid mismatch when PULLFROG_AGENT forces a specific agent - // (DB model may belong to a different provider than the forced agent supports) + // (DB model may belong to a different provider than the forced agent supports). + // precedence: testConfig.env > process.env.PULLFROG_MODEL > per-agent default. + // the process.env pass-through lets CI (models-live matrix) pin an alias per job. if (!Object.hasOwn(env, "PULLFROG_MODEL")) { - const defaultModels: Record = { - claude: "anthropic/claude-sonnet-4-6", - opencode: "anthropic/claude-sonnet-4-6", - }; - const model = defaultModels[ctx.agent]; - if (model) { - env.PULLFROG_MODEL = model; + if (process.env.PULLFROG_MODEL) { + env.PULLFROG_MODEL = process.env.PULLFROG_MODEL; + } else { + const defaultModels: Record = { + claude: "anthropic/claude-sonnet-4-6", + opencode: "anthropic/claude-sonnet-4-6", + }; + const model = defaultModels[ctx.agent]; + if (model) { + env.PULLFROG_MODEL = model; + } } } diff --git a/vitest.config.ts b/vitest.config.ts index 047c343..cb5cd53 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -4,7 +4,15 @@ export default defineConfig({ test: { globals: true, environment: "node", - exclude: ["**/node_modules/**", "**/.temp/**", "**/.pnpm-store/**"], + exclude: [ + "**/node_modules/**", + "**/.temp/**", + "**/.pnpm-store/**", + // *.main.test.ts files run only on main (e.g. catalog drift against + // models.dev + OpenRouter). run them via `pnpm test:catalog`, which + // points at vitest.main.config.ts. + "**/*.main.test.ts", + ], globalSetup: ["./vitest.global-setup.ts"], setupFiles: ["./vitest.setup.ts"], }, diff --git a/vitest.main.config.ts b/vitest.main.config.ts new file mode 100644 index 0000000..9384e93 --- /dev/null +++ b/vitest.main.config.ts @@ -0,0 +1,15 @@ +import { defineConfig } from "vitest/config"; + +// config for main-only tests (see *.main.test.ts). runs the catalog drift +// suite explicitly; the default vitest.config.ts excludes these files so PR +// CI stays unaffected by upstream catalog changes. +export default defineConfig({ + test: { + globals: true, + environment: "node", + include: ["**/*.main.test.ts"], + exclude: ["**/node_modules/**", "**/.temp/**", "**/.pnpm-store/**"], + globalSetup: ["./vitest.global-setup.ts"], + setupFiles: ["./vitest.setup.ts"], + }, +});