move models.dev drift tests to main-only; add per-alias live smoke matrix

PR CI kept breaking on upstream catalog drift (new model ships on models.dev,
OpenRouter renames an id, etc.) — failures unrelated to the PR's contents.
split the model-alias test suite so PRs only see pure-logic checks, and push
the external-state drift + end-to-end coverage to main.

test organization:
- action/test/models.test.ts keeps pure invariants: openRouterResolve
  completeness and fallback-chain resolution. runs on every PR.
- action/test/models-catalog.main.test.ts gets the 4 network-dependent
  describes (models.dev validity x2, OpenRouter API validity, latest-model
  snapshot). runs only on main push via a dedicated vitest config
  (vitest.main.config.ts + `pnpm test:catalog`).

new CI jobs in .github/workflows/test.yml:
- models-catalog: `pnpm test:catalog` on every main push. detects upstream
  catalog drift so we can react at the next convenient window.
- models-live: 38-entry matrix that invokes the agent harness end-to-end
  against the real provider for each alias in models.ts. generated from
  action/test/list-aliases.ts. runs only on main push AND only when
  resolution-affecting files changed (action/models.ts, action/package.json,
  action/agents/**) — the exact shape of the opus 4.7 incident.

test/run.ts: PULLFROG_MODEL now flows through from process.env so the live
matrix can pin an alias per job without the per-agent default clobbering it.

Made-with: Cursor

agents/claude.ts

@@ -71,8 +71,8 @@ function stripProviderPrefix(specifier: string): string {
   return slashIndex > 0 ? specifier.slice(slashIndex + 1) : specifier;
 }
 
-// `max` effort is Opus 4.6 only — errors on other models.
+// `max` effort is supported on Opus 4.6 / 4.7; other models fall back to `high`.
-// use `max` when the resolved model is Opus, `high` otherwise.
+// claude-code deny-lists older opus/sonnet generations from `max` at invocation time.
 function resolveEffort(model: string | undefined): "max" | "high" {
   if (model?.includes("opus")) return "max";
   return "high";

package.json

@@ -12,6 +12,7 @@
   ],
   "scripts": {
     "test": "vitest",
+    "test:catalog": "vitest run --config vitest.main.config.ts",
     "typecheck": "tsc --noEmit",
     "build": "node esbuild.config.js && tsc -p tsconfig.exports.json",
     "check:entrypoints": "node scripts/check-entrypoint-imports.ts",
@@ -24,7 +25,7 @@
   },
   "devDependencies": {
     "@actions/core": "^1.11.1",
-    "@anthropic-ai/claude-code": "2.1.85",
+    "@anthropic-ai/claude-code": "2.1.112",
     "@ark/fs": "0.56.0",
     "@ark/util": "0.56.0",
     "@clack/prompts": "^1.2.0",

pnpm-lock.yaml

@@ -12,8 +12,8 @@ importers:
         specifier: ^1.11.1
         version: 1.11.1
       '@anthropic-ai/claude-code':
-        specifier: 2.1.85
+        specifier: 2.1.112
-        version: 2.1.85
+        version: 2.1.112
       '@ark/fs':
         specifier: 0.56.0
         version: 0.56.0
@@ -128,8 +128,8 @@ packages:
   '@actions/io@1.1.3':
     resolution: {integrity: sha512-wi9JjgKLYS7U/z8PPbco+PvTb/nRWjeoFlJ1Qer83k/3C5PHQi28hiVdeE2kHXmIL99mQFawx8qt/JPjZilJ8Q==}
 
-  '@anthropic-ai/claude-code@2.1.85':
+  '@anthropic-ai/claude-code@2.1.112':
-    resolution: {integrity: sha512-3/q3xTpk9EnBfQ/XsHGkOZniOgQx4sqD95CDKw1mvN1Qw5+9IZTp6ILdds02d7vOM6YuLL0G0zhqsMSAFVse4w==}
+    resolution: {integrity: sha512-9FUgJ0EOvILyhIqxFKNVliebiUjL68dwpEW3eGSSe0vkVDJ1c5qMDNWc22gW3zkD7zRAqtfQPSGv0t4vMM2DPA==}
     engines: {node: '>=18.0.0'}
     hasBin: true
 
@@ -1960,7 +1960,7 @@ snapshots:
 
   '@actions/io@1.1.3': {}
 
-  '@anthropic-ai/claude-code@2.1.85':
+  '@anthropic-ai/claude-code@2.1.112':
     optionalDependencies:
       '@img/sharp-darwin-arm64': 0.34.5
       '@img/sharp-darwin-x64': 0.34.5

test/list-aliases.ts

@@ -0,0 +1,21 @@
+/**
+ * emits a JSON array of { slug, agent, name } entries for the `models-live`
+ * matrix job. `agent` is auto-derived from the alias provider and matches the
+ * harness the runtime would pick in production.
+ *
+ * usage: `node action/test/list-aliases.ts`
+ */
+import { modelAliases } from "../models.ts";
+
+function agentForSlug(slug: string): "claude" | "opencode" {
+  return slug.startsWith("anthropic/") ? "claude" : "opencode";
+}
+
+const matrix = modelAliases.map((alias) => ({
+  slug: alias.slug,
+  agent: agentForSlug(alias.slug),
+  // readable display name (GHA renders slashes awkwardly in matrix job titles)
+  name: alias.slug.replace("/", "-"),
+}));
+
+process.stdout.write(JSON.stringify(matrix));

test/models-catalog.main.test.ts

@@ -0,0 +1,145 @@
+import { describe, expect, it } from "vitest";
+import { type ModelProvider, modelAliases, providers } from "../models.ts";
+
+// ── catalog drift tests — main-only ─────────────────────────────────────────────
+//
+// these tests fetch models.dev and openrouter.ai to verify that every alias in
+// models.ts still corresponds to a live, non-deprecated upstream model. upstream
+// catalog drift (new model ships, old model deprecated, etc.) causes failures
+// that are unrelated to any code change in the PR — so these run only on main.
+//
+// run locally with `pnpm test:catalog`.
+// in CI, gated to push events on main.
+
+type ModelsDevModel = {
+  name: string;
+  status?: string;
+  release_date?: string;
+};
+
+type ModelsDevProvider = {
+  name: string;
+  models: Record<string, ModelsDevModel>;
+};
+
+type ModelsDevApi = Record<string, ModelsDevProvider>;
+
+const api = fetch("https://models.dev/api.json").then((r) => r.json() as Promise<ModelsDevApi>);
+
+function parseResolve(resolve: string): { provider: string; modelId: string } {
+  const idx = resolve.indexOf("/");
+  return { provider: resolve.slice(0, idx), modelId: resolve.slice(idx + 1) };
+}
+
+describe("models.dev validity", async () => {
+  const data = await api;
+
+  for (const alias of modelAliases) {
+    const parsed = parseResolve(alias.resolve);
+
+    it(`${alias.resolve} exists on models.dev`, () => {
+      const providerData = data[parsed.provider];
+      expect(providerData, `provider "${parsed.provider}" not found on models.dev`).toBeDefined();
+      const model = providerData.models[parsed.modelId];
+      expect(
+        model,
+        `model "${parsed.modelId}" not found under ${parsed.provider} on models.dev`
+      ).toBeDefined();
+    });
+
+    if (!alias.fallback) {
+      it(`${alias.resolve} is not deprecated`, () => {
+        const model = data[parsed.provider]?.models[parsed.modelId];
+        if (!model) return; // covered by existence test above
+        expect(model.status, `${alias.resolve} is deprecated on models.dev`).not.toBe("deprecated");
+      });
+    }
+  }
+});
+
+describe("openRouterResolve models.dev validity", async () => {
+  const data = await api;
+  const seen = new Set<string>();
+
+  for (const alias of modelAliases) {
+    if (!alias.openRouterResolve) continue;
+    if (seen.has(alias.openRouterResolve)) continue;
+    seen.add(alias.openRouterResolve);
+
+    const parsed = parseResolve(alias.openRouterResolve);
+
+    it(`${alias.openRouterResolve} exists on models.dev`, () => {
+      const providerData = data[parsed.provider];
+      expect(providerData, `provider "${parsed.provider}" not found on models.dev`).toBeDefined();
+      const model = providerData.models[parsed.modelId];
+      expect(
+        model,
+        `model "${parsed.modelId}" not found under ${parsed.provider} on models.dev`
+      ).toBeDefined();
+    });
+  }
+});
+
+type OpenRouterModel = { id: string };
+type OpenRouterModelsResponse = { data: OpenRouterModel[] };
+
+const openRouterApi = fetch("https://openrouter.ai/api/v1/models").then(
+  (r) => r.json() as Promise<OpenRouterModelsResponse>
+);
+
+describe("openRouterResolve OpenRouter API validity", async () => {
+  const orData = await openRouterApi;
+  const orModelIds = new Set(orData.data.map((m) => m.id));
+  const seen = new Set<string>();
+
+  for (const alias of modelAliases) {
+    if (!alias.openRouterResolve) continue;
+    const orModelId = alias.openRouterResolve.slice("openrouter/".length);
+    if (seen.has(orModelId)) continue;
+    seen.add(orModelId);
+
+    it(`${orModelId} exists on OpenRouter`, () => {
+      expect(
+        orModelIds.has(orModelId),
+        `model "${orModelId}" not found in OpenRouter API (/api/v1/models)`
+      ).toBe(true);
+    });
+  }
+});
+
+describe("latest model per provider snapshot", async () => {
+  const data = await api;
+  const providerKeys = Object.keys(providers) as ModelProvider[];
+
+  const latestByProvider: Record<string, { modelId: string; releaseDate: string }> = {};
+
+  for (const key of providerKeys) {
+    const providerData = data[key];
+    if (!providerData) continue;
+
+    let latest: { modelId: string; releaseDate: string } | undefined;
+    for (const [modelId, model] of Object.entries(providerData.models)) {
+      // skip non-GA models so beta/nightly churn doesn't break the snapshot
+      if (model.status) continue;
+      const rd = model.release_date;
+      if (!rd) continue;
+      // tiebreak by modelId for stable ordering when release dates match
+      if (
+        !latest ||
+        rd > latest.releaseDate ||
+        (rd === latest.releaseDate && modelId > latest.modelId)
+      ) {
+        latest = { modelId, releaseDate: rd };
+      }
+    }
+    if (latest) {
+      latestByProvider[key] = latest;
+    }
+  }
+
+  // when this fails, a provider shipped a new model. check whether we need
+  // to add or update an alias in models.ts before updating the snapshot.
+  it("matches snapshot", () => {
+    expect(latestByProvider).toMatchSnapshot();
+  });
+});

test/models.test.ts

@@ -1,64 +1,11 @@
 import { describe, expect, it } from "vitest";
-import { type ModelProvider, modelAliases, providers, resolveCliModel } from "../models.ts";
+import { modelAliases, resolveCliModel } from "../models.ts";
 
-type ModelsDevModel = {
+// ── pure alias-registry invariants ──────────────────────────────────────────────
-  name: string;
+//
-  status?: string;
+// these tests validate our alias data structure without hitting external APIs.
-  release_date?: string;
+// network-dependent checks (models.dev / OpenRouter catalog drift, latest-model
-};
+// snapshot) live in models-catalog.main.test.ts and run only on main.
-
-type ModelsDevProvider = {
-  name: string;
-  models: Record<string, ModelsDevModel>;
-};
-
-type ModelsDevApi = Record<string, ModelsDevProvider>;
-
-const api = fetch("https://models.dev/api.json").then((r) => r.json() as Promise<ModelsDevApi>);
-
-/** split a resolve slug into the models.dev provider key and model key */
-function parseResolve(resolve: string): { provider: string; modelId: string } {
-  const idx = resolve.indexOf("/");
-  return { provider: resolve.slice(0, idx), modelId: resolve.slice(idx + 1) };
-}
-
-describe("models.dev validity", async () => {
-  const data = await api;
-
-  for (const alias of modelAliases) {
-    const parsed = parseResolve(alias.resolve);
-
-    it(`${alias.resolve} exists on models.dev`, () => {
-      const providerData = data[parsed.provider];
-      expect(providerData, `provider "${parsed.provider}" not found on models.dev`).toBeDefined();
-      const model = providerData.models[parsed.modelId];
-      expect(
-        model,
-        `model "${parsed.modelId}" not found under ${parsed.provider} on models.dev`
-      ).toBeDefined();
-    });
-
-    if (!alias.fallback) {
-      it(`${alias.resolve} is not deprecated`, () => {
-        const model = data[parsed.provider]?.models[parsed.modelId];
-        if (!model) return; // covered by existence test above
-        expect(model.status, `${alias.resolve} is deprecated on models.dev`).not.toBe("deprecated");
-      });
-    }
-  }
-
-  for (const alias of modelAliases.filter((a) => a.fallback)) {
-    it(`${alias.slug} fallback chain resolves to a non-deprecated model`, () => {
-      const resolved = resolveCliModel(alias.slug);
-      expect(
-        resolved,
-        `fallback chain for "${alias.slug}" does not resolve to a non-deprecated model`
-      ).toBeDefined();
-    });
-  }
-});
-
-// ── openRouterResolve coverage ─────────────────────────────────────────────────
 
 // models that have no OpenRouter equivalent and require BYOK.
 // add a model here ONLY when it genuinely doesn't exist on both models.dev and OpenRouter.
@@ -84,91 +31,14 @@ describe("openRouterResolve completeness", () => {
   }
 });
 
-describe("openRouterResolve models.dev validity", async () => {
+describe("fallback chain resolution", () => {
-  const data = await api;
+  for (const alias of modelAliases.filter((a) => a.fallback)) {
-  const seen = new Set<string>();
+    it(`${alias.slug} fallback chain resolves to a non-deprecated model`, () => {
-
+      const resolved = resolveCliModel(alias.slug);
-  for (const alias of modelAliases) {
-    if (!alias.openRouterResolve) continue;
-    if (seen.has(alias.openRouterResolve)) continue;
-    seen.add(alias.openRouterResolve);
-
-    const parsed = parseResolve(alias.openRouterResolve);
-
-    it(`${alias.openRouterResolve} exists on models.dev`, () => {
-      const providerData = data[parsed.provider];
-      expect(providerData, `provider "${parsed.provider}" not found on models.dev`).toBeDefined();
-      const model = providerData.models[parsed.modelId];
       expect(
-        model,
+        resolved,
-        `model "${parsed.modelId}" not found under ${parsed.provider} on models.dev`
+        `fallback chain for "${alias.slug}" does not resolve to a non-deprecated model`
       ).toBeDefined();
     });
   }
 });
-
-type OpenRouterModel = { id: string };
-type OpenRouterModelsResponse = { data: OpenRouterModel[] };
-
-const openRouterApi = fetch("https://openrouter.ai/api/v1/models").then(
-  (r) => r.json() as Promise<OpenRouterModelsResponse>
-);
-
-describe("openRouterResolve OpenRouter API validity", async () => {
-  const orData = await openRouterApi;
-  const orModelIds = new Set(orData.data.map((m) => m.id));
-  const seen = new Set<string>();
-
-  for (const alias of modelAliases) {
-    if (!alias.openRouterResolve) continue;
-    const orModelId = alias.openRouterResolve.slice("openrouter/".length);
-    if (seen.has(orModelId)) continue;
-    seen.add(orModelId);
-
-    it(`${orModelId} exists on OpenRouter`, () => {
-      expect(
-        orModelIds.has(orModelId),
-        `model "${orModelId}" not found in OpenRouter API (/api/v1/models)`
-      ).toBe(true);
-    });
-  }
-});
-
-describe("latest model per provider snapshot", async () => {
-  const data = await api;
-  const providerKeys = Object.keys(providers) as ModelProvider[];
-
-  const latestByProvider: Record<string, { modelId: string; releaseDate: string }> = {};
-
-  for (const key of providerKeys) {
-    const providerData = data[key];
-    if (!providerData) continue;
-
-    let latest: { modelId: string; releaseDate: string } | undefined;
-    for (const [modelId, model] of Object.entries(providerData.models)) {
-      // skip non-GA models so beta/nightly churn doesn't break the snapshot
-      if (model.status) continue;
-      const rd = model.release_date;
-      if (!rd) continue;
-      // tiebreak by modelId for stable ordering when release dates match
-      if (
-        !latest ||
-        rd > latest.releaseDate ||
-        (rd === latest.releaseDate && modelId > latest.modelId)
-      ) {
-        latest = { modelId, releaseDate: rd };
-      }
-    }
-    if (latest) {
-      latestByProvider[key] = latest;
-    }
-  }
-
-  // when this fails, a provider shipped a new model. usually that just means
-  // bumping the `resolve` on an existing alias in models.ts (e.g. point
-  // "claude-opus" at the latest opus) rather than introducing a new alias.
-  // refresh the alias resolution first, then update this snapshot.
-  it("matches snapshot", () => {
-    expect(latestByProvider).toMatchSnapshot();
-  });
-});

test/run.ts

@@ -299,15 +299,21 @@ async function runTestForAgent(ctx: RunContext): Promise<ValidationResult> {
   env.PULLFROG_AGENT = ctx.agent;
 
   // override DB model to avoid mismatch when PULLFROG_AGENT forces a specific agent
-  // (DB model may belong to a different provider than the forced agent supports)
+  // (DB model may belong to a different provider than the forced agent supports).
+  // precedence: testConfig.env > process.env.PULLFROG_MODEL > per-agent default.
+  // the process.env pass-through lets CI (models-live matrix) pin an alias per job.
   if (!Object.hasOwn(env, "PULLFROG_MODEL")) {
-    const defaultModels: Record<string, string> = {
+    if (process.env.PULLFROG_MODEL) {
-      claude: "anthropic/claude-sonnet-4-6",
+      env.PULLFROG_MODEL = process.env.PULLFROG_MODEL;
-      opencode: "anthropic/claude-sonnet-4-6",
+    } else {
-    };
+      const defaultModels: Record<string, string> = {
-    const model = defaultModels[ctx.agent];
+        claude: "anthropic/claude-sonnet-4-6",
-    if (model) {
+        opencode: "anthropic/claude-sonnet-4-6",
-      env.PULLFROG_MODEL = model;
+      };
+      const model = defaultModels[ctx.agent];
+      if (model) {
+        env.PULLFROG_MODEL = model;
+      }
     }
   }
 

vitest.config.ts

@@ -4,7 +4,15 @@ export default defineConfig({
   test: {
     globals: true,
     environment: "node",
-    exclude: ["**/node_modules/**", "**/.temp/**", "**/.pnpm-store/**"],
+    exclude: [
+      "**/node_modules/**",
+      "**/.temp/**",
+      "**/.pnpm-store/**",
+      // *.main.test.ts files run only on main (e.g. catalog drift against
+      // models.dev + OpenRouter). run them via `pnpm test:catalog`, which
+      // points at vitest.main.config.ts.
+      "**/*.main.test.ts",
+    ],
     globalSetup: ["./vitest.global-setup.ts"],
     setupFiles: ["./vitest.setup.ts"],
   },

vitest.main.config.ts

@@ -0,0 +1,15 @@
+import { defineConfig } from "vitest/config";
+
+// config for main-only tests (see *.main.test.ts). runs the catalog drift
+// suite explicitly; the default vitest.config.ts excludes these files so PR
+// CI stays unaffected by upstream catalog changes.
+export default defineConfig({
+  test: {
+    globals: true,
+    environment: "node",
+    include: ["**/*.main.test.ts"],
+    exclude: ["**/node_modules/**", "**/.temp/**", "**/.pnpm-store/**"],
+    globalSetup: ["./vitest.global-setup.ts"],
+    setupFiles: ["./vitest.setup.ts"],
+  },
+});