Colin McDonnell
5e6ff67623
PR CI kept breaking on upstream catalog drift (new model ships on models.dev, OpenRouter renames an id, etc.) — failures unrelated to the PR's contents. split the model-alias test suite so PRs only see pure-logic checks, and push the external-state drift + end-to-end coverage to main. test organization: - action/test/models.test.ts keeps pure invariants: openRouterResolve completeness and fallback-chain resolution. runs on every PR. - action/test/models-catalog.main.test.ts gets the 4 network-dependent describes (models.dev validity x2, OpenRouter API validity, latest-model snapshot). runs only on main push via a dedicated vitest config (vitest.main.config.ts + `pnpm test:catalog`). new CI jobs in .github/workflows/test.yml: - models-catalog: `pnpm test:catalog` on every main push. detects upstream catalog drift so we can react at the next convenient window. - models-live: 38-entry matrix that invokes the agent harness end-to-end against the real provider for each alias in models.ts. generated from action/test/list-aliases.ts. runs only on main push AND only when resolution-affecting files changed (action/models.ts, action/package.json, action/agents/**) — the exact shape of the opus 4.7 incident. test/run.ts: PULLFROG_MODEL now flows through from process.env so the live matrix can pin an alias per job without the per-agent default clobbering it. Made-with: Cursor
45 lines
1.7 KiB
TypeScript
45 lines
1.7 KiB
TypeScript
import { describe, expect, it } from "vitest";
|
|
import { modelAliases, resolveCliModel } from "../models.ts";
|
|
|
|
// ── pure alias-registry invariants ──────────────────────────────────────────────
|
|
//
|
|
// these tests validate our alias data structure without hitting external APIs.
|
|
// network-dependent checks (models.dev / OpenRouter catalog drift, latest-model
|
|
// snapshot) live in models-catalog.main.test.ts and run only on main.
|
|
|
|
// models that have no OpenRouter equivalent and require BYOK.
|
|
// add a model here ONLY when it genuinely doesn't exist on both models.dev and OpenRouter.
|
|
const BYOK_ONLY_MODELS = new Set(["openai/o3"]);
|
|
|
|
describe("openRouterResolve completeness", () => {
|
|
for (const alias of modelAliases) {
|
|
if (alias.isFree) continue;
|
|
if (BYOK_ONLY_MODELS.has(alias.slug)) continue;
|
|
it(`${alias.slug} has openRouterResolve`, () => {
|
|
expect(
|
|
alias.openRouterResolve,
|
|
`non-free model "${alias.slug}" is missing openRouterResolve — add it or add to BYOK_ONLY_MODELS`
|
|
).toBeDefined();
|
|
});
|
|
}
|
|
|
|
for (const alias of modelAliases) {
|
|
if (!alias.isFree) continue;
|
|
it(`${alias.slug} (free) does not need openRouterResolve`, () => {
|
|
expect(alias.openRouterResolve).toBeUndefined();
|
|
});
|
|
}
|
|
});
|
|
|
|
describe("fallback chain resolution", () => {
|
|
for (const alias of modelAliases.filter((a) => a.fallback)) {
|
|
it(`${alias.slug} fallback chain resolves to a non-deprecated model`, () => {
|
|
const resolved = resolveCliModel(alias.slug);
|
|
expect(
|
|
resolved,
|
|
`fallback chain for "${alias.slug}" does not resolve to a non-deprecated model`
|
|
).toBeDefined();
|
|
});
|
|
}
|
|
});
|