add models-bump cron + drop snapshot test

every 12h, scripts/find-newer-models.ts scans models.dev for newer GA
versions of every alias in action/models.ts and writes a focused
per-alias diff. .github/workflows/models-bump.yml short-circuits when
no candidates exist; otherwise hands the diff to pullfrog/pullfrog@main
to evaluate against the policy in wiki/model-resolution.md and open a
single living PR on the pullfrog/models-bump branch.

drops the brittle "latest model per provider" snapshot block in
action/test/models-catalog.main.test.ts (and its .snap file) — the cron
keeps the registry in sync with upstreams, and the remaining validity
tests act as the integrity gate on the bump PR.
This commit is contained in:
Colin McDonnell
2026-05-08 23:27:42 +00:00
committed by pullfrog[bot]
parent 5f3e46c42d
commit 3c9799adda
2 changed files with 7 additions and 76 deletions
@@ -1,38 +0,0 @@
// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
exports[`latest model per provider snapshot > matches snapshot 1`] = `
{
"anthropic": {
"modelId": "claude-opus-4-7",
"releaseDate": "2026-04-16",
},
"deepseek": {
"modelId": "deepseek-v4-pro",
"releaseDate": "2026-04-24",
},
"google": {
"modelId": "gemini-3.1-flash-lite",
"releaseDate": "2026-05-07",
},
"moonshotai": {
"modelId": "kimi-k2.6",
"releaseDate": "2026-04-21",
},
"openai": {
"modelId": "gpt-5.5-pro",
"releaseDate": "2026-04-23",
},
"opencode": {
"modelId": "gpt-5.5-pro",
"releaseDate": "2026-04-24",
},
"openrouter": {
"modelId": "x-ai/grok-4.3",
"releaseDate": "2026-05-01",
},
"xai": {
"modelId": "grok-4.3",
"releaseDate": "2026-05-01",
},
}
`;
+7 -38
View File
@@ -1,5 +1,5 @@
import { describe, expect, it } from "vitest";
import { type ModelProvider, modelAliases, providers } from "../models.ts";
import { modelAliases } from "../models.ts";
// ── catalog drift tests — main-only ─────────────────────────────────────────────
//
@@ -8,6 +8,12 @@ import { type ModelProvider, modelAliases, providers } from "../models.ts";
// catalog drift (new model ships, old model deprecated, etc.) causes failures
// that are unrelated to any code change in the PR — so these run only on main.
//
// the registry is kept in sync with upstreams by the `models-bump` cron
// (`.github/workflows/models-bump.yml`), which scans models.dev every 12h and
// opens a PR bumping `resolve` / `openRouterResolve` for any alias whose
// upstream has shipped a newer GA version. these tests are the integrity gate
// for that PR — they catch typos, removed models, and openrouter mismatches.
//
// run locally with `pnpm test:catalog`.
// in CI, gated to push events on main.
@@ -106,40 +112,3 @@ describe("openRouterResolve OpenRouter API validity", async () => {
});
}
});
describe("latest model per provider snapshot", async () => {
const data = await api;
const providerKeys = Object.keys(providers) as ModelProvider[];
const latestByProvider: Record<string, { modelId: string; releaseDate: string }> = {};
for (const key of providerKeys) {
const providerData = data[key];
if (!providerData) continue;
let latest: { modelId: string; releaseDate: string } | undefined;
for (const [modelId, model] of Object.entries(providerData.models)) {
// skip non-GA models so beta/nightly churn doesn't break the snapshot
if (model.status) continue;
const rd = model.release_date;
if (!rd) continue;
// tiebreak by modelId for stable ordering when release dates match
if (
!latest ||
rd > latest.releaseDate ||
(rd === latest.releaseDate && modelId > latest.modelId)
) {
latest = { modelId, releaseDate: rd };
}
}
if (latest) {
latestByProvider[key] = latest;
}
}
// when this fails, a provider shipped a new model. check whether we need
// to add or update an alias in models.ts before updating the snapshot.
it("matches snapshot", () => {
expect(latestByProvider).toMatchSnapshot();
});
});