add models-bump cron + drop snapshot test
every 12h, scripts/find-newer-models.ts scans models.dev for newer GA versions of every alias in action/models.ts and writes a focused per-alias diff. .github/workflows/models-bump.yml short-circuits when no candidates exist; otherwise hands the diff to pullfrog/pullfrog@main to evaluate against the policy in wiki/model-resolution.md and open a single living PR on the pullfrog/models-bump branch. drops the brittle "latest model per provider" snapshot block in action/test/models-catalog.main.test.ts (and its .snap file) — the cron keeps the registry in sync with upstreams, and the remaining validity tests act as the integrity gate on the bump PR.
This commit is contained in:
committed by
pullfrog[bot]
parent
5f3e46c42d
commit
3c9799adda
@@ -1,38 +0,0 @@
|
|||||||
// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
|
|
||||||
|
|
||||||
exports[`latest model per provider snapshot > matches snapshot 1`] = `
|
|
||||||
{
|
|
||||||
"anthropic": {
|
|
||||||
"modelId": "claude-opus-4-7",
|
|
||||||
"releaseDate": "2026-04-16",
|
|
||||||
},
|
|
||||||
"deepseek": {
|
|
||||||
"modelId": "deepseek-v4-pro",
|
|
||||||
"releaseDate": "2026-04-24",
|
|
||||||
},
|
|
||||||
"google": {
|
|
||||||
"modelId": "gemini-3.1-flash-lite",
|
|
||||||
"releaseDate": "2026-05-07",
|
|
||||||
},
|
|
||||||
"moonshotai": {
|
|
||||||
"modelId": "kimi-k2.6",
|
|
||||||
"releaseDate": "2026-04-21",
|
|
||||||
},
|
|
||||||
"openai": {
|
|
||||||
"modelId": "gpt-5.5-pro",
|
|
||||||
"releaseDate": "2026-04-23",
|
|
||||||
},
|
|
||||||
"opencode": {
|
|
||||||
"modelId": "gpt-5.5-pro",
|
|
||||||
"releaseDate": "2026-04-24",
|
|
||||||
},
|
|
||||||
"openrouter": {
|
|
||||||
"modelId": "x-ai/grok-4.3",
|
|
||||||
"releaseDate": "2026-05-01",
|
|
||||||
},
|
|
||||||
"xai": {
|
|
||||||
"modelId": "grok-4.3",
|
|
||||||
"releaseDate": "2026-05-01",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
`;
|
|
||||||
@@ -1,5 +1,5 @@
|
|||||||
import { describe, expect, it } from "vitest";
|
import { describe, expect, it } from "vitest";
|
||||||
import { type ModelProvider, modelAliases, providers } from "../models.ts";
|
import { modelAliases } from "../models.ts";
|
||||||
|
|
||||||
// ── catalog drift tests — main-only ─────────────────────────────────────────────
|
// ── catalog drift tests — main-only ─────────────────────────────────────────────
|
||||||
//
|
//
|
||||||
@@ -8,6 +8,12 @@ import { type ModelProvider, modelAliases, providers } from "../models.ts";
|
|||||||
// catalog drift (new model ships, old model deprecated, etc.) causes failures
|
// catalog drift (new model ships, old model deprecated, etc.) causes failures
|
||||||
// that are unrelated to any code change in the PR — so these run only on main.
|
// that are unrelated to any code change in the PR — so these run only on main.
|
||||||
//
|
//
|
||||||
|
// the registry is kept in sync with upstreams by the `models-bump` cron
|
||||||
|
// (`.github/workflows/models-bump.yml`), which scans models.dev every 12h and
|
||||||
|
// opens a PR bumping `resolve` / `openRouterResolve` for any alias whose
|
||||||
|
// upstream has shipped a newer GA version. these tests are the integrity gate
|
||||||
|
// for that PR — they catch typos, removed models, and openrouter mismatches.
|
||||||
|
//
|
||||||
// run locally with `pnpm test:catalog`.
|
// run locally with `pnpm test:catalog`.
|
||||||
// in CI, gated to push events on main.
|
// in CI, gated to push events on main.
|
||||||
|
|
||||||
@@ -106,40 +112,3 @@ describe("openRouterResolve OpenRouter API validity", async () => {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
describe("latest model per provider snapshot", async () => {
|
|
||||||
const data = await api;
|
|
||||||
const providerKeys = Object.keys(providers) as ModelProvider[];
|
|
||||||
|
|
||||||
const latestByProvider: Record<string, { modelId: string; releaseDate: string }> = {};
|
|
||||||
|
|
||||||
for (const key of providerKeys) {
|
|
||||||
const providerData = data[key];
|
|
||||||
if (!providerData) continue;
|
|
||||||
|
|
||||||
let latest: { modelId: string; releaseDate: string } | undefined;
|
|
||||||
for (const [modelId, model] of Object.entries(providerData.models)) {
|
|
||||||
// skip non-GA models so beta/nightly churn doesn't break the snapshot
|
|
||||||
if (model.status) continue;
|
|
||||||
const rd = model.release_date;
|
|
||||||
if (!rd) continue;
|
|
||||||
// tiebreak by modelId for stable ordering when release dates match
|
|
||||||
if (
|
|
||||||
!latest ||
|
|
||||||
rd > latest.releaseDate ||
|
|
||||||
(rd === latest.releaseDate && modelId > latest.modelId)
|
|
||||||
) {
|
|
||||||
latest = { modelId, releaseDate: rd };
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (latest) {
|
|
||||||
latestByProvider[key] = latest;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// when this fails, a provider shipped a new model. check whether we need
|
|
||||||
// to add or update an alias in models.ts before updating the snapshot.
|
|
||||||
it("matches snapshot", () => {
|
|
||||||
expect(latestByProvider).toMatchSnapshot();
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|||||||
Reference in New Issue
Block a user