Colin McDonnell
c6a3ee0e9a
* show model name in footer, drop pullfrog.com link add model slug to buildPullfrogFooter so every Pullfrog comment displays the active model (e.g. "Using `Big Pickle` (free)" or "Using `Claude Opus`"). remove the pullfrog.com link from all footers. Made-with: Cursor * reject <br/> tags in comment bodies, add prompt guidance add runtime validation in addFooter that throws if <br/> is followed by a non-blank line (breaks GitHub heading rendering). the agent sees the error and retries with clean markdown. also update Summarize mode prompt to explicitly forbid <br/> tags. Made-with: Cursor * fix <br/> guidance: move to event instructions, clarify blank line rule the formatting rule belongs in DEFAULT_PR_SUMMARY_INSTRUCTIONS (event instructions), not the Summarize mode prompt. clarify that <br/> must always be followed by a blank line before headings. Made-with: Cursor * generalize block-level HTML rule in summary instructions add a prominent top-level rule about requiring blank lines between ALL block-level HTML elements and markdown syntax, not just <br/>. Made-with: Cursor * move model to toolState instead of threading through params model is set once at startup and read everywhere — it belongs on toolState, not threaded as a separate param through 8 call sites. postCleanup runs without toolState so it just omits the model label. Made-with: Cursor * update models.dev snapshot (openai latest changed) Made-with: Cursor * add comment to models snapshot test explaining its purpose Made-with: Cursor
85 lines
2.8 KiB
TypeScript
85 lines
2.8 KiB
TypeScript
import { describe, expect, it } from "vitest";
|
|
import { type ModelProvider, modelAliases, providers } from "../models.ts";
|
|
|
|
type ModelsDevModel = {
|
|
name: string;
|
|
status?: string;
|
|
release_date?: string;
|
|
};
|
|
|
|
type ModelsDevProvider = {
|
|
name: string;
|
|
models: Record<string, ModelsDevModel>;
|
|
};
|
|
|
|
type ModelsDevApi = Record<string, ModelsDevProvider>;
|
|
|
|
const api = fetch("https://models.dev/api.json").then((r) => r.json() as Promise<ModelsDevApi>);
|
|
|
|
/** split a resolve slug into the models.dev provider key and model key */
|
|
function parseResolve(resolve: string): { provider: string; modelId: string } {
|
|
const idx = resolve.indexOf("/");
|
|
return { provider: resolve.slice(0, idx), modelId: resolve.slice(idx + 1) };
|
|
}
|
|
|
|
describe("models.dev validity", async () => {
|
|
const data = await api;
|
|
|
|
for (const alias of modelAliases) {
|
|
const parsed = parseResolve(alias.resolve);
|
|
|
|
it(`${alias.resolve} exists on models.dev`, () => {
|
|
const providerData = data[parsed.provider];
|
|
expect(providerData, `provider "${parsed.provider}" not found on models.dev`).toBeDefined();
|
|
const model = providerData.models[parsed.modelId];
|
|
expect(
|
|
model,
|
|
`model "${parsed.modelId}" not found under ${parsed.provider} on models.dev`
|
|
).toBeDefined();
|
|
});
|
|
|
|
it(`${alias.resolve} is not deprecated`, () => {
|
|
const model = data[parsed.provider]?.models[parsed.modelId];
|
|
if (!model) return; // covered by existence test above
|
|
expect(model.status, `${alias.resolve} is deprecated on models.dev`).not.toBe("deprecated");
|
|
});
|
|
}
|
|
});
|
|
|
|
describe("latest model per provider snapshot", async () => {
|
|
const data = await api;
|
|
const providerKeys = Object.keys(providers) as ModelProvider[];
|
|
|
|
const latestByProvider: Record<string, { modelId: string; releaseDate: string }> = {};
|
|
|
|
for (const key of providerKeys) {
|
|
const providerData = data[key];
|
|
if (!providerData) continue;
|
|
|
|
let latest: { modelId: string; releaseDate: string } | undefined;
|
|
for (const [modelId, model] of Object.entries(providerData.models)) {
|
|
// skip non-GA models so beta/nightly churn doesn't break the snapshot
|
|
if (model.status) continue;
|
|
const rd = model.release_date;
|
|
if (!rd) continue;
|
|
// tiebreak by modelId for stable ordering when release dates match
|
|
if (
|
|
!latest ||
|
|
rd > latest.releaseDate ||
|
|
(rd === latest.releaseDate && modelId > latest.modelId)
|
|
) {
|
|
latest = { modelId, releaseDate: rd };
|
|
}
|
|
}
|
|
if (latest) {
|
|
latestByProvider[key] = latest;
|
|
}
|
|
}
|
|
|
|
// when this fails, a provider shipped a new model. check whether we need
|
|
// to add or update an alias in models.ts before updating the snapshot.
|
|
it("matches snapshot", () => {
|
|
expect(latestByProvider).toMatchSnapshot();
|
|
});
|
|
});
|