diff --git a/test/list-aliases.ts b/test/list-aliases.ts index 18e2cbd..fbfdaee 100644 --- a/test/list-aliases.ts +++ b/test/list-aliases.ts @@ -1,67 +1,95 @@ /** - * emits a JSON array of { slug, agent, name } entries for the `models-live` - * matrix job. `agent` is auto-derived from the alias provider and matches the - * harness the runtime would pick in production. + * emits a JSON array of { slug, agent, name } entries for one of two CI matrix + * jobs. `agent` mirrors the harness the runtime would pick in production + * (anthropic/* → claude-code, everything else → opencode). * - * set MATRIX_FILTER to a substring to restrict the matrix to matching aliases - * — useful for iterating on a single provider without paying for every model. + * MODE=aliases (default) — every alias minus pruned passthroughs. consumed by + * `models-live`, which runs the cheap top-level CLI smoke per alias + * (`action/test/model-smoke.ts`) to validate resolution + auth. * - * passthrough pruning: openrouter/* aliases and keyed opencode/* aliases are - * just routing-layer wrappers around models we already smoke-test directly - * (anthropic/*, openai/*, google/*, etc). running every passthrough burns CI - * minutes without catching anything the direct smoke doesn't. we keep one - * canary per routing layer to validate the routing layer itself is alive; - * slug-drift is caught separately by the `models-catalog` job. set - * INCLUDE_ALL_PASSTHROUGHS=1 to bypass this for full validation. + * MODE=flagships — one standard-tier model per provider. consumed by + * `providers-live`, which runs the full harness smoke + * (`pnpm runtest smoke `) to validate provider-class tool-calling + * (e.g. Gemini schema sanitizer, OpenAI tool-call format). + * + * passthrough pruning (aliases mode): openrouter/* aliases and keyed opencode/* + * aliases are routing-layer wrappers around models we already smoke-test + * directly. running every passthrough burns CI minutes without catching + * anything new — slug-drift is covered by the `models-catalog` job. one canary + * per routing layer proves the routing surface (auth, tool-call translation) + * is alive; set INCLUDE_PASSTHROUGHS=1 to bypass for full validation. * * usage: * node action/test/list-aliases.ts + * MODE=flagships node action/test/list-aliases.ts * MATRIX_FILTER=gemini node action/test/list-aliases.ts - * INCLUDE_ALL_PASSTHROUGHS=1 node action/test/list-aliases.ts - * INCLUDE_EXPENSIVE=1 node action/test/list-aliases.ts + * INCLUDE_PASSTHROUGHS=1 node action/test/list-aliases.ts */ import { modelAliases } from "../models.ts"; -function agentForSlug(slug: string): "claude" | "opencode" { - return slug.startsWith("anthropic/") ? "claude" : "opencode"; -} - -// one canary per routing layer — proves the routing surface (auth, tool-call -// translation) is alive without re-testing every underlying model. const ROUTING_CANARIES = new Set(["openrouter/claude-sonnet", "opencode/claude-sonnet"]); -// pruned by default; opt back in with INCLUDE_EXPENSIVE=1 or MATRIX_FILTER. -// matched against `alias.resolve` so every routing layer (openai/, opencode/, -// openrouter/) is covered without enumerating each slug separately. -// gpt-5.5-pro burns ~$2.40/run on this fixture — too expensive per-push. -const EXPENSIVE_RESOLVE_SUBSTRINGS = ["gpt-5.5-pro"]; - -function isExpensive(alias: (typeof modelAliases)[number]): boolean { - return EXPENSIVE_RESOLVE_SUBSTRINGS.some((s) => alias.resolve.includes(s)); -} +// hand-picked "standard good model" per provider — not the pro/opus tier (too +// expensive for per-push) and not the free/experimental tier (too flaky). these +// aliases anchor the harness smoke job that catches provider-class regressions +// like Gemini schema sanitization or OpenAI tool-call format drift. the +// assertion below catches slug-drift loudly, but adding a NEW provider without +// an entry here silently omits it from `providers-live` — see +// wiki/models-catalog.md "To add a provider". +const FLAGSHIPS = [ + "anthropic/claude-sonnet", + "openai/gpt", + "google/gemini-pro", + "xai/grok", + "deepseek/deepseek-pro", + "moonshotai/kimi-k2", + "opencode/big-pickle", + "openrouter/claude-sonnet", +]; function isPrunablePassthrough(alias: (typeof modelAliases)[number]): boolean { if (ROUTING_CANARIES.has(alias.slug)) return false; if (alias.provider === "openrouter") return true; // opencode FREE models (big-pickle, mimo, minimax, gpt-5-nano) are unique // to opencode and used in prod — keep them. only prune the keyed mirrors. - if (alias.provider === "opencode" && !alias.isFree) return true; - return false; + return alias.provider === "opencode" && !alias.isFree; } -const filter = process.env.MATRIX_FILTER?.trim() ?? ""; -const includeAllPassthroughs = process.env.INCLUDE_ALL_PASSTHROUGHS === "1"; -const includeExpensive = process.env.INCLUDE_EXPENSIVE === "1" || filter !== ""; - -const matrix = modelAliases - .filter((alias) => (filter ? alias.slug.toLowerCase().includes(filter.toLowerCase()) : true)) - .filter((alias) => includeAllPassthroughs || !isPrunablePassthrough(alias)) - .filter((alias) => includeExpensive || !isExpensive(alias)) - .map((alias) => ({ +function toMatrixEntry(alias: (typeof modelAliases)[number]) { + return { slug: alias.slug, - agent: agentForSlug(alias.slug), + agent: alias.slug.startsWith("anthropic/") ? "claude" : "opencode", // readable display name (GHA renders slashes awkwardly in matrix job titles) name: alias.slug.replace("/", "-"), - })); + }; +} + +const mode = process.env.MODE === "flagships" ? "flagships" : "aliases"; +const filter = process.env.MATRIX_FILTER?.trim().toLowerCase() ?? ""; +const includePassthroughs = process.env.INCLUDE_PASSTHROUGHS === "1"; + +const aliasBySlug = new Map(modelAliases.map((a) => [a.slug, a])); +const matrix = (() => { + if (mode === "flagships") { + return FLAGSHIPS.map((slug) => { + const alias = aliasBySlug.get(slug); + if (!alias) { + throw new Error( + `list-aliases: flagship "${slug}" missing from modelAliases — update FLAGSHIPS` + ); + } + return alias; + }) + .filter((alias) => !filter || alias.slug.toLowerCase().includes(filter)) + .map(toMatrixEntry); + } + return modelAliases + .filter((alias) => { + if (filter && !alias.slug.toLowerCase().includes(filter)) return false; + if (!includePassthroughs && isPrunablePassthrough(alias)) return false; + return true; + }) + .map(toMatrixEntry); +})(); process.stdout.write(JSON.stringify(matrix)); diff --git a/test/model-smoke.ts b/test/model-smoke.ts new file mode 100644 index 0000000..9b535d6 --- /dev/null +++ b/test/model-smoke.ts @@ -0,0 +1,170 @@ +/** + * model-smoke: per-alias resolution + auth check that bypasses the Pullfrog + * harness. resolves a model alias to its concrete provider/model + agent CLI, + * invokes the CLI directly with a trivial "reply OK" prompt, and asserts the + * provider replied. validates exactly the surface that changes when models.ts + * changes — alias → resolve mapping, agent classification, env-var wiring — + * without booting Docker, MCP, or the full agent runtime. + * + * tool-calling correctness is a property of the underlying model, not the + * alias; the `providers-live` job runs the full harness smoke once per + * provider (one standard-tier model each), which is enough. + * + * usage: + * node action/test/model-smoke.ts --slug openai/gpt + * PULLFROG_MODEL=openai/gpt node action/test/model-smoke.ts + */ +import { spawn } from "node:child_process"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { config } from "dotenv"; +import { modelAliases, resolveCliModel } from "../models.ts"; +import { installFromNpmTarball } from "../utils/install.ts"; +import { getDevDependencyVersion } from "../utils/version.ts"; + +config({ path: join(import.meta.dirname, "..", ".env") }); +config({ path: join(import.meta.dirname, "..", "..", ".env") }); + +const PROMPT = "Reply with exactly OK and nothing else."; +const MATCH = /\bOK\b/i; +const TIMEOUT_MS = 60_000; + +function parseSlug(): string { + const argIdx = process.argv.indexOf("--slug"); + if (argIdx >= 0 && process.argv[argIdx + 1]) return process.argv[argIdx + 1]; + if (process.env.PULLFROG_MODEL) return process.env.PULLFROG_MODEL; + throw new Error("model-smoke: pass --slug or set PULLFROG_MODEL"); +} + +type Plan = + | { agent: "opencode"; cliPath: string; args: string[] } + | { agent: "claude"; cliPath: string; args: string[] }; + +async function plan(slug: string): Promise { + const alias = modelAliases.find((a) => a.slug === slug); + if (!alias) throw new Error(`model-smoke: unknown alias "${slug}"`); + + // walk the fallback chain so deprecated aliases (those with `fallback` set, + // e.g. opencode/mimo-v2-pro-free → opencode/big-pickle) hit their replacement + // instead of the dead resolve target. mirrors production via resolveCliModel. + const cliModel = resolveCliModel(slug); + if (!cliModel) throw new Error(`model-smoke: fallback chain for "${slug}" is broken or cyclic`); + + // anthropic/* aliases run through claude-code in production; everything else + // (openai, google, xai, deepseek, moonshot, opencode, openrouter) runs through + // opencode. mirrors the inline classification in list-aliases.ts toMatrixEntry(). + if (slug.startsWith("anthropic/")) { + const cliPath = await installFromNpmTarball({ + packageName: "@anthropic-ai/claude-code", + version: getDevDependencyVersion("@anthropic-ai/claude-code"), + executablePath: "cli.js", + installDependencies: false, + }); + // claude expects a bare model id (e.g. "claude-sonnet-4-6"), not "anthropic/claude-sonnet-4-6" + const bareModel = cliModel.split("/").slice(1).join("/"); + return { + agent: "claude", + cliPath, + args: [cliPath, "-p", PROMPT, "--model", bareModel], + }; + } + + const cliPath = await installFromNpmTarball({ + packageName: "opencode-ai", + version: getDevDependencyVersion("opencode-ai"), + executablePath: "bin/opencode", + installDependencies: true, + }); + return { + agent: "opencode", + cliPath, + args: ["run", "--model", cliModel, PROMPT], + }; +} + +type SpawnResult = { ok: boolean; output: string; reason: string }; + +function runCli(p: Plan, env: NodeJS.ProcessEnv): Promise { + // claude's cli.js shebangs to env node, but we invoke node explicitly to + // avoid PATH-resolution surprises in CI runners; opencode is a real binary. + const command = p.agent === "claude" ? "node" : p.cliPath; + + return new Promise((resolve) => { + const child = spawn(command, p.args, { env, stdio: ["ignore", "pipe", "pipe"] }); + + let stdout = ""; + let stderr = ""; + child.stdout?.on("data", (chunk: Buffer) => { + stdout += chunk.toString(); + }); + child.stderr?.on("data", (chunk: Buffer) => { + stderr += chunk.toString(); + }); + + const timer = setTimeout(() => { + child.kill("SIGKILL"); + }, TIMEOUT_MS); + + child.on("close", (code, signal) => { + clearTimeout(timer); + const output = stdout + (stderr ? `\n---stderr---\n${stderr}` : ""); + if (signal === "SIGKILL") { + resolve({ ok: false, output, reason: `timed out after ${TIMEOUT_MS / 1000}s` }); + return; + } + if (code !== 0) { + resolve({ ok: false, output, reason: `exit ${code}` }); + return; + } + if (!MATCH.test(stdout)) { + resolve({ ok: false, output, reason: "no OK in stdout" }); + return; + } + resolve({ ok: true, output, reason: "ok" }); + }); + + child.on("error", (err) => { + clearTimeout(timer); + resolve({ ok: false, output: stderr, reason: `spawn error: ${err.message}` }); + }); + }); +} + +async function main(): Promise { + const slug = parseSlug(); + const tempDir = mkdtempSync(join(tmpdir(), "model-smoke-")); + const homeDir = join(tempDir, "home"); + + // installFromNpmTarball reads PULLFROG_TEMP_DIR from process.env, not from + // the spawn env, so we mutate process.env up-front. HOME/XDG_CONFIG_HOME are + // redirected to keep the agent CLIs from picking up the dev user's config. + process.env.PULLFROG_TEMP_DIR = tempDir; + process.env.HOME = homeDir; + process.env.XDG_CONFIG_HOME = join(homeDir, ".config"); + // opencode reads GOOGLE_GENERATIVE_AI_API_KEY for gemini; mirror the harness fallback. + if (!process.env.GOOGLE_GENERATIVE_AI_API_KEY && process.env.GEMINI_API_KEY) { + process.env.GOOGLE_GENERATIVE_AI_API_KEY = process.env.GEMINI_API_KEY; + } + + console.log(`» model-smoke ${slug}`); + const p = await plan(slug); + console.log( + `» agent=${p.agent} cmd=${[p.agent === "claude" ? "node" : p.cliPath, ...p.args].join(" ")}` + ); + + const result = await runCli(p, process.env); + if (result.ok) { + console.log(`✓ ${slug} (${p.agent})`); + process.exit(0); + } + + console.error(`✗ ${slug} (${p.agent}): ${result.reason}`); + if (result.output) console.error(result.output); + process.exit(1); +} + +main().catch((err: unknown) => { + console.error(err instanceof Error ? (err.stack ?? err.message) : String(err)); + process.exit(1); +});