test: switch opus defaults to sonnet/kimi for cheaper coverage

- opencode test default: claude-sonnet-4-6 → moonshotai/kimi-k2. claude-code
  default stays on sonnet (the agent-under-test for that path is Claude
  Code); the opencode path doesn't need an Anthropic model.
- byok-no-keys-fallback fixture: anthropic/claude-opus → moonshotai/kimi-k2.
  test verifies fallback to opencode/big-pickle, so the configured model
  is never invoked; picking a non-opus alias avoids burning credits if
  the fallback ever regresses.
- BEDROCK_MODEL_ID in both test workflows: us.anthropic.claude-opus-4-6-v1
  → us.anthropic.claude-sonnet-4-6. env var is required by the ci.test
  invariant that every provider's env vars are wired into the workflow.
This commit is contained in:
Colin McDonnell
2026-05-23 01:07:20 +00:00
committed by pullfrog[bot]
parent fd2c67ab50
commit e52206b8ca
3 changed files with 7 additions and 7 deletions
+1 -1
View File
@@ -62,7 +62,7 @@ jobs:
OPENCODE_API_KEY: ${{ secrets.OPENCODE_API_KEY }} OPENCODE_API_KEY: ${{ secrets.OPENCODE_API_KEY }}
AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }} AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }}
AWS_REGION: us-east-1 AWS_REGION: us-east-1
BEDROCK_MODEL_ID: us.anthropic.claude-opus-4-6-v1 BEDROCK_MODEL_ID: us.anthropic.claude-sonnet-4-6
VERTEX_SERVICE_ACCOUNT_JSON: ${{ secrets.VERTEX_SERVICE_ACCOUNT_JSON }} VERTEX_SERVICE_ACCOUNT_JSON: ${{ secrets.VERTEX_SERVICE_ACCOUNT_JSON }}
GOOGLE_CLOUD_PROJECT: pullfrog GOOGLE_CLOUD_PROJECT: pullfrog
VERTEX_LOCATION: global VERTEX_LOCATION: global
+5 -5
View File
@@ -3,7 +3,7 @@ import { defineFixture, getAgentOutput } from "../utils.ts";
/** /**
* BYOK-no-keys fallback test — proves that an account configured for a * BYOK-no-keys fallback test — proves that an account configured for a
* BYOK model (here: `anthropic/claude-opus`) but with no provider API * BYOK model (here: `moonshotai/kimi-k2`) but with no provider API
* keys present in the runner env still gets a successful run by falling * keys present in the runner env still gets a successful run by falling
* back to a free OpenCode model. * back to a free OpenCode model.
* *
@@ -59,10 +59,10 @@ export const test: TestRunnerOptions = {
BEDROCK_MODEL_ID: "", BEDROCK_MODEL_ID: "",
// configure a model that requires a BYOK key — the fallback only // configure a model that requires a BYOK key — the fallback only
// engages when there's a configured model whose provider key is // engages when there's a configured model whose provider key is
// absent, so we have to pin one. anthropic/claude-opus is the // absent, so we have to pin one. any BYOK alias works; we pick
// most common first-run choice (it's the catalog "preferred" for // a cheap non-Anthropic model so the test doesn't burn opus
// the anthropic provider). // credits if the fallback ever regresses.
PULLFROG_MODEL: "anthropic/claude-opus", PULLFROG_MODEL: "moonshotai/kimi-k2",
}, },
tags: ["agnostic"], tags: ["agnostic"],
coverage: [ coverage: [
+1 -1
View File
@@ -312,7 +312,7 @@ async function runTestForAgent(ctx: RunContext): Promise<ValidationResult> {
} else { } else {
const defaultModels: Record<string, string> = { const defaultModels: Record<string, string> = {
claude: "anthropic/claude-sonnet-4-6", claude: "anthropic/claude-sonnet-4-6",
opencode: "anthropic/claude-sonnet-4-6", opencode: "moonshotai/kimi-k2",
}; };
const model = defaultModels[ctx.agent]; const model = defaultModels[ctx.agent];
if (model) { if (model) {