test: switch opus defaults to sonnet/kimi for cheaper coverage

- opencode test default: claude-sonnet-4-6 → moonshotai/kimi-k2. claude-code
  default stays on sonnet (the agent-under-test for that path is Claude
  Code); the opencode path doesn't need an Anthropic model.
- byok-no-keys-fallback fixture: anthropic/claude-opus → moonshotai/kimi-k2.
  test verifies fallback to opencode/big-pickle, so the configured model
  is never invoked; picking a non-opus alias avoids burning credits if
  the fallback ever regresses.
- BEDROCK_MODEL_ID in both test workflows: us.anthropic.claude-opus-4-6-v1
  → us.anthropic.claude-sonnet-4-6. env var is required by the ci.test
  invariant that every provider's env vars are wired into the workflow.
This commit is contained in:
Colin McDonnell
2026-05-23 01:07:20 +00:00
committed by pullfrog[bot]
parent fd2c67ab50
commit e52206b8ca
3 changed files with 7 additions and 7 deletions
+1 -1
View File
@@ -62,7 +62,7 @@ jobs:
OPENCODE_API_KEY: ${{ secrets.OPENCODE_API_KEY }}
AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }}
AWS_REGION: us-east-1
BEDROCK_MODEL_ID: us.anthropic.claude-opus-4-6-v1
BEDROCK_MODEL_ID: us.anthropic.claude-sonnet-4-6
VERTEX_SERVICE_ACCOUNT_JSON: ${{ secrets.VERTEX_SERVICE_ACCOUNT_JSON }}
GOOGLE_CLOUD_PROJECT: pullfrog
VERTEX_LOCATION: global
+5 -5
View File
@@ -3,7 +3,7 @@ import { defineFixture, getAgentOutput } from "../utils.ts";
/**
* BYOK-no-keys fallback test — proves that an account configured for a
* BYOK model (here: `anthropic/claude-opus`) but with no provider API
* BYOK model (here: `moonshotai/kimi-k2`) but with no provider API
* keys present in the runner env still gets a successful run by falling
* back to a free OpenCode model.
*
@@ -59,10 +59,10 @@ export const test: TestRunnerOptions = {
BEDROCK_MODEL_ID: "",
// configure a model that requires a BYOK key — the fallback only
// engages when there's a configured model whose provider key is
// absent, so we have to pin one. anthropic/claude-opus is the
// most common first-run choice (it's the catalog "preferred" for
// the anthropic provider).
PULLFROG_MODEL: "anthropic/claude-opus",
// absent, so we have to pin one. any BYOK alias works; we pick
// a cheap non-Anthropic model so the test doesn't burn opus
// credits if the fallback ever regresses.
PULLFROG_MODEL: "moonshotai/kimi-k2",
},
tags: ["agnostic"],
coverage: [
+1 -1
View File
@@ -312,7 +312,7 @@ async function runTestForAgent(ctx: RunContext): Promise<ValidationResult> {
} else {
const defaultModels: Record<string, string> = {
claude: "anthropic/claude-sonnet-4-6",
opencode: "anthropic/claude-sonnet-4-6",
opencode: "moonshotai/kimi-k2",
};
const model = defaultModels[ctx.agent];
if (model) {