test: switch opus defaults to sonnet/kimi for cheaper coverage
- opencode test default: claude-sonnet-4-6 → moonshotai/kimi-k2. claude-code default stays on sonnet (the agent-under-test for that path is Claude Code); the opencode path doesn't need an Anthropic model. - byok-no-keys-fallback fixture: anthropic/claude-opus → moonshotai/kimi-k2. test verifies fallback to opencode/big-pickle, so the configured model is never invoked; picking a non-opus alias avoids burning credits if the fallback ever regresses. - BEDROCK_MODEL_ID in both test workflows: us.anthropic.claude-opus-4-6-v1 → us.anthropic.claude-sonnet-4-6. env var is required by the ci.test invariant that every provider's env vars are wired into the workflow.
This commit is contained in:
committed by
pullfrog[bot]
parent
fd2c67ab50
commit
e52206b8ca
@@ -62,7 +62,7 @@ jobs:
|
|||||||
OPENCODE_API_KEY: ${{ secrets.OPENCODE_API_KEY }}
|
OPENCODE_API_KEY: ${{ secrets.OPENCODE_API_KEY }}
|
||||||
AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }}
|
AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }}
|
||||||
AWS_REGION: us-east-1
|
AWS_REGION: us-east-1
|
||||||
BEDROCK_MODEL_ID: us.anthropic.claude-opus-4-6-v1
|
BEDROCK_MODEL_ID: us.anthropic.claude-sonnet-4-6
|
||||||
VERTEX_SERVICE_ACCOUNT_JSON: ${{ secrets.VERTEX_SERVICE_ACCOUNT_JSON }}
|
VERTEX_SERVICE_ACCOUNT_JSON: ${{ secrets.VERTEX_SERVICE_ACCOUNT_JSON }}
|
||||||
GOOGLE_CLOUD_PROJECT: pullfrog
|
GOOGLE_CLOUD_PROJECT: pullfrog
|
||||||
VERTEX_LOCATION: global
|
VERTEX_LOCATION: global
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ import { defineFixture, getAgentOutput } from "../utils.ts";
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* BYOK-no-keys fallback test — proves that an account configured for a
|
* BYOK-no-keys fallback test — proves that an account configured for a
|
||||||
* BYOK model (here: `anthropic/claude-opus`) but with no provider API
|
* BYOK model (here: `moonshotai/kimi-k2`) but with no provider API
|
||||||
* keys present in the runner env still gets a successful run by falling
|
* keys present in the runner env still gets a successful run by falling
|
||||||
* back to a free OpenCode model.
|
* back to a free OpenCode model.
|
||||||
*
|
*
|
||||||
@@ -59,10 +59,10 @@ export const test: TestRunnerOptions = {
|
|||||||
BEDROCK_MODEL_ID: "",
|
BEDROCK_MODEL_ID: "",
|
||||||
// configure a model that requires a BYOK key — the fallback only
|
// configure a model that requires a BYOK key — the fallback only
|
||||||
// engages when there's a configured model whose provider key is
|
// engages when there's a configured model whose provider key is
|
||||||
// absent, so we have to pin one. anthropic/claude-opus is the
|
// absent, so we have to pin one. any BYOK alias works; we pick
|
||||||
// most common first-run choice (it's the catalog "preferred" for
|
// a cheap non-Anthropic model so the test doesn't burn opus
|
||||||
// the anthropic provider).
|
// credits if the fallback ever regresses.
|
||||||
PULLFROG_MODEL: "anthropic/claude-opus",
|
PULLFROG_MODEL: "moonshotai/kimi-k2",
|
||||||
},
|
},
|
||||||
tags: ["agnostic"],
|
tags: ["agnostic"],
|
||||||
coverage: [
|
coverage: [
|
||||||
|
|||||||
+1
-1
@@ -312,7 +312,7 @@ async function runTestForAgent(ctx: RunContext): Promise<ValidationResult> {
|
|||||||
} else {
|
} else {
|
||||||
const defaultModels: Record<string, string> = {
|
const defaultModels: Record<string, string> = {
|
||||||
claude: "anthropic/claude-sonnet-4-6",
|
claude: "anthropic/claude-sonnet-4-6",
|
||||||
opencode: "anthropic/claude-sonnet-4-6",
|
opencode: "moonshotai/kimi-k2",
|
||||||
};
|
};
|
||||||
const model = defaultModels[ctx.agent];
|
const model = defaultModels[ctx.agent];
|
||||||
if (model) {
|
if (model) {
|
||||||
|
|||||||
Reference in New Issue
Block a user