Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| fb22cb3ae3 | |||
| c43ed65c3b |
@@ -38,6 +38,8 @@ jobs:
|
||||
skill-invoke-opencode,
|
||||
smoke,
|
||||
token-exfil,
|
||||
vertex-claude,
|
||||
vertex-opencode,
|
||||
]
|
||||
exclude:
|
||||
- agent: claude
|
||||
@@ -61,6 +63,10 @@ jobs:
|
||||
AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }}
|
||||
AWS_REGION: us-east-1
|
||||
BEDROCK_MODEL_ID: us.anthropic.claude-opus-4-6-v1
|
||||
VERTEX_SERVICE_ACCOUNT_JSON: ${{ secrets.VERTEX_SERVICE_ACCOUNT_JSON }}
|
||||
GOOGLE_CLOUD_PROJECT: pullfrog
|
||||
VERTEX_LOCATION: global
|
||||
VERTEX_MODEL_ID: gemini-2.5-flash
|
||||
PULLFROG_MODEL: ${{ vars.PULLFROG_MODEL }}
|
||||
# CI smoke-testing shortcut only — production stores this in Pullfrog's
|
||||
# per-org secret store (Postgres), set via `pullfrog auth codex`. GH
|
||||
|
||||
+61
-29
@@ -16,7 +16,12 @@ import { mkdirSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { performance } from "node:perf_hooks";
|
||||
import { pullfrogMcpName } from "../external.ts";
|
||||
import { BEDROCK_MODEL_ID_ENV, isBedrockAnthropicId } from "../models.ts";
|
||||
import {
|
||||
BEDROCK_MODEL_ID_ENV,
|
||||
isBedrockAnthropicId,
|
||||
isVertexAnthropicId,
|
||||
VERTEX_MODEL_ID_ENV,
|
||||
} from "../models.ts";
|
||||
|
||||
import {
|
||||
getIdleMs,
|
||||
@@ -39,6 +44,7 @@ import {
|
||||
import { ThinkingTimer } from "../utils/timer.ts";
|
||||
import type { TodoTracker } from "../utils/todoTracking.ts";
|
||||
import { getDevDependencyVersion } from "../utils/version.ts";
|
||||
import { applyClaudeVertexEnv } from "../utils/vertex.ts";
|
||||
import {
|
||||
buildLearningsReflectionPrompt,
|
||||
runPostRunRetryLoop,
|
||||
@@ -825,36 +831,50 @@ const MANAGED_SETTINGS_PATH = `${MANAGED_SETTINGS_DIR}/managed-settings.json`;
|
||||
// tail, sed) and survives bypassPermissions mode. See wiki/codex-auth.md.
|
||||
const CODEX_AUTH_DENY_PATH = "~/.local/share/opencode/auth.json";
|
||||
|
||||
const managedSettings = {
|
||||
allowManagedPermissionRulesOnly: true,
|
||||
allowManagedHooksOnly: true,
|
||||
permissions: {
|
||||
deny: [
|
||||
"Read(//proc/**)",
|
||||
"Read(//sys/**)",
|
||||
"Grep(//proc/**)",
|
||||
"Grep(//sys/**)",
|
||||
"Edit(//proc/**)",
|
||||
"Edit(//sys/**)",
|
||||
"Glob(//proc/**)",
|
||||
"Glob(//sys/**)",
|
||||
`Read(${CODEX_AUTH_DENY_PATH})`,
|
||||
`Grep(${CODEX_AUTH_DENY_PATH})`,
|
||||
`Edit(${CODEX_AUTH_DENY_PATH})`,
|
||||
`Glob(${CODEX_AUTH_DENY_PATH})`,
|
||||
],
|
||||
},
|
||||
sandbox: {
|
||||
filesystem: {
|
||||
denyRead: ["/proc", "/sys", CODEX_AUTH_DENY_PATH],
|
||||
function buildManagedSettings(ctx: AgentRunContext) {
|
||||
const secretDenyPaths = ctx.secretDenyPaths ?? [];
|
||||
const toolDeny = secretDenyPaths.flatMap((path) => [
|
||||
`Read(${path}/**)`,
|
||||
`Read(/${path}/**)`,
|
||||
`Grep(${path}/**)`,
|
||||
`Grep(/${path}/**)`,
|
||||
`Edit(${path}/**)`,
|
||||
`Edit(/${path}/**)`,
|
||||
`Glob(${path}/**)`,
|
||||
`Glob(/${path}/**)`,
|
||||
]);
|
||||
return {
|
||||
allowManagedPermissionRulesOnly: true,
|
||||
allowManagedHooksOnly: true,
|
||||
permissions: {
|
||||
deny: [
|
||||
"Read(//proc/**)",
|
||||
"Read(//sys/**)",
|
||||
"Grep(//proc/**)",
|
||||
"Grep(//sys/**)",
|
||||
"Edit(//proc/**)",
|
||||
"Edit(//sys/**)",
|
||||
"Glob(//proc/**)",
|
||||
"Glob(//sys/**)",
|
||||
`Read(${CODEX_AUTH_DENY_PATH})`,
|
||||
`Grep(${CODEX_AUTH_DENY_PATH})`,
|
||||
`Edit(${CODEX_AUTH_DENY_PATH})`,
|
||||
`Glob(${CODEX_AUTH_DENY_PATH})`,
|
||||
...toolDeny,
|
||||
],
|
||||
},
|
||||
},
|
||||
};
|
||||
sandbox: {
|
||||
filesystem: {
|
||||
denyRead: ["/proc", "/sys", CODEX_AUTH_DENY_PATH, ...secretDenyPaths],
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function installManagedSettings(): void {
|
||||
function installManagedSettings(ctx: AgentRunContext): void {
|
||||
if (process.env.CI !== "true") return;
|
||||
|
||||
const content = JSON.stringify(managedSettings, null, 2);
|
||||
const content = JSON.stringify(buildManagedSettings(ctx), null, 2);
|
||||
try {
|
||||
execFileSync("sudo", ["mkdir", "-p", MANAGED_SETTINGS_DIR]);
|
||||
execFileSync("sudo", ["tee", MANAGED_SETTINGS_PATH], {
|
||||
@@ -887,11 +907,19 @@ export const claude = agent({
|
||||
bedrockModelId !== undefined &&
|
||||
bedrockModelId === specifier &&
|
||||
isBedrockAnthropicId(specifier);
|
||||
const vertexModelId = process.env[VERTEX_MODEL_ID_ENV]?.trim();
|
||||
const isVertexRoute =
|
||||
specifier !== undefined &&
|
||||
vertexModelId !== undefined &&
|
||||
vertexModelId === specifier &&
|
||||
isVertexAnthropicId(specifier);
|
||||
const model = !specifier
|
||||
? undefined
|
||||
: isBedrockRoute
|
||||
? specifier
|
||||
: stripProviderPrefix(specifier);
|
||||
: isVertexRoute
|
||||
? undefined
|
||||
: stripProviderPrefix(specifier);
|
||||
|
||||
const homeEnv = {
|
||||
HOME: ctx.tmpdir,
|
||||
@@ -913,7 +941,7 @@ export const claude = agent({
|
||||
const mcpConfigPath = writeMcpConfig(ctx);
|
||||
const effort = resolveEffort(model);
|
||||
|
||||
installManagedSettings();
|
||||
installManagedSettings(ctx);
|
||||
|
||||
// base args shared between initial run and continue runs
|
||||
const baseArgs = [
|
||||
@@ -967,6 +995,10 @@ export const claude = agent({
|
||||
if (isBedrockRoute) {
|
||||
env.CLAUDE_CODE_USE_BEDROCK = "1";
|
||||
}
|
||||
if (isVertexRoute) {
|
||||
applyClaudeVertexEnv(env);
|
||||
env.ANTHROPIC_MODEL = specifier;
|
||||
}
|
||||
|
||||
// claude-code's `Vw()` resolver prefers ANTHROPIC_API_KEY over the OAuth
|
||||
// token when both are set, so we strip the API key to fall through to the
|
||||
|
||||
+9
-1
@@ -40,6 +40,7 @@ import {
|
||||
import { ThinkingTimer } from "../utils/timer.ts";
|
||||
import type { TodoTracker } from "../utils/todoTracking.ts";
|
||||
import { getDevDependencyVersion } from "../utils/version.ts";
|
||||
import { resolveVertexOpenCodeModel } from "../utils/vertex.ts";
|
||||
import {
|
||||
PULLFROG_BUS_EVENT_TYPE,
|
||||
PULLFROG_OPENCODE_PLUGIN_FILENAME,
|
||||
@@ -1130,7 +1131,14 @@ export const opencode = agent({
|
||||
const bedrockModelId = process.env[BEDROCK_MODEL_ID_ENV]?.trim();
|
||||
const isBedrockRoute =
|
||||
rawModel !== undefined && bedrockModelId !== undefined && bedrockModelId === rawModel;
|
||||
const model = isBedrockRoute ? `amazon-bedrock/${rawModel}` : rawModel;
|
||||
let model = rawModel;
|
||||
if (isBedrockRoute) {
|
||||
model = `amazon-bedrock/${rawModel}`;
|
||||
}
|
||||
const vertexModel = resolveVertexOpenCodeModel(rawModel);
|
||||
if (vertexModel) {
|
||||
model = vertexModel;
|
||||
}
|
||||
|
||||
const homeEnv = {
|
||||
HOME: ctx.tmpdir,
|
||||
|
||||
@@ -57,6 +57,7 @@ import {
|
||||
} from "../utils/subprocess.ts";
|
||||
import type { TodoTracker } from "../utils/todoTracking.ts";
|
||||
import { getDevDependencyVersion } from "../utils/version.ts";
|
||||
import { resolveVertexOpenCodeModel } from "../utils/vertex.ts";
|
||||
import {
|
||||
PULLFROG_BUS_EVENT_TYPE,
|
||||
PULLFROG_OPENCODE_PLUGIN_FILENAME,
|
||||
@@ -906,7 +907,8 @@ export const opencode = agent({
|
||||
const bedrockModelId = process.env[BEDROCK_MODEL_ID_ENV]?.trim();
|
||||
const isBedrockRoute =
|
||||
rawModel !== undefined && bedrockModelId !== undefined && bedrockModelId === rawModel;
|
||||
const model = isBedrockRoute ? `amazon-bedrock/${rawModel}` : rawModel;
|
||||
const vertexModel = resolveVertexOpenCodeModel(rawModel);
|
||||
const model = vertexModel ?? (isBedrockRoute ? `amazon-bedrock/${rawModel}` : rawModel);
|
||||
|
||||
const homeEnv = {
|
||||
HOME: ctx.tmpdir,
|
||||
|
||||
@@ -131,6 +131,8 @@ export interface AgentRunContext {
|
||||
resolvedModel?: string | undefined;
|
||||
mcpServerUrl: string;
|
||||
tmpdir: string;
|
||||
/** harness-owned secret paths that agent filesystem tools must never read. */
|
||||
secretDenyPaths?: string[] | undefined;
|
||||
instructions: ResolvedInstructions;
|
||||
todoTracker?: TodoTracker | undefined;
|
||||
/**
|
||||
|
||||
@@ -49,6 +49,11 @@ import { resolveTimeoutMs, TIMEOUT_DISABLED } from "./utils/time.ts";
|
||||
import { Timer } from "./utils/timer.ts";
|
||||
import { createTodoTracker } from "./utils/todoTracking.ts";
|
||||
import { getJobToken, resolveTokens } from "./utils/token.ts";
|
||||
import {
|
||||
cleanupVertexCredentials,
|
||||
materializeVertexCredentials,
|
||||
type VertexCredentials,
|
||||
} from "./utils/vertex.ts";
|
||||
import { resolveRun } from "./utils/workflow.ts";
|
||||
|
||||
export { Inputs } from "./utils/payload.ts";
|
||||
@@ -177,6 +182,7 @@ export async function main(): Promise<MainResult> {
|
||||
let toolContext: ToolContext | undefined;
|
||||
let progressCallbackDisabled = false;
|
||||
let todoTracker: ReturnType<typeof createTodoTracker> | undefined;
|
||||
let vertexCredentials: VertexCredentials | undefined;
|
||||
|
||||
try {
|
||||
if (payload.cwd && process.cwd() !== payload.cwd) {
|
||||
@@ -233,6 +239,8 @@ export async function main(): Promise<MainResult> {
|
||||
toolState.modelFallback = { from: fallback.from };
|
||||
}
|
||||
|
||||
vertexCredentials = materializeVertexCredentials({ model: resolvedModel });
|
||||
|
||||
const agent = resolveAgent({ model: resolvedModel });
|
||||
|
||||
// surface the effective model in comment/review footers. payload.model is
|
||||
@@ -475,6 +483,7 @@ export async function main(): Promise<MainResult> {
|
||||
resolvedModel,
|
||||
mcpServerUrl: mcpHttpServer.url,
|
||||
tmpdir,
|
||||
secretDenyPaths: vertexCredentials ? [vertexCredentials.secretDir] : [],
|
||||
instructions,
|
||||
todoTracker,
|
||||
stopScript: runContext.repoSettings.stopScript,
|
||||
@@ -619,5 +628,6 @@ export async function main(): Promise<MainResult> {
|
||||
await patchWorkflowRunFields(toolContext, patch);
|
||||
}
|
||||
}
|
||||
cleanupVertexCredentials(vertexCredentials);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ import {
|
||||
getModelEnvVars,
|
||||
getModelProvider,
|
||||
isBedrockAnthropicId,
|
||||
isVertexAnthropicId,
|
||||
modelAliases,
|
||||
parseModel,
|
||||
providers,
|
||||
@@ -258,6 +259,20 @@ describe("isBedrockAnthropicId", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("isVertexAnthropicId", () => {
|
||||
it("matches Claude Vertex IDs by anchored prefix", () => {
|
||||
expect(isVertexAnthropicId("claude-opus-4-1@20250805")).toBe(true);
|
||||
});
|
||||
|
||||
it("rejects Gemini IDs", () => {
|
||||
expect(isVertexAnthropicId("gemini-2.5-pro")).toBe(false);
|
||||
});
|
||||
|
||||
it("ignores Anthropic substrings outside the prefix", () => {
|
||||
expect(isVertexAnthropicId("publishers/anthropic/models/claude-opus-4-1")).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("providers registry", () => {
|
||||
it("every provider has envVars", () => {
|
||||
for (const [key, config] of Object.entries(providers)) {
|
||||
|
||||
@@ -13,15 +13,17 @@
|
||||
*
|
||||
* `"bedrock"` means the actual model ID comes from `BEDROCK_MODEL_ID`
|
||||
* (an AWS-canonical Bedrock model ID like `us.anthropic.claude-opus-4-7`
|
||||
* or `amazon.nova-pro-v1:0`). enterprise Bedrock customers self-select for
|
||||
* version control — silent alias bumps would break compliance review,
|
||||
* model-access enrollment, and provisioned-throughput contracts. so the
|
||||
* single `bedrock/byok` entry is a routing slug, not a model alias: the
|
||||
* harness reads `BEDROCK_MODEL_ID` and routes to claude-code (when the ID
|
||||
* contains "anthropic") or opencode (everything else, with an
|
||||
* `amazon-bedrock/` prefix).
|
||||
* or `amazon.nova-pro-v1:0`). `"vertex"` means the actual model ID comes
|
||||
* from `VERTEX_MODEL_ID` (a Vertex AI model ID like
|
||||
* `claude-opus-4-1@20250805` or `gemini-2.5-pro`). enterprise hosted-model
|
||||
* customers self-select for version control — silent alias bumps would break
|
||||
* compliance review, model-access enrollment, and provisioned-throughput
|
||||
* contracts. so the single `bedrock/byok` and `vertex/byok` entries are
|
||||
* routing slugs, not model aliases: the harness reads the backend-specific
|
||||
* env var and routes to claude-code for Anthropic IDs or opencode for
|
||||
* everything else.
|
||||
*/
|
||||
export type ModelRouting = "bedrock";
|
||||
export type ModelRouting = "bedrock" | "vertex";
|
||||
|
||||
export interface ModelAlias {
|
||||
/** stable alias stored in DB, e.g. "anthropic/claude-opus" */
|
||||
@@ -375,6 +377,25 @@ export const providers = {
|
||||
},
|
||||
},
|
||||
}),
|
||||
vertex: provider({
|
||||
displayName: "Google Vertex AI",
|
||||
envVars: [
|
||||
"VERTEX_SERVICE_ACCOUNT_JSON",
|
||||
"GOOGLE_CLOUD_PROJECT",
|
||||
"VERTEX_LOCATION",
|
||||
"VERTEX_MODEL_ID",
|
||||
],
|
||||
models: {
|
||||
// single routing entry — the actual Vertex AI model ID is read from
|
||||
// VERTEX_MODEL_ID at run time. see ModelRouting docs for why we don't
|
||||
// catalog individual Vertex models.
|
||||
byok: {
|
||||
displayName: "Google Vertex AI",
|
||||
resolve: "vertex",
|
||||
routing: "vertex",
|
||||
},
|
||||
},
|
||||
}),
|
||||
openrouter: provider({
|
||||
displayName: "OpenRouter",
|
||||
envVars: ["OPENROUTER_API_KEY"],
|
||||
@@ -605,6 +626,9 @@ export function resolveOpenRouterModel(slug: string): string | undefined {
|
||||
/** env var that supplies the Bedrock model ID for the `bedrock/byok` slug. */
|
||||
export const BEDROCK_MODEL_ID_ENV = "BEDROCK_MODEL_ID";
|
||||
|
||||
/** env var that supplies the Vertex AI model ID for the `vertex/byok` slug. */
|
||||
export const VERTEX_MODEL_ID_ENV = "VERTEX_MODEL_ID";
|
||||
|
||||
/**
|
||||
* the Bedrock model ID passed to claude-code or opencode is whatever the
|
||||
* user set in `BEDROCK_MODEL_ID` — Pullfrog never resolves or upgrades it.
|
||||
@@ -636,3 +660,13 @@ export function isBedrockAnthropicId(bedrockModelId: string): boolean {
|
||||
// foundation segment sits between `/` and `.` inside the resource name).
|
||||
return bedrockModelId.toLowerCase().split(/[./:]/).includes("anthropic");
|
||||
}
|
||||
|
||||
/**
|
||||
* Vertex Anthropic model IDs start with the Claude family name, e.g.
|
||||
* `claude-opus-4-1@20250805`. partner-model resource paths can contain the
|
||||
* substring "anthropic" elsewhere, so the Bedrock segment check does not
|
||||
* transfer — anchor on the model ID prefix instead.
|
||||
*/
|
||||
export function isVertexAnthropicId(vertexModelId: string): boolean {
|
||||
return /^claude-/i.test(vertexModelId.trim());
|
||||
}
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "pullfrog",
|
||||
"version": "0.1.10",
|
||||
"version": "0.1.11",
|
||||
"type": "module",
|
||||
"bin": {
|
||||
"pullfrog": "dist/cli.mjs",
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
import type { AgentResult, TestRunnerOptions, ValidationCheck } from "../utils.ts";
|
||||
import { defineFixture } from "../utils.ts";
|
||||
|
||||
const fixture = defineFixture(
|
||||
{
|
||||
prompt: `Call set_output with "VERTEX CLAUDE SMOKE PASSED".`,
|
||||
},
|
||||
{ localOnly: true }
|
||||
);
|
||||
|
||||
function validator(result: AgentResult): ValidationCheck[] {
|
||||
const output = result.structuredOutput;
|
||||
const setOutputCalled = output !== null;
|
||||
const correctValue = setOutputCalled && /VERTEX CLAUDE SMOKE PASSED/i.test(output);
|
||||
|
||||
return [
|
||||
{ name: "set_output", passed: setOutputCalled },
|
||||
{ name: "correct_value", passed: correctValue },
|
||||
];
|
||||
}
|
||||
|
||||
export const test: TestRunnerOptions = {
|
||||
name: "vertex-claude",
|
||||
agents: ["claude"],
|
||||
fixture,
|
||||
validator,
|
||||
env: {
|
||||
PULLFROG_DISABLE_SECURITY_INSTRUCTIONS: "1",
|
||||
PULLFROG_MODEL: "vertex/byok",
|
||||
VERTEX_MODEL_ID: "claude-haiku-4-5@20251001",
|
||||
VERTEX_LOCATION: "global",
|
||||
},
|
||||
coverage: [
|
||||
"action/models.ts",
|
||||
"action/main.ts",
|
||||
"action/agents/claude.ts",
|
||||
"action/utils/{agent,apiKeys,vertex}.ts",
|
||||
],
|
||||
};
|
||||
@@ -0,0 +1,39 @@
|
||||
import type { AgentResult, TestRunnerOptions, ValidationCheck } from "../utils.ts";
|
||||
import { defineFixture } from "../utils.ts";
|
||||
|
||||
const fixture = defineFixture(
|
||||
{
|
||||
prompt: `Call set_output with "VERTEX OPENCODE SMOKE PASSED".`,
|
||||
},
|
||||
{ localOnly: true }
|
||||
);
|
||||
|
||||
function validator(result: AgentResult): ValidationCheck[] {
|
||||
const output = result.structuredOutput;
|
||||
const setOutputCalled = output !== null;
|
||||
const correctValue = setOutputCalled && /VERTEX OPENCODE SMOKE PASSED/i.test(output);
|
||||
|
||||
return [
|
||||
{ name: "set_output", passed: setOutputCalled },
|
||||
{ name: "correct_value", passed: correctValue },
|
||||
];
|
||||
}
|
||||
|
||||
export const test: TestRunnerOptions = {
|
||||
name: "vertex-opencode",
|
||||
agents: ["opencode"],
|
||||
fixture,
|
||||
validator,
|
||||
env: {
|
||||
PULLFROG_DISABLE_SECURITY_INSTRUCTIONS: "1",
|
||||
PULLFROG_MODEL: "vertex/byok",
|
||||
VERTEX_MODEL_ID: "gemini-2.5-flash",
|
||||
VERTEX_LOCATION: "global",
|
||||
},
|
||||
coverage: [
|
||||
"action/models.ts",
|
||||
"action/main.ts",
|
||||
"action/agents/opencode.ts",
|
||||
"action/utils/{agent,apiKeys,vertex}.ts",
|
||||
],
|
||||
};
|
||||
@@ -1,5 +1,9 @@
|
||||
import { mkdtempSync, readFileSync, rmSync, statSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
||||
import { resolveAgent, resolveModel } from "./agent.ts";
|
||||
import { cleanupVertexCredentials, materializeVertexCredentials } from "./vertex.ts";
|
||||
|
||||
const savedEnv = { ...process.env };
|
||||
|
||||
@@ -12,6 +16,12 @@ const STRIPPED = [
|
||||
/^AWS_SESSION_TOKEN$/,
|
||||
/^AWS_REGION$/,
|
||||
/^BEDROCK_MODEL_ID$/,
|
||||
/^GOOGLE_APPLICATION_CREDENTIALS$/,
|
||||
/^GOOGLE_CLOUD_PROJECT$/,
|
||||
/^VERTEX_SERVICE_ACCOUNT_JSON$/,
|
||||
/^VERTEX_LOCATION$/,
|
||||
/^VERTEX_MODEL_ID$/,
|
||||
/^PULLFROG_SECRET_HOME$/,
|
||||
/^PULLFROG_MODEL$/,
|
||||
/^PULLFROG_AGENT$/,
|
||||
];
|
||||
@@ -83,6 +93,20 @@ describe("resolveAgent", () => {
|
||||
expect(resolveAgent({ model: "us.anthropic.claude-opus-4-7" }).name).toBe("opencode");
|
||||
});
|
||||
});
|
||||
|
||||
describe("vertex routing", () => {
|
||||
it("routes Anthropic Vertex IDs to claude", () => {
|
||||
process.env.VERTEX_SERVICE_ACCOUNT_JSON = "{}";
|
||||
process.env.VERTEX_MODEL_ID = "claude-opus-4-1@20250805";
|
||||
expect(resolveAgent({ model: "claude-opus-4-1@20250805" }).name).toBe("claude");
|
||||
});
|
||||
|
||||
it("routes Gemini Vertex IDs to opencode", () => {
|
||||
process.env.VERTEX_SERVICE_ACCOUNT_JSON = "{}";
|
||||
process.env.VERTEX_MODEL_ID = "gemini-2.5-pro";
|
||||
expect(resolveAgent({ model: "gemini-2.5-pro" }).name).toBe("opencode");
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("resolveModel", () => {
|
||||
@@ -128,4 +152,47 @@ describe("resolveModel", () => {
|
||||
process.env.PULLFROG_MODEL = "bedrock/byok";
|
||||
expect(() => resolveModel({ slug: "openai/gpt" })).toThrow("BEDROCK_MODEL_ID");
|
||||
});
|
||||
|
||||
it("resolves vertex/byok to VERTEX_MODEL_ID", () => {
|
||||
process.env.VERTEX_MODEL_ID = "claude-opus-4-1@20250805";
|
||||
expect(resolveModel({ slug: "vertex/byok" })).toBe("claude-opus-4-1@20250805");
|
||||
});
|
||||
|
||||
it("throws when vertex/byok is selected without VERTEX_MODEL_ID", () => {
|
||||
expect(() => resolveModel({ slug: "vertex/byok" })).toThrow("VERTEX_MODEL_ID");
|
||||
});
|
||||
|
||||
it("PULLFROG_MODEL=vertex/byok defers to VERTEX_MODEL_ID, not the sentinel", () => {
|
||||
process.env.PULLFROG_MODEL = "vertex/byok";
|
||||
process.env.VERTEX_MODEL_ID = "gemini-2.5-pro";
|
||||
expect(resolveModel({ slug: "openai/gpt" })).toBe("gemini-2.5-pro");
|
||||
});
|
||||
});
|
||||
|
||||
describe("materializeVertexCredentials", () => {
|
||||
it("writes service-account JSON outside tmpdir and defaults project from project_id", () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "vertex-creds-test-"));
|
||||
process.env.VERTEX_MODEL_ID = "claude-opus-4-1@20250805";
|
||||
process.env.PULLFROG_SECRET_HOME = dir;
|
||||
process.env.VERTEX_SERVICE_ACCOUNT_JSON = JSON.stringify({
|
||||
project_id: "test-project",
|
||||
client_email: "pullfrog@test-project.iam.gserviceaccount.com",
|
||||
});
|
||||
|
||||
try {
|
||||
const credentials = materializeVertexCredentials({ model: "claude-opus-4-1@20250805" });
|
||||
|
||||
if (!credentials) throw new Error("expected vertex credentials");
|
||||
expect(credentials.credentialsPath).toContain(join(dir, ".pullfrog", "secrets"));
|
||||
expect(process.env.GOOGLE_APPLICATION_CREDENTIALS).toBe(credentials.credentialsPath);
|
||||
expect(process.env.GOOGLE_CLOUD_PROJECT).toBe("test-project");
|
||||
expect(readFileSync(credentials.credentialsPath, "utf8")).toBe(
|
||||
process.env.VERTEX_SERVICE_ACCOUNT_JSON
|
||||
);
|
||||
expect(statSync(credentials.credentialsPath).mode & 0o777).toBe(0o600);
|
||||
cleanupVertexCredentials(credentials);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
+28
-3
@@ -4,10 +4,13 @@ import {
|
||||
BEDROCK_MODEL_ID_ENV,
|
||||
getModelProvider,
|
||||
isBedrockAnthropicId,
|
||||
isVertexAnthropicId,
|
||||
resolveCliModel,
|
||||
resolveDisplayAlias,
|
||||
VERTEX_MODEL_ID_ENV,
|
||||
} from "../models.ts";
|
||||
import { log } from "./cli.ts";
|
||||
import { VERTEX_SERVICE_ACCOUNT_JSON_ENV } from "./vertex.ts";
|
||||
|
||||
function hasEnvVar(name: string): boolean {
|
||||
const val = process.env[name];
|
||||
@@ -25,6 +28,10 @@ function hasBedrockAuth(): boolean {
|
||||
);
|
||||
}
|
||||
|
||||
function hasVertexAuth(): boolean {
|
||||
return hasEnvVar(VERTEX_SERVICE_ACCOUNT_JSON_ENV);
|
||||
}
|
||||
|
||||
/**
|
||||
* resolve a single slug to its CLI-ready model string. routing aliases
|
||||
* (e.g. `bedrock/byok`) defer to their backing env var instead of the
|
||||
@@ -40,12 +47,23 @@ function resolveSlug(slug: string): string | undefined {
|
||||
if (!bedrockId) {
|
||||
throw new Error(
|
||||
`${BEDROCK_MODEL_ID_ENV} env var is required when the model is set to "${slug}". ` +
|
||||
`set it to an AWS Bedrock model ID (e.g. "us.anthropic.claude-opus-4-7", "amazon.nova-pro-v1:0"). ` +
|
||||
`set it to an AWS Bedrock model ID from the Bedrock console. ` +
|
||||
`see https://docs.pullfrog.com/bedrock for setup.`
|
||||
);
|
||||
}
|
||||
return bedrockId;
|
||||
}
|
||||
if (alias?.routing === "vertex") {
|
||||
const vertexId = process.env[VERTEX_MODEL_ID_ENV]?.trim();
|
||||
if (!vertexId) {
|
||||
throw new Error(
|
||||
`${VERTEX_MODEL_ID_ENV} env var is required when the model is set to "${slug}". ` +
|
||||
`set it to a Google Vertex AI model ID from Model Garden. ` +
|
||||
`see https://docs.pullfrog.com/vertex for setup.`
|
||||
);
|
||||
}
|
||||
return vertexId;
|
||||
}
|
||||
return resolveCliModel(slug);
|
||||
}
|
||||
|
||||
@@ -97,7 +115,14 @@ export function resolveAgent(ctx: { model?: string | undefined }): Agent {
|
||||
return isBedrockAnthropicId(ctx.model) ? agents.claude : agents.opencode;
|
||||
}
|
||||
|
||||
// 3. if model is Anthropic and Claude Code credentials are available, use Claude Code
|
||||
// 3. Vertex routing: same shape as Bedrock, but Anthropic Vertex IDs are
|
||||
// anchored `claude-*` IDs and non-Anthropic models use opencode's
|
||||
// `google-vertex` provider.
|
||||
if (ctx.model && hasVertexAuth() && process.env[VERTEX_MODEL_ID_ENV]?.trim() === ctx.model) {
|
||||
return isVertexAnthropicId(ctx.model) ? agents.claude : agents.opencode;
|
||||
}
|
||||
|
||||
// 4. if model is Anthropic and Claude Code credentials are available, use Claude Code
|
||||
if (ctx.model) {
|
||||
try {
|
||||
const provider = getModelProvider(ctx.model);
|
||||
@@ -109,6 +134,6 @@ export function resolveAgent(ctx: { model?: string | undefined }): Agent {
|
||||
}
|
||||
}
|
||||
|
||||
// 4. default: OpenCode (universal, supports all providers)
|
||||
// 5. default: OpenCode (universal, supports all providers)
|
||||
return agents.opencode;
|
||||
}
|
||||
|
||||
@@ -22,6 +22,11 @@ const STRIPPED_PREFIXES_OR_NAMES = [
|
||||
/^AWS_SESSION_TOKEN$/,
|
||||
/^AWS_REGION$/,
|
||||
/^BEDROCK_MODEL_ID$/,
|
||||
/^GOOGLE_APPLICATION_CREDENTIALS$/,
|
||||
/^GOOGLE_CLOUD_PROJECT$/,
|
||||
/^VERTEX_SERVICE_ACCOUNT_JSON$/,
|
||||
/^VERTEX_LOCATION$/,
|
||||
/^VERTEX_MODEL_ID$/,
|
||||
];
|
||||
|
||||
beforeEach(() => {
|
||||
@@ -156,6 +161,76 @@ describe("validateAgentApiKey", () => {
|
||||
).toThrow("AWS_BEARER_TOKEN_BEDROCK");
|
||||
});
|
||||
});
|
||||
|
||||
describe("vertex routing slug", () => {
|
||||
it("passes with service-account JSON + project + location + model id", () => {
|
||||
process.env.VERTEX_SERVICE_ACCOUNT_JSON = "{}";
|
||||
process.env.GOOGLE_CLOUD_PROJECT = "test-project";
|
||||
process.env.VERTEX_LOCATION = "us-east5";
|
||||
process.env.VERTEX_MODEL_ID = "claude-opus-4-1@20250805";
|
||||
expect(() => validateAgentApiKey({ ...base, model: "vertex/byok" })).not.toThrow();
|
||||
});
|
||||
|
||||
it("passes when project is derivable from service-account JSON", () => {
|
||||
process.env.VERTEX_SERVICE_ACCOUNT_JSON = JSON.stringify({ project_id: "test-project" });
|
||||
process.env.VERTEX_LOCATION = "us-east5";
|
||||
process.env.VERTEX_MODEL_ID = "gemini-2.5-pro";
|
||||
expect(() => validateAgentApiKey({ ...base, model: "vertex/byok" })).not.toThrow();
|
||||
});
|
||||
|
||||
it("throws when VERTEX_MODEL_ID is missing", () => {
|
||||
process.env.VERTEX_SERVICE_ACCOUNT_JSON = "{}";
|
||||
process.env.GOOGLE_CLOUD_PROJECT = "test-project";
|
||||
process.env.VERTEX_LOCATION = "us-east5";
|
||||
expect(() => validateAgentApiKey({ ...base, model: "vertex/byok" })).toThrow(
|
||||
"VERTEX_MODEL_ID"
|
||||
);
|
||||
});
|
||||
|
||||
it("throws when VERTEX_LOCATION is missing", () => {
|
||||
process.env.VERTEX_SERVICE_ACCOUNT_JSON = "{}";
|
||||
process.env.GOOGLE_CLOUD_PROJECT = "test-project";
|
||||
process.env.VERTEX_MODEL_ID = "claude-opus-4-1@20250805";
|
||||
expect(() => validateAgentApiKey({ ...base, model: "vertex/byok" })).toThrow(
|
||||
"VERTEX_LOCATION"
|
||||
);
|
||||
});
|
||||
|
||||
it("throws when GOOGLE_CLOUD_PROJECT is missing and not derivable", () => {
|
||||
process.env.VERTEX_SERVICE_ACCOUNT_JSON = "{}";
|
||||
process.env.VERTEX_LOCATION = "us-east5";
|
||||
process.env.VERTEX_MODEL_ID = "claude-opus-4-1@20250805";
|
||||
expect(() => validateAgentApiKey({ ...base, model: "vertex/byok" })).toThrow(
|
||||
"GOOGLE_CLOUD_PROJECT"
|
||||
);
|
||||
});
|
||||
|
||||
it("throws when no auth path is set", () => {
|
||||
process.env.GOOGLE_CLOUD_PROJECT = "test-project";
|
||||
process.env.VERTEX_LOCATION = "us-east5";
|
||||
process.env.VERTEX_MODEL_ID = "claude-opus-4-1@20250805";
|
||||
expect(() => validateAgentApiKey({ ...base, model: "vertex/byok" })).toThrow(
|
||||
"VERTEX_SERVICE_ACCOUNT_JSON"
|
||||
);
|
||||
});
|
||||
|
||||
it("accepts a raw Vertex model ID (post-resolveModel) without throwing", () => {
|
||||
process.env.VERTEX_SERVICE_ACCOUNT_JSON = "{}";
|
||||
process.env.GOOGLE_CLOUD_PROJECT = "test-project";
|
||||
process.env.VERTEX_LOCATION = "us-east5";
|
||||
process.env.VERTEX_MODEL_ID = "gemini-2.5-pro";
|
||||
expect(() => validateAgentApiKey({ ...base, model: "gemini-2.5-pro" })).not.toThrow();
|
||||
});
|
||||
|
||||
it("throws on raw Vertex model ID when auth is missing", () => {
|
||||
process.env.GOOGLE_CLOUD_PROJECT = "test-project";
|
||||
process.env.VERTEX_LOCATION = "us-east5";
|
||||
process.env.VERTEX_MODEL_ID = "gemini-2.5-pro";
|
||||
expect(() => validateAgentApiKey({ ...base, model: "gemini-2.5-pro" })).toThrow(
|
||||
"VERTEX_SERVICE_ACCOUNT_JSON"
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("isApiKeyAuthError", () => {
|
||||
|
||||
+52
-5
@@ -3,8 +3,15 @@ import {
|
||||
getModelEnvVars,
|
||||
providers,
|
||||
resolveDisplayAlias,
|
||||
VERTEX_MODEL_ID_ENV,
|
||||
} from "../models.ts";
|
||||
import { getApiUrl } from "./apiUrl.ts";
|
||||
import {
|
||||
GOOGLE_CLOUD_PROJECT_ENV,
|
||||
readProjectIdFromVertexServiceAccountJson,
|
||||
VERTEX_LOCATION_ENV,
|
||||
VERTEX_SERVICE_ACCOUNT_JSON_ENV,
|
||||
} from "./vertex.ts";
|
||||
|
||||
const knownApiKeys: Set<string> = new Set(
|
||||
Object.values(providers).flatMap((p) => [...p.envVars, ...(p.managedCredentials ?? [])])
|
||||
@@ -45,6 +52,21 @@ add the missing secret(s) to your GitHub repository at ${githubSecretsUrl}, then
|
||||
for full setup instructions, see https://docs.pullfrog.com/bedrock`;
|
||||
}
|
||||
|
||||
function buildVertexSetupError(params: { owner: string; name: string; missing: string[] }): string {
|
||||
const githubSecretsUrl = `https://github.com/${params.owner}/${params.name}/settings/secrets/actions`;
|
||||
|
||||
return `Google Vertex AI model selected but required configuration is missing: ${params.missing.join(", ")}.
|
||||
|
||||
add the missing secret(s) to your GitHub repository at ${githubSecretsUrl}, then reference them in your workflow's \`env:\` block:
|
||||
|
||||
${VERTEX_SERVICE_ACCOUNT_JSON_ENV}: \${{ secrets.${VERTEX_SERVICE_ACCOUNT_JSON_ENV} }}
|
||||
${GOOGLE_CLOUD_PROJECT_ENV}: my-project
|
||||
${VERTEX_LOCATION_ENV}: global
|
||||
${VERTEX_MODEL_ID_ENV}: <vertex-model-id>
|
||||
|
||||
for full setup instructions, see https://docs.pullfrog.com/vertex`;
|
||||
}
|
||||
|
||||
function hasEnvVar(name: string): boolean {
|
||||
const value = process.env[name];
|
||||
return typeof value === "string" && value.length > 0;
|
||||
@@ -73,6 +95,23 @@ function validateBedrockSetup(params: { owner: string; name: string }): void {
|
||||
}
|
||||
}
|
||||
|
||||
function validateVertexSetup(params: { owner: string; name: string }): void {
|
||||
const hasAuth = hasEnvVar(VERTEX_SERVICE_ACCOUNT_JSON_ENV);
|
||||
const hasProject =
|
||||
hasEnvVar(GOOGLE_CLOUD_PROJECT_ENV) ||
|
||||
readProjectIdFromVertexServiceAccountJson() !== undefined;
|
||||
|
||||
const missing: string[] = [];
|
||||
if (!hasAuth) missing.push(VERTEX_SERVICE_ACCOUNT_JSON_ENV);
|
||||
if (!hasProject) missing.push(GOOGLE_CLOUD_PROJECT_ENV);
|
||||
if (!hasEnvVar(VERTEX_LOCATION_ENV)) missing.push(VERTEX_LOCATION_ENV);
|
||||
if (!hasEnvVar(VERTEX_MODEL_ID_ENV)) missing.push(VERTEX_MODEL_ID_ENV);
|
||||
|
||||
if (missing.length > 0) {
|
||||
throw new Error(buildVertexSetupError({ owner: params.owner, name: params.name, missing }));
|
||||
}
|
||||
}
|
||||
|
||||
export function validateAgentApiKey(params: {
|
||||
agent: { name: string };
|
||||
model: string | undefined;
|
||||
@@ -90,15 +129,23 @@ export function validateAgentApiKey(params: {
|
||||
validateBedrockSetup({ owner: params.owner, name: params.name });
|
||||
return;
|
||||
}
|
||||
if (alias?.routing === "vertex") {
|
||||
validateVertexSetup({ owner: params.owner, name: params.name });
|
||||
return;
|
||||
}
|
||||
|
||||
// upstream `resolveModel` translates `bedrock/byok` into the raw Bedrock
|
||||
// model ID (e.g. `us.anthropic.claude-opus-4-6-v1`), which has no `/`
|
||||
// upstream `resolveModel` translates routing slugs into raw backend
|
||||
// model IDs (e.g. `us.anthropic.claude-opus-4-6-v1`), which have no `/`
|
||||
// and so isn't parseable as `provider/model`. these IDs only reach this
|
||||
// function via routing aliases, so re-run the bedrock setup check rather
|
||||
// function via routing aliases, so re-run the matching setup check rather
|
||||
// than falling through to `getModelEnvVars` (which would throw inside
|
||||
// parseModel). resolveModel itself already enforced BEDROCK_MODEL_ID,
|
||||
// but auth + region are still validated here.
|
||||
// parseModel). resolveModel itself already enforced the model-id env var,
|
||||
// but auth + location/region are still validated here.
|
||||
if (!params.model.includes("/")) {
|
||||
if (process.env[VERTEX_MODEL_ID_ENV]?.trim() === params.model) {
|
||||
validateVertexSetup({ owner: params.owner, name: params.name });
|
||||
return;
|
||||
}
|
||||
validateBedrockSetup({ owner: params.owner, name: params.name });
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { mkdirSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { homedir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { VERTEX_MODEL_ID_ENV } from "../models.ts";
|
||||
|
||||
export const VERTEX_SERVICE_ACCOUNT_JSON_ENV = "VERTEX_SERVICE_ACCOUNT_JSON";
|
||||
export const GOOGLE_APPLICATION_CREDENTIALS_ENV = "GOOGLE_APPLICATION_CREDENTIALS";
|
||||
export const GOOGLE_CLOUD_PROJECT_ENV = "GOOGLE_CLOUD_PROJECT";
|
||||
export const VERTEX_LOCATION_ENV = "VERTEX_LOCATION";
|
||||
|
||||
export type VertexCredentials = {
|
||||
credentialsPath: string;
|
||||
secretDir: string;
|
||||
};
|
||||
|
||||
function hasEnvVar(name: string): boolean {
|
||||
const value = process.env[name];
|
||||
return typeof value === "string" && value.length > 0;
|
||||
}
|
||||
|
||||
export function isVertexRoute(model: string | undefined): boolean {
|
||||
const vertexId = process.env[VERTEX_MODEL_ID_ENV]?.trim();
|
||||
return model !== undefined && vertexId !== undefined && vertexId === model;
|
||||
}
|
||||
|
||||
export function readProjectIdFromVertexServiceAccountJson(): string | undefined {
|
||||
const blob = process.env[VERTEX_SERVICE_ACCOUNT_JSON_ENV];
|
||||
if (!blob) return undefined;
|
||||
|
||||
try {
|
||||
const parsed: unknown = JSON.parse(blob);
|
||||
if (!parsed || typeof parsed !== "object" || !("project_id" in parsed)) {
|
||||
return undefined;
|
||||
}
|
||||
const projectId = parsed.project_id;
|
||||
return typeof projectId === "string" && projectId.length > 0 ? projectId : undefined;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function createSecretDir(): string {
|
||||
const base = process.env.PULLFROG_SECRET_HOME || process.env.HOME || homedir();
|
||||
const secretDir = join(base, ".pullfrog", "secrets", randomUUID());
|
||||
mkdirSync(secretDir, { recursive: true, mode: 0o700 });
|
||||
return secretDir;
|
||||
}
|
||||
|
||||
export function materializeVertexCredentials(params: {
|
||||
model: string | undefined;
|
||||
}): VertexCredentials | undefined {
|
||||
if (!isVertexRoute(params.model)) return undefined;
|
||||
|
||||
const blob = process.env[VERTEX_SERVICE_ACCOUNT_JSON_ENV];
|
||||
if (!blob) return undefined;
|
||||
|
||||
const secretDir = createSecretDir();
|
||||
const credentialsPath = join(secretDir, "vertex-sa.json");
|
||||
writeFileSync(credentialsPath, blob, { mode: 0o600 });
|
||||
process.env[GOOGLE_APPLICATION_CREDENTIALS_ENV] = credentialsPath;
|
||||
|
||||
const projectId = readProjectIdFromVertexServiceAccountJson();
|
||||
if (projectId && !hasEnvVar(GOOGLE_CLOUD_PROJECT_ENV)) {
|
||||
process.env[GOOGLE_CLOUD_PROJECT_ENV] = projectId;
|
||||
}
|
||||
|
||||
return { credentialsPath, secretDir };
|
||||
}
|
||||
|
||||
export function cleanupVertexCredentials(credentials: VertexCredentials | undefined): void {
|
||||
if (!credentials) return;
|
||||
rmSync(credentials.secretDir, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
export function applyClaudeVertexEnv(env: Record<string, string | undefined>): void {
|
||||
env.CLAUDE_CODE_USE_VERTEX = "1";
|
||||
env.ANTHROPIC_VERTEX_PROJECT_ID ??= env[GOOGLE_CLOUD_PROJECT_ENV];
|
||||
env.CLOUD_ML_REGION ??= env[VERTEX_LOCATION_ENV];
|
||||
}
|
||||
|
||||
export function resolveVertexOpenCodeModel(model: string | undefined): string | undefined {
|
||||
return isVertexRoute(model) && model ? `google-vertex/${model}` : undefined;
|
||||
}
|
||||
Reference in New Issue
Block a user