Compare commits
96 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| fb22cb3ae3 | |||
| c43ed65c3b | |||
| 09344a9ec9 | |||
| 1b201352b5 | |||
| 6c166ac1cc | |||
| a0576a702a | |||
| 4d1fd5ea1a | |||
| 2f1f136da8 | |||
| cb0dbcd371 | |||
| 7e90e5cae6 | |||
| f49d4206aa | |||
| 69c7d4b8cd | |||
| f3d18401ac | |||
| 8dff91ac49 | |||
| 0d7955d87d | |||
| 6e94f513df | |||
| dd26d35137 | |||
| 3514bbc39f | |||
| 8ac954a27f | |||
| 88f170e19a | |||
| 0abaaa1e37 | |||
| e20f32fb09 | |||
| c0988e35b0 | |||
| efc1b67e7b | |||
| 0a64659ee7 | |||
| a78b1542da | |||
| ddbc610569 | |||
| a0dce200d0 | |||
| 7907fac64e | |||
| 76879b27ec | |||
| 8e1acfba99 | |||
| fa7ddcee4a | |||
| 3add2cbc49 | |||
| 5abb3072c7 | |||
| 74b7329f64 | |||
| ba7f5a0b89 | |||
| b9383bbcfd | |||
| 8d6460da1c | |||
| 1f4c3031be | |||
| 4ad649ebb9 | |||
| 2960d51493 | |||
| b6df2860c3 | |||
| d495f0b984 | |||
| 206c11fe7c | |||
| 7414c1e9ca | |||
| 8f9208bd3f | |||
| 1a9d3c1f82 | |||
| 951745ec89 | |||
| 56793d4a81 | |||
| d857e06731 | |||
| b9f0938405 | |||
| b8ac42e875 | |||
| 868576a474 | |||
| b2b1e588e7 | |||
| 5caeb75344 | |||
| 5518890b18 | |||
| d04c1ca3da | |||
| ae976e7159 | |||
| 5aabd1e4a9 | |||
| 60cc8772a6 | |||
| 4260984257 | |||
| d5f881e9fc | |||
| 1dc53043a6 | |||
| 076e5a17b5 | |||
| d5d8a0d7ac | |||
| 159389fad2 | |||
| 43bb14bf87 | |||
| d8f825034f | |||
| f0805b78f5 | |||
| e20b4d5515 | |||
| 8c6cd2bda2 | |||
| e4d0fc7e3d | |||
| a4a5010441 | |||
| cf94773bf0 | |||
| 8e36f76cfa | |||
| dee13b160f | |||
| ef394277c1 | |||
| ee479474ce | |||
| 96910f0f50 | |||
| 4cc6d95a91 | |||
| 10590993f4 | |||
| 10aeaf8c11 | |||
| 85d25a6fe6 | |||
| 653fae47a5 | |||
| 363e4cbed8 | |||
| c8888cecde | |||
| c0de70431e | |||
| b0274e3265 | |||
| 8f36eca62a | |||
| 3c9799adda | |||
| 5f3e46c42d | |||
| 3d393c36a3 | |||
| d6de1c369a | |||
| 2e6c01670e | |||
| 17b610e1a1 | |||
| ca913c76ea |
@@ -0,0 +1,31 @@
|
||||
# the Dockerfile only `COPY`s docker-entrypoint.sh, so most of this is
|
||||
# defense-in-depth — modern docker BuildKit (default since docker 23)
|
||||
# already prunes unreferenced files from the build context. but:
|
||||
# - documents intent for future maintainers who add `COPY . .`
|
||||
# - resurfaces the bytes-saved win if someone disables BuildKit
|
||||
# (DOCKER_BUILDKIT=0) or adopts a builder that doesn't prune
|
||||
# - keeps `docker build` snappy even on cold builders that DO send
|
||||
# everything
|
||||
|
||||
# pnpm-managed workspace deps — large and never needed at build time
|
||||
node_modules/
|
||||
|
||||
# secrets — must never enter an image, even by accident
|
||||
.env
|
||||
.env.*
|
||||
!.env.example
|
||||
|
||||
# build outputs
|
||||
dist/
|
||||
build/
|
||||
*.log
|
||||
|
||||
# editor / VCS noise
|
||||
.DS_Store
|
||||
.idea/
|
||||
.vscode/
|
||||
|
||||
# tests + fixtures we don't need at build time
|
||||
coverage/
|
||||
test/
|
||||
.scripts/
|
||||
@@ -11,6 +11,10 @@ permissions:
|
||||
|
||||
jobs:
|
||||
test-token:
|
||||
# only run in the upstream publish target. forks inherit this file but
|
||||
# haven't installed the pullfrog github app — running it there 404s our
|
||||
# token endpoint and pollutes our error logs (see #693).
|
||||
if: github.repository == 'pullfrog/pullfrog'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Get installation token
|
||||
|
||||
@@ -30,6 +30,7 @@ jobs:
|
||||
agent: [claude, opencode]
|
||||
test:
|
||||
[
|
||||
codex-auth,
|
||||
mcpmerge,
|
||||
nobash,
|
||||
restricted,
|
||||
@@ -37,10 +38,14 @@ jobs:
|
||||
skill-invoke-opencode,
|
||||
smoke,
|
||||
token-exfil,
|
||||
vertex-claude,
|
||||
vertex-opencode,
|
||||
]
|
||||
exclude:
|
||||
- agent: claude
|
||||
test: skill-invoke-opencode
|
||||
- agent: claude
|
||||
test: codex-auth
|
||||
- agent: opencode
|
||||
test: skill-invoke-claude
|
||||
env:
|
||||
@@ -55,7 +60,21 @@ jobs:
|
||||
OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
|
||||
MOONSHOT_API_KEY: ${{ secrets.MOONSHOT_API_KEY }}
|
||||
OPENCODE_API_KEY: ${{ secrets.OPENCODE_API_KEY }}
|
||||
AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }}
|
||||
AWS_REGION: us-east-1
|
||||
BEDROCK_MODEL_ID: us.anthropic.claude-opus-4-6-v1
|
||||
VERTEX_SERVICE_ACCOUNT_JSON: ${{ secrets.VERTEX_SERVICE_ACCOUNT_JSON }}
|
||||
GOOGLE_CLOUD_PROJECT: pullfrog
|
||||
VERTEX_LOCATION: global
|
||||
VERTEX_MODEL_ID: gemini-2.5-flash
|
||||
PULLFROG_MODEL: ${{ vars.PULLFROG_MODEL }}
|
||||
# CI smoke-testing shortcut only — production stores this in Pullfrog's
|
||||
# per-org secret store (Postgres), set via `pullfrog auth codex`. GH
|
||||
# Actions secrets are immutable at runtime so the post-hook can't write
|
||||
# back the rotated refresh token; CI accepts the staleness and we
|
||||
# manually re-provision when smoke tests start failing. Do not copy this
|
||||
# pattern into user-facing workflows. See wiki/codex-auth.md.
|
||||
CODEX_AUTH_JSON: ${{ secrets.CODEX_AUTH_JSON }}
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: pnpm/action-setup@v4
|
||||
@@ -78,6 +97,7 @@ jobs:
|
||||
matrix:
|
||||
test:
|
||||
[
|
||||
byok-no-keys-fallback,
|
||||
git-permissions,
|
||||
githooks,
|
||||
pkg-json-scripts,
|
||||
|
||||
@@ -10,8 +10,10 @@ permissions:
|
||||
|
||||
jobs:
|
||||
trigger:
|
||||
# skip if pushed by our bot (breaks the loop)
|
||||
if: github.actor != 'pullfrog[bot]'
|
||||
# only run in the upstream publish target (forks inherit this file but
|
||||
# can't dispatch into pullfrog/app), and skip if pushed by our bot (breaks
|
||||
# the loop).
|
||||
if: github.repository == 'pullfrog/pullfrog' && github.actor != 'pullfrog[bot]'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
+78
@@ -0,0 +1,78 @@
|
||||
# pullfrog GHA-like test container.
|
||||
#
|
||||
# baked once at image build time, used by `pnpm docker`. all runtime cost
|
||||
# (apt-get, useradd, sudoers wiring) is paid here so each `docker` invocation
|
||||
# is a single `docker run` with no in-container setup.
|
||||
#
|
||||
# rebuild is content-hash gated by docker.ts (Dockerfile + docker-entrypoint.sh).
|
||||
# bump anything in this file or the entrypoint and the next `pnpm docker` rebuilds.
|
||||
|
||||
FROM ubuntu:24.04
|
||||
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
# core toolset matching what GHA `ubuntu-24.04` runners ship: gh, jq, git,
|
||||
# python3, ssh client, plus the compression + build-essential surface that
|
||||
# `pnpm install` / `node-gyp` / agent shell calls regularly need. keeps
|
||||
# test-time invocations of these tools honest (no "works on the runner,
|
||||
# breaks in the local container").
|
||||
RUN apt-get update -qq \
|
||||
&& apt-get install -qq -y --no-install-recommends \
|
||||
build-essential \
|
||||
ca-certificates \
|
||||
curl \
|
||||
file \
|
||||
git \
|
||||
gnupg \
|
||||
jq \
|
||||
openssh-client \
|
||||
python3 \
|
||||
sudo \
|
||||
unzip \
|
||||
wget \
|
||||
xz-utils \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# node 24 from nodesource + corepack (provides pnpm without a global install).
|
||||
RUN curl -fsSL https://deb.nodesource.com/setup_24.x | bash - \
|
||||
&& apt-get install -y nodejs \
|
||||
&& rm -rf /var/lib/apt/lists/* \
|
||||
&& corepack enable
|
||||
|
||||
# gh cli (matches GHA pre-installed tooling).
|
||||
RUN mkdir -p /etc/apt/keyrings \
|
||||
&& curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg \
|
||||
| gpg --dearmor -o /etc/apt/keyrings/githubcli-archive-keyring.gpg \
|
||||
&& chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg \
|
||||
&& echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" \
|
||||
> /etc/apt/sources.list.d/github-cli.list \
|
||||
&& apt-get update -qq \
|
||||
&& apt-get install -qq -y gh \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# ubuntu:24.04 ships a default `ubuntu` user at uid 1000 — remove it so we
|
||||
# can place `testuser` at 1000 (the typical macOS dev uid). the entrypoint
|
||||
# remaps to the host uid/gid at runtime if they differ.
|
||||
RUN userdel -r ubuntu 2>/dev/null || true \
|
||||
&& groupadd -g 1000 testuser \
|
||||
&& useradd -u 1000 -g 1000 -m -s /bin/bash testuser \
|
||||
&& echo "testuser ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/testuser \
|
||||
&& chmod 0440 /etc/sudoers.d/testuser
|
||||
|
||||
# layout matching the bind mount + named volume targets in docker.ts.
|
||||
RUN mkdir -p /app/action /app/action/node_modules /tmp/home/.config /tmp/home/.cache \
|
||||
&& chown -R testuser:testuser /app /tmp/home
|
||||
|
||||
# CI=true is critical: `shell.ts` PID-namespace sandbox keys off it. baking
|
||||
# it ensures security tests can't pass vacuously because someone forgot the
|
||||
# flag.
|
||||
ENV HOME=/tmp/home \
|
||||
TMPDIR=/tmp \
|
||||
CI=true \
|
||||
COREPACK_ENABLE_DOWNLOAD_PROMPT=0
|
||||
|
||||
COPY docker-entrypoint.sh /entrypoint.sh
|
||||
RUN chmod +x /entrypoint.sh
|
||||
|
||||
WORKDIR /app/action
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
@@ -36,6 +36,11 @@ outputs:
|
||||
runs:
|
||||
using: "node24"
|
||||
main: "entry.ts"
|
||||
# Always-run post step persists best-effort state that must survive
|
||||
# cancellation, timeouts, and unhandled errors in the main step. Today's
|
||||
# only consumer: Codex auth.json refresh write-back. See wiki/codex-auth.md.
|
||||
post: "entryPost.ts"
|
||||
post-if: "always()"
|
||||
|
||||
branding:
|
||||
icon: "code"
|
||||
|
||||
+400
-80
@@ -16,19 +16,42 @@ import { mkdirSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { performance } from "node:perf_hooks";
|
||||
import { pullfrogMcpName } from "../external.ts";
|
||||
import {
|
||||
BEDROCK_MODEL_ID_ENV,
|
||||
isBedrockAnthropicId,
|
||||
isVertexAnthropicId,
|
||||
VERTEX_MODEL_ID_ENV,
|
||||
} from "../models.ts";
|
||||
|
||||
import { getIdleMs, markActivity } from "../utils/activity.ts";
|
||||
import { log } from "../utils/cli.ts";
|
||||
import {
|
||||
getIdleMs,
|
||||
isActivitySuspended,
|
||||
markActivity,
|
||||
resumeActivity,
|
||||
suspendActivity,
|
||||
} from "../utils/activity.ts";
|
||||
import { formatJsonValue, log } from "../utils/cli.ts";
|
||||
import { installFromNpmTarball } from "../utils/install.ts";
|
||||
import { detectProviderError } from "../utils/providerErrors.ts";
|
||||
import { findProviderErrorMatch } from "../utils/providerErrors.ts";
|
||||
import { addSkill, installBundledSkills } from "../utils/skills.ts";
|
||||
import { SPAWN_ACTIVITY_TIMEOUT_CODE, SpawnTimeoutError, spawn } from "../utils/subprocess.ts";
|
||||
import {
|
||||
DEFAULT_MAX_RETAINED_BYTES,
|
||||
SPAWN_ACTIVITY_TIMEOUT_CODE,
|
||||
SpawnTimeoutError,
|
||||
spawn,
|
||||
TailBuffer,
|
||||
} from "../utils/subprocess.ts";
|
||||
import { ThinkingTimer } from "../utils/timer.ts";
|
||||
import type { TodoTracker } from "../utils/todoTracking.ts";
|
||||
import { getDevDependencyVersion } from "../utils/version.ts";
|
||||
import { buildLearningsReflectionPrompt, runPostRunRetryLoop } from "./postRun.ts";
|
||||
import { applyClaudeVertexEnv } from "../utils/vertex.ts";
|
||||
import {
|
||||
buildLearningsReflectionPrompt,
|
||||
runPostRunRetryLoop,
|
||||
shouldRunReflection,
|
||||
} from "./postRun.ts";
|
||||
import { REVIEWER_AGENT_NAME, REVIEWER_SYSTEM_PROMPT } from "./reviewer.ts";
|
||||
import { deriveLabelFromTaskInput } from "./sessionLabeler.ts";
|
||||
import { formatWithLabel, ORCHESTRATOR_LABEL, SessionLabeler } from "./sessionLabeler.ts";
|
||||
import {
|
||||
type AgentResult,
|
||||
type AgentRunContext,
|
||||
@@ -66,17 +89,27 @@ function writeMcpConfig(ctx: AgentRunContext): string {
|
||||
|
||||
/**
|
||||
* Build the `--agents` JSON definition for the `reviewfrog` subagent.
|
||||
*
|
||||
* The Claude Code path always runs against an Anthropic model (see
|
||||
* resolveAgent), so we hardcode the cheaper-sibling downshift: lenses run
|
||||
* on Sonnet, the orchestrator stays on whatever model `--model` was passed.
|
||||
*
|
||||
* Per-call model override is also possible (Task tool's `model` arg accepts
|
||||
* 'sonnet' | 'opus' | 'haiku') and takes precedence over what's set here —
|
||||
* we don't pass it; the per-subagent `model` field is the right default.
|
||||
*
|
||||
* The non-mutative + non-recursive contract is enforced by the prose system
|
||||
* prompt baked into the agent — see action/agents/reviewer.ts for why we no
|
||||
* longer wire per-agent `disallowedTools` here.
|
||||
* prompt baked into the agent — see action/agents/reviewer.ts for why we
|
||||
* no longer wire per-agent `disallowedTools` here.
|
||||
*/
|
||||
function buildAgentsJson(): string {
|
||||
const agents = {
|
||||
[REVIEWER_AGENT_NAME]: {
|
||||
description:
|
||||
"Read-only review subagent for self-review and lens-based code review. " +
|
||||
"Read-only review subagent for lens-based code review (correctness, security, billing-subsystem, etc.). " +
|
||||
"Reads only — no writes, no state-changing shell or MCP calls, no nested subagent dispatch.",
|
||||
prompt: REVIEWER_SYSTEM_PROMPT,
|
||||
model: "claude-sonnet-4-6",
|
||||
},
|
||||
};
|
||||
return JSON.stringify(agents);
|
||||
@@ -90,10 +123,13 @@ function stripProviderPrefix(specifier: string): string {
|
||||
return slashIndex > 0 ? specifier.slice(slashIndex + 1) : specifier;
|
||||
}
|
||||
|
||||
// `max` effort is supported on Opus 4.6 / 4.7; other models fall back to `high`.
|
||||
// claude-code deny-lists older opus/sonnet generations from `max` at invocation time.
|
||||
function resolveEffort(model: string | undefined): "max" | "high" {
|
||||
if (model?.includes("opus")) return "max";
|
||||
// `high` is the model's tuned default ("equivalent to not setting the parameter"
|
||||
// per Anthropic docs). `max` is "absolute maximum capability with no constraints
|
||||
// on token spending" — meaningfully slower and burns more thinking budget per
|
||||
// turn. We default everyone to `high`; PRs that genuinely need full-send can
|
||||
// opt in via a future per-run override rather than paying the wall-time cost on
|
||||
// every Opus run.
|
||||
function resolveEffort(_model: string | undefined): "high" {
|
||||
return "high";
|
||||
}
|
||||
|
||||
@@ -111,13 +147,21 @@ interface ContentBlock {
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
// SDK schema (per claude-agent-sdk docs) puts `session_id` and
|
||||
// `parent_tool_use_id` at the top level of every Assistant/User/System/Result
|
||||
// message, not inside `message`. Subagent events carry a non-null
|
||||
// `parent_tool_use_id` pointing at the orchestrator's Task/Agent tool_use id.
|
||||
interface ClaudeSystemEvent {
|
||||
type: "system";
|
||||
session_id?: string;
|
||||
parent_tool_use_id?: string | null;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
interface ClaudeAssistantEvent {
|
||||
type: "assistant";
|
||||
session_id?: string;
|
||||
parent_tool_use_id?: string | null;
|
||||
message?: {
|
||||
role?: string;
|
||||
content?: ContentBlock[];
|
||||
@@ -135,6 +179,8 @@ interface ClaudeAssistantEvent {
|
||||
|
||||
interface ClaudeUserEvent {
|
||||
type: "user";
|
||||
session_id?: string;
|
||||
parent_tool_use_id?: string | null;
|
||||
message?: {
|
||||
role?: string;
|
||||
content?: ContentBlock[];
|
||||
@@ -146,6 +192,15 @@ interface ClaudeUserEvent {
|
||||
interface ClaudeResultEvent {
|
||||
type: "result";
|
||||
subtype?: string;
|
||||
// claude CLI sets `is_error: true` (alongside `subtype: "success"`) when
|
||||
// an upstream provider fails mid-stream. `api_error_status` carries the
|
||||
// provider HTTP status (e.g. 401 for invalid API key). per the official
|
||||
// SDK types, `api_error_status` is `number | null`, and the `error_*`
|
||||
// subtypes carry their actionable payload in `errors: string[]` instead
|
||||
// of `result`.
|
||||
is_error?: boolean;
|
||||
api_error_status?: number | null;
|
||||
errors?: string[];
|
||||
result?: string;
|
||||
session_id?: string;
|
||||
num_turns?: number;
|
||||
@@ -203,14 +258,78 @@ type RunParams = {
|
||||
|
||||
type ClaudeRunResult = AgentResult & { sessionId?: string | undefined };
|
||||
|
||||
async function runClaude(params: RunParams): Promise<ClaudeRunResult> {
|
||||
/**
|
||||
* Return the tail of `text` capped at `maxCodeUnits` UTF-16 code units,
|
||||
* dropping any partial first line. used in the exit-non-zero stdout fallback
|
||||
* so we never surface a truncated NDJSON event to operators —
|
||||
* `result.stdout.slice(-2048)` would otherwise cut mid-line and produce a
|
||||
* syntactically broken JSON fragment. code units rather than bytes because
|
||||
* `String.prototype.slice` operates on UTF-16 units; for multi-byte UTF-8
|
||||
* content the effective byte budget can be up to 4× the nominal limit.
|
||||
*/
|
||||
function tailLines(text: string, maxCodeUnits: number): string {
|
||||
if (text.length <= maxCodeUnits) return text;
|
||||
const tail = text.slice(-maxCodeUnits);
|
||||
const firstNewline = tail.indexOf("\n");
|
||||
// if no newline in window or it's at the very start, return as-is;
|
||||
// otherwise drop the partial first line.
|
||||
return firstNewline > 0 && firstNewline < tail.length - 1 ? tail.slice(firstNewline + 1) : tail;
|
||||
}
|
||||
|
||||
export async function runClaude(params: RunParams): Promise<ClaudeRunResult> {
|
||||
const startTime = performance.now();
|
||||
let eventCount = 0;
|
||||
const thinkingTimer = new ThinkingTimer();
|
||||
|
||||
// per-session labeler so parallel subagent log lines can be differentiated.
|
||||
// claude-agent-sdk runs subagents inside the orchestrator's session — they
|
||||
// share `session_id` — and stamps every subagent message with a non-null
|
||||
// `parent_tool_use_id` pointing at the Agent tool_use that spawned them.
|
||||
// we bind each Agent tool_use id to its dispatched label up front, then
|
||||
// labelFor short-circuits to the direct mapping when parent_tool_use_id is
|
||||
// set. orchestrator events (parent_tool_use_id === null) flow through the
|
||||
// sessionID path and bind to ORCHESTRATOR_LABEL on first sighting.
|
||||
const labeler = new SessionLabeler();
|
||||
function eventLabel(event: { session_id?: string; parent_tool_use_id?: string | null }): string {
|
||||
return labeler.labelFor(event.session_id ?? null, event.parent_tool_use_id ?? null);
|
||||
}
|
||||
function withLabel(label: string, message: string): string {
|
||||
return label === ORCHESTRATOR_LABEL ? message : formatWithLabel(label, message);
|
||||
}
|
||||
|
||||
// one ThinkingTimer per session — sharing a single timer across sessions
|
||||
// conflated cross-session interleaving as parent thinking time. each timer
|
||||
// formats its log lines through the session label so attribution is visible.
|
||||
const thinkingTimers = new Map<string, ThinkingTimer>();
|
||||
function timerFor(label: string): ThinkingTimer {
|
||||
let t = thinkingTimers.get(label);
|
||||
if (!t) {
|
||||
const formatLine = (line: string) =>
|
||||
label === ORCHESTRATOR_LABEL ? line : formatWithLabel(label, line);
|
||||
t = new ThinkingTimer(formatLine);
|
||||
thinkingTimers.set(label, t);
|
||||
}
|
||||
return t;
|
||||
}
|
||||
|
||||
let finalOutput = "";
|
||||
let sessionId: string | undefined;
|
||||
let resultErrorSubtype: string | null = null;
|
||||
// captures the structured error string from a result event with
|
||||
// `is_error: true` (e.g. mid-stream provider auth failures the CLI
|
||||
// surfaces as `subtype: "success"` synthetic-stop events, or the
|
||||
// `errors[]` array from `error_*` subtypes). preferred over raw
|
||||
// stdout/stderr in the exit-non-zero path so the GitHub Actions
|
||||
// `##[error]` line shows the actionable message instead of an 8KB+
|
||||
// NDJSON dump.
|
||||
let lastResultError: string | null = null;
|
||||
// set only for synthetic-stop `subtype: "success"` + `is_error: true`
|
||||
// events, where `accumulatedTokens` from prior `assistant` events is
|
||||
// stale and logging it would mislead operators into thinking billable
|
||||
// tokens were spent on a successful turn. deliberately NOT set for
|
||||
// `error_max_turns` / `error_during_execution` / `error_*` subtypes
|
||||
// because those runs genuinely consumed tokens and operators need
|
||||
// billing visibility for them.
|
||||
let syntheticStopFailure = false;
|
||||
let accumulatedTokens = { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 };
|
||||
// Claude CLI reports a single end-of-run `total_cost_usd` on the result
|
||||
// event. per-message events don't carry cost, so there's nothing to sum —
|
||||
@@ -234,43 +353,73 @@ async function runClaude(params: RunParams): Promise<ClaudeRunResult> {
|
||||
}
|
||||
|
||||
const handlers = {
|
||||
system: (_event: ClaudeSystemEvent) => {
|
||||
log.debug(`» ${params.label} system event`);
|
||||
system: (event: ClaudeSystemEvent) => {
|
||||
// claude-agent-sdk only emits system:init for the top-level query, so
|
||||
// this binds the orchestrator label and never appears in subagent flow.
|
||||
// we still route through eventLabel so a subagent system event (if the
|
||||
// SDK ever adds one) wouldn't go silently misattributed.
|
||||
const label = eventLabel(event);
|
||||
log.debug(withLabel(label, `» ${params.label} system event`));
|
||||
},
|
||||
assistant: (event: ClaudeAssistantEvent) => {
|
||||
const content = event.message?.content;
|
||||
if (!content) return;
|
||||
|
||||
const label = eventLabel(event);
|
||||
const boxTitle = label === ORCHESTRATOR_LABEL ? params.label : `${params.label} [${label}]`;
|
||||
|
||||
for (const block of content) {
|
||||
if (block.type === "text" && block.text?.trim()) {
|
||||
const message = block.text.trim();
|
||||
log.box(message, { title: params.label });
|
||||
finalOutput = message;
|
||||
log.box(message, { title: boxTitle });
|
||||
// only the orchestrator's text becomes the run's "output" — subagent
|
||||
// report-back text would otherwise clobber the parent's final answer.
|
||||
if (label === ORCHESTRATOR_LABEL) {
|
||||
finalOutput = message;
|
||||
}
|
||||
} else if (block.type === "tool_use") {
|
||||
const toolName = block.name || "unknown";
|
||||
// suspend the activity watchdog across the tool call. claude's
|
||||
// stdout pipe goes silent while it awaits the synchronous MCP
|
||||
// tools/call HTTP response; without this, long fetches/deepens
|
||||
// (issue #760) trip the spawn-level idle timer at 300s. paired
|
||||
// with resumeActivity() in tool_result below; bounded by the
|
||||
// MAX_TOOL_CALL_SUSPENSION_MS auto-resume in activity.ts.
|
||||
suspendActivity();
|
||||
if (params.onToolUse) {
|
||||
params.onToolUse({
|
||||
toolName,
|
||||
input: block.input,
|
||||
});
|
||||
}
|
||||
thinkingTimer.markToolCall();
|
||||
log.toolCall({ toolName, input: block.input || {} });
|
||||
timerFor(label).markToolCall();
|
||||
const inputFormatted = formatJsonValue(block.input || {});
|
||||
const toolCallLine =
|
||||
inputFormatted !== "{}" ? `» ${toolName}(${inputFormatted})` : `» ${toolName}()`;
|
||||
log.info(withLabel(label, toolCallLine));
|
||||
|
||||
// surface the subagent identity when the orchestrator dispatches a
|
||||
// Task — claude rolls subagent activity up into a single tool_result
|
||||
// (no per-event session_id in its stream), so this log line is the
|
||||
// only attribution available before the subagent's report-back.
|
||||
if (toolName === "Task" && block.input && typeof block.input === "object") {
|
||||
// when the orchestrator dispatches a subagent, bind the Agent
|
||||
// tool_use id to the dispatched label so future events carrying
|
||||
// `parent_tool_use_id === block.id` resolve directly to the right
|
||||
// lens. v2.1.63+ renamed the tool to "Agent"; older versions
|
||||
// emitted "Task". match both for forward-compat.
|
||||
if (
|
||||
(toolName === "Task" || toolName === "Agent") &&
|
||||
block.input &&
|
||||
typeof block.input === "object"
|
||||
) {
|
||||
const taskInput = block.input as {
|
||||
description?: string;
|
||||
subagent_type?: string;
|
||||
prompt?: string;
|
||||
};
|
||||
const label = deriveLabelFromTaskInput(taskInput);
|
||||
const dispatchedLabel = labeler.recordTaskDispatch(taskInput, block.id ?? null);
|
||||
log.info(
|
||||
`» dispatching subagent: ${label}` +
|
||||
(taskInput.subagent_type ? ` (subagent_type=${taskInput.subagent_type})` : "")
|
||||
withLabel(
|
||||
label,
|
||||
`» dispatching subagent: ${dispatchedLabel}` +
|
||||
(taskInput.subagent_type ? ` (subagent_type=${taskInput.subagent_type})` : "")
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
@@ -280,8 +429,14 @@ async function runClaude(params: RunParams): Promise<ClaudeRunResult> {
|
||||
params.todoTracker.cancel();
|
||||
}
|
||||
|
||||
// parse TodoWrite events for live progress tracking
|
||||
if (toolName === "TodoWrite" && params.todoTracker?.enabled) {
|
||||
// parse TodoWrite events for live progress tracking. only honor the
|
||||
// orchestrator's todos — subagents emit their own todo lists which
|
||||
// would otherwise clobber the visible progress comment.
|
||||
if (
|
||||
toolName === "TodoWrite" &&
|
||||
params.todoTracker?.enabled &&
|
||||
label === ORCHESTRATOR_LABEL
|
||||
) {
|
||||
params.todoTracker.update(block.input);
|
||||
}
|
||||
}
|
||||
@@ -302,10 +457,13 @@ async function runClaude(params: RunParams): Promise<ClaudeRunResult> {
|
||||
const content = event.message?.content;
|
||||
if (!content) return;
|
||||
|
||||
const label = eventLabel(event);
|
||||
|
||||
for (const block of content) {
|
||||
if (typeof block === "string") continue;
|
||||
if (block.type === "tool_result") {
|
||||
thinkingTimer.markToolResult();
|
||||
resumeActivity();
|
||||
timerFor(label).markToolResult();
|
||||
|
||||
const outputContent =
|
||||
typeof block.content === "string"
|
||||
@@ -323,9 +481,9 @@ async function runClaude(params: RunParams): Promise<ClaudeRunResult> {
|
||||
: String(block.content);
|
||||
|
||||
if (block.is_error) {
|
||||
log.info(`» tool error: ${outputContent}`);
|
||||
log.info(withLabel(label, `» tool error: ${outputContent}`));
|
||||
} else {
|
||||
log.debug(`» tool output: ${outputContent}`);
|
||||
log.debug(withLabel(label, `» tool output: ${outputContent}`));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -335,6 +493,27 @@ async function runClaude(params: RunParams): Promise<ClaudeRunResult> {
|
||||
const subtype = event.subtype || "unknown";
|
||||
const numTurns = event.num_turns || 0;
|
||||
|
||||
// claude CLI emits synthetic-stop result events with `subtype: "success"`
|
||||
// but `is_error: true` when an upstream provider fails mid-stream (e.g.
|
||||
// 401 from anthropic). short-circuit before the usage/token-table path
|
||||
// so we don't log a usage table for a failed attempt and so downstream
|
||||
// (`resultErrorSubtype` branch) surfaces the structured error. gated on
|
||||
// `subtype === "success"` because the `error_*` subtypes also set
|
||||
// `is_error: true` but carry their payload in `errors: string[]` and
|
||||
// are handled by the dedicated branches below.
|
||||
if (event.is_error === true && subtype === "success") {
|
||||
const apiStatus = event.api_error_status;
|
||||
lastResultError =
|
||||
event.result?.trim() ||
|
||||
`claude reported is_error=true with no result text (api_error_status=${apiStatus ?? "unknown"})`;
|
||||
resultErrorSubtype = subtype;
|
||||
syntheticStopFailure = true;
|
||||
log.info(
|
||||
`» ${params.label} result error: subtype=${subtype}, api_error_status=${apiStatus ?? "unknown"}, message=${lastResultError}`
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
if (subtype === "success") {
|
||||
// extract detailed usage from result event (most accurate source).
|
||||
// note: `input` here is non-cached input tokens only, matching the
|
||||
@@ -369,12 +548,15 @@ async function runClaude(params: RunParams): Promise<ClaudeRunResult> {
|
||||
}
|
||||
} else if (subtype === "error_max_turns") {
|
||||
resultErrorSubtype = subtype;
|
||||
lastResultError = event.errors?.join("\n").trim() || null;
|
||||
log.info(`» ${params.label} max turns reached: ${JSON.stringify(event)}`);
|
||||
} else if (subtype === "error_during_execution") {
|
||||
resultErrorSubtype = subtype;
|
||||
lastResultError = event.errors?.join("\n").trim() || null;
|
||||
log.info(`» ${params.label} execution error: ${JSON.stringify(event)}`);
|
||||
} else if (subtype.startsWith("error")) {
|
||||
resultErrorSubtype = subtype;
|
||||
lastResultError = event.errors?.join("\n").trim() || null;
|
||||
log.info(`» ${params.label} result: subtype=${subtype}, data=${JSON.stringify(event)}`);
|
||||
} else {
|
||||
log.info(`» ${params.label} result: subtype=${subtype}, data=${JSON.stringify(event)}`);
|
||||
@@ -392,10 +574,18 @@ async function runClaude(params: RunParams): Promise<ClaudeRunResult> {
|
||||
};
|
||||
|
||||
const recentStderr: string[] = [];
|
||||
// ring buffer of recent non-JSON stdout lines. Claude CLI prints
|
||||
// human-readable TTY chrome (status bubbles, quota notices, etc.)
|
||||
// alongside the NDJSON event stream. when the CLI exits non-zero without
|
||||
// emitting a structured error event, these lines are the only actionable
|
||||
// signal — preferring them over the NDJSON tail keeps progress comments
|
||||
// readable. issue #643.
|
||||
const recentNonJsonStdout: string[] = [];
|
||||
|
||||
let lastProviderError: string | null = null;
|
||||
|
||||
let output = "";
|
||||
// capped accumulator — see opencode.ts for rationale (issue #680).
|
||||
const output = new TailBuffer(DEFAULT_MAX_RETAINED_BYTES);
|
||||
let stdoutBuffer = "";
|
||||
|
||||
try {
|
||||
@@ -406,10 +596,22 @@ async function runClaude(params: RunParams): Promise<ClaudeRunResult> {
|
||||
env: params.env,
|
||||
activityTimeout: 300_000,
|
||||
onActivityTimeout: params.onActivityTimeout,
|
||||
isPausedExternally: isActivitySuspended,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
// run claude in its own process group so SIGKILL on activity timeout /
|
||||
// outer cancellation reaches any subprocesses it spawns (rg, file
|
||||
// watchers, mcp transports, etc). claude itself is a node bundle so
|
||||
// there's no shim-orphan issue like opencode-ai/bin/opencode, but
|
||||
// detached + killGroup is the right default for any agent runtime.
|
||||
killGroup: true,
|
||||
// claude already drains every chunk via onStdout (NDJSON parsing) and
|
||||
// onStderr (recentStderr ring buffer). retaining a second copy in the
|
||||
// spawn wrapper would grow unbounded for long sessions and previously
|
||||
// crashed the wrapper with RangeError. see issue #680.
|
||||
retain: "none",
|
||||
onStdout: async (chunk) => {
|
||||
const text = chunk.toString();
|
||||
output += text;
|
||||
output.append(text);
|
||||
markActivity();
|
||||
|
||||
stdoutBuffer += text;
|
||||
@@ -425,6 +627,8 @@ async function runClaude(params: RunParams): Promise<ClaudeRunResult> {
|
||||
event = JSON.parse(trimmed) as ClaudeEvent;
|
||||
} catch {
|
||||
log.debug(`» non-JSON stdout line: ${trimmed.substring(0, 200)}`);
|
||||
recentNonJsonStdout.push(trimmed);
|
||||
if (recentNonJsonStdout.length > MAX_STDERR_LINES) recentNonJsonStdout.shift();
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -460,10 +664,10 @@ async function runClaude(params: RunParams): Promise<ClaudeRunResult> {
|
||||
recentStderr.push(trimmed);
|
||||
if (recentStderr.length > MAX_STDERR_LINES) recentStderr.shift();
|
||||
|
||||
const providerError = detectProviderError(trimmed);
|
||||
if (providerError) {
|
||||
lastProviderError = providerError;
|
||||
log.info(`» provider error detected (${providerError}): ${trimmed.substring(0, 500)}`);
|
||||
const match = findProviderErrorMatch(trimmed);
|
||||
if (match) {
|
||||
lastProviderError = match.label;
|
||||
log.info(`» provider error detected (${match.label}): ${match.excerpt}`);
|
||||
} else {
|
||||
log.debug(trimmed);
|
||||
}
|
||||
@@ -490,8 +694,16 @@ async function runClaude(params: RunParams): Promise<ClaudeRunResult> {
|
||||
if (stderrContext) log.info(`» last stderr output:\n${stderrContext}`);
|
||||
}
|
||||
|
||||
// skip the fallback token table only for the synthetic-stop
|
||||
// `subtype: "success"` + `is_error: true` case: `accumulatedTokens` from
|
||||
// prior `assistant` events is stale there and logging it would mislead
|
||||
// operators into thinking billable tokens were spent on a successful turn.
|
||||
// `error_max_turns` / `error_during_execution` / `error_*` subtypes
|
||||
// represent runs that genuinely consumed tokens, so they still get the
|
||||
// table for billing visibility.
|
||||
if (
|
||||
!tokensLogged &&
|
||||
!syntheticStopFailure &&
|
||||
(accumulatedTokens.input > 0 ||
|
||||
accumulatedTokens.output > 0 ||
|
||||
accumulatedTokens.cacheRead > 0 ||
|
||||
@@ -505,18 +717,39 @@ async function runClaude(params: RunParams): Promise<ClaudeRunResult> {
|
||||
|
||||
if (result.exitCode !== 0) {
|
||||
const errorContext = lastProviderError ? ` (${lastProviderError})` : "";
|
||||
// prefer the structured `lastResultError` (parsed from a result event
|
||||
// with `is_error: true`) over raw stdout. raw stdout is the full NDJSON
|
||||
// event stream — dumping it into a GitHub Actions `##[error]` line both
|
||||
// hides the actionable provider message and pollutes the run log. cap
|
||||
// the stdout fallback to the last 2KB so it stays readable when neither
|
||||
// a structured error nor stderr is available.
|
||||
//
|
||||
// result.stdout / result.stderr are empty because we pass retain:"none"
|
||||
// to spawn (see issue #680); the agent layer keeps its own bounded
|
||||
// mirrors via `output` (TailBuffer) and `recentStderr` (ring buffer).
|
||||
const stdoutSnapshot = output.toString();
|
||||
const stderrSnapshot = recentStderr.join("\n");
|
||||
const truncatedStdout = stdoutSnapshot ? tailLines(stdoutSnapshot, 2048) : "";
|
||||
// prefer non-JSON stdout (human-readable TTY chrome the CLI prints,
|
||||
// including status bubbles and quota notices) over the raw NDJSON
|
||||
// tail. when the CLI exits 1 without emitting `is_error` (issue #643),
|
||||
// the NDJSON fallback would otherwise dump 2KB of `system/init` events
|
||||
// into the progress comment with no mention of the actual cause.
|
||||
const nonJsonStdoutSnapshot = recentNonJsonStdout.join("\n");
|
||||
const errorMessage =
|
||||
result.stderr ||
|
||||
result.stdout ||
|
||||
lastResultError ||
|
||||
stderrSnapshot ||
|
||||
nonJsonStdoutSnapshot ||
|
||||
truncatedStdout ||
|
||||
`unknown error - no output from Claude CLI${errorContext}`;
|
||||
log.error(
|
||||
`${params.label} exited with code ${result.exitCode}${errorContext}: ${errorMessage}`
|
||||
);
|
||||
log.debug(`stdout: ${result.stdout?.substring(0, 500)}`);
|
||||
log.debug(`stderr: ${result.stderr?.substring(0, 500)}`);
|
||||
log.debug(`stdout: ${stdoutSnapshot.substring(0, 500)}`);
|
||||
log.debug(`stderr: ${stderrSnapshot.substring(0, 500)}`);
|
||||
return {
|
||||
success: false,
|
||||
output: finalOutput || output,
|
||||
output: finalOutput || stdoutSnapshot,
|
||||
error: errorMessage,
|
||||
usage,
|
||||
sessionId,
|
||||
@@ -526,7 +759,7 @@ async function runClaude(params: RunParams): Promise<ClaudeRunResult> {
|
||||
if (eventCount === 0 && lastProviderError) {
|
||||
return {
|
||||
success: false,
|
||||
output: finalOutput || output,
|
||||
output: finalOutput || output.toString(),
|
||||
error: `provider error: ${lastProviderError}`,
|
||||
usage,
|
||||
sessionId,
|
||||
@@ -536,14 +769,14 @@ async function runClaude(params: RunParams): Promise<ClaudeRunResult> {
|
||||
if (resultErrorSubtype) {
|
||||
return {
|
||||
success: false,
|
||||
output: finalOutput || output,
|
||||
error: `result subtype: ${resultErrorSubtype}`,
|
||||
output: finalOutput || output.toString(),
|
||||
error: lastResultError || `result subtype: ${resultErrorSubtype}`,
|
||||
usage,
|
||||
sessionId,
|
||||
};
|
||||
}
|
||||
|
||||
return { success: true, output: finalOutput || output, usage, sessionId };
|
||||
return { success: true, output: finalOutput || output.toString(), usage, sessionId };
|
||||
} catch (error) {
|
||||
params.todoTracker?.cancel();
|
||||
const duration = performance.now() - startTime;
|
||||
@@ -569,7 +802,7 @@ async function runClaude(params: RunParams): Promise<ClaudeRunResult> {
|
||||
|
||||
return {
|
||||
success: false,
|
||||
output: finalOutput || output,
|
||||
output: finalOutput || output.toString(),
|
||||
error: `${errorMessage} [${diagnosis}]`,
|
||||
usage: buildUsage(),
|
||||
sessionId,
|
||||
@@ -590,32 +823,58 @@ const MANAGED_SETTINGS_PATH = `${MANAGED_SETTINGS_DIR}/managed-settings.json`;
|
||||
// allowManagedPermissionRulesOnly prevents malicious PRs from adding allow rules that override
|
||||
// our deny rules — safe in CI because --dangerously-skip-permissions makes allow/ask irrelevant.
|
||||
// allowManagedHooksOnly prevents malicious project hooks from bypassing deny rules.
|
||||
const managedSettings = {
|
||||
allowManagedPermissionRulesOnly: true,
|
||||
allowManagedHooksOnly: true,
|
||||
permissions: {
|
||||
deny: [
|
||||
"Read(//proc/**)",
|
||||
"Read(//sys/**)",
|
||||
"Grep(//proc/**)",
|
||||
"Grep(//sys/**)",
|
||||
"Edit(//proc/**)",
|
||||
"Edit(//sys/**)",
|
||||
"Glob(//proc/**)",
|
||||
"Glob(//sys/**)",
|
||||
],
|
||||
},
|
||||
sandbox: {
|
||||
filesystem: {
|
||||
denyRead: ["/proc", "/sys"],
|
||||
},
|
||||
},
|
||||
};
|
||||
// Codex auth.json (Pullfrog-stored ChatGPT subscription credential) lives at
|
||||
// `~/.local/share/opencode/auth.json` when the opencode harness materialized
|
||||
// it. Claude shouldn't be running OpenAI models — they route to opencode —
|
||||
// but defense-in-depth: deny the file regardless. Per Claude Code permissions
|
||||
// docs, Read(...) deny ALSO blocks file-reading Bash commands (cat, head,
|
||||
// tail, sed) and survives bypassPermissions mode. See wiki/codex-auth.md.
|
||||
const CODEX_AUTH_DENY_PATH = "~/.local/share/opencode/auth.json";
|
||||
|
||||
function installManagedSettings(): void {
|
||||
function buildManagedSettings(ctx: AgentRunContext) {
|
||||
const secretDenyPaths = ctx.secretDenyPaths ?? [];
|
||||
const toolDeny = secretDenyPaths.flatMap((path) => [
|
||||
`Read(${path}/**)`,
|
||||
`Read(/${path}/**)`,
|
||||
`Grep(${path}/**)`,
|
||||
`Grep(/${path}/**)`,
|
||||
`Edit(${path}/**)`,
|
||||
`Edit(/${path}/**)`,
|
||||
`Glob(${path}/**)`,
|
||||
`Glob(/${path}/**)`,
|
||||
]);
|
||||
return {
|
||||
allowManagedPermissionRulesOnly: true,
|
||||
allowManagedHooksOnly: true,
|
||||
permissions: {
|
||||
deny: [
|
||||
"Read(//proc/**)",
|
||||
"Read(//sys/**)",
|
||||
"Grep(//proc/**)",
|
||||
"Grep(//sys/**)",
|
||||
"Edit(//proc/**)",
|
||||
"Edit(//sys/**)",
|
||||
"Glob(//proc/**)",
|
||||
"Glob(//sys/**)",
|
||||
`Read(${CODEX_AUTH_DENY_PATH})`,
|
||||
`Grep(${CODEX_AUTH_DENY_PATH})`,
|
||||
`Edit(${CODEX_AUTH_DENY_PATH})`,
|
||||
`Glob(${CODEX_AUTH_DENY_PATH})`,
|
||||
...toolDeny,
|
||||
],
|
||||
},
|
||||
sandbox: {
|
||||
filesystem: {
|
||||
denyRead: ["/proc", "/sys", CODEX_AUTH_DENY_PATH, ...secretDenyPaths],
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function installManagedSettings(ctx: AgentRunContext): void {
|
||||
if (process.env.CI !== "true") return;
|
||||
|
||||
const content = JSON.stringify(managedSettings, null, 2);
|
||||
const content = JSON.stringify(buildManagedSettings(ctx), null, 2);
|
||||
try {
|
||||
execFileSync("sudo", ["mkdir", "-p", MANAGED_SETTINGS_DIR]);
|
||||
execFileSync("sudo", ["tee", MANAGED_SETTINGS_PATH], {
|
||||
@@ -637,7 +896,30 @@ export const claude = agent({
|
||||
const cliPath = await installClaudeCli();
|
||||
|
||||
const specifier = ctx.payload.proxyModel ?? ctx.resolvedModel;
|
||||
const model = specifier ? stripProviderPrefix(specifier) : undefined;
|
||||
// claude-code on Bedrock takes the bare AWS model ID — no provider prefix
|
||||
// to strip, since the ID is already in `provider.model` form (e.g.
|
||||
// `us.anthropic.claude-opus-4-7`). detect via the env-var sentinel: if
|
||||
// BEDROCK_MODEL_ID is set and matches the resolved specifier, this is a
|
||||
// bedrock route. see `wiki/model-resolution.md` for the routing pattern.
|
||||
const bedrockModelId = process.env[BEDROCK_MODEL_ID_ENV]?.trim();
|
||||
const isBedrockRoute =
|
||||
specifier !== undefined &&
|
||||
bedrockModelId !== undefined &&
|
||||
bedrockModelId === specifier &&
|
||||
isBedrockAnthropicId(specifier);
|
||||
const vertexModelId = process.env[VERTEX_MODEL_ID_ENV]?.trim();
|
||||
const isVertexRoute =
|
||||
specifier !== undefined &&
|
||||
vertexModelId !== undefined &&
|
||||
vertexModelId === specifier &&
|
||||
isVertexAnthropicId(specifier);
|
||||
const model = !specifier
|
||||
? undefined
|
||||
: isBedrockRoute
|
||||
? specifier
|
||||
: isVertexRoute
|
||||
? undefined
|
||||
: stripProviderPrefix(specifier);
|
||||
|
||||
const homeEnv = {
|
||||
HOME: ctx.tmpdir,
|
||||
@@ -659,7 +941,7 @@ export const claude = agent({
|
||||
const mcpConfigPath = writeMcpConfig(ctx);
|
||||
const effort = resolveEffort(model);
|
||||
|
||||
installManagedSettings();
|
||||
installManagedSettings(ctx);
|
||||
|
||||
// base args shared between initial run and continue runs
|
||||
const baseArgs = [
|
||||
@@ -684,12 +966,49 @@ export const claude = agent({
|
||||
|
||||
// agent process gets full env — needs LLM API keys, PATH, locale, etc.
|
||||
// security is enforced via managed-settings.json, --disallowedTools (Bash), and MCP tool filtering.
|
||||
//
|
||||
// bedrock route: claude-code reads `CLAUDE_CODE_USE_BEDROCK=1` to switch
|
||||
// its provider implementation from the direct Anthropic API to Bedrock.
|
||||
// AWS_BEARER_TOKEN_BEDROCK / AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY +
|
||||
// AWS_REGION are already in process.env from the workflow's `env:` block.
|
||||
// see https://docs.claude.com/en/docs/claude-code/amazon-bedrock.
|
||||
//
|
||||
// we only force CLAUDE_CODE_USE_BEDROCK=1 when this is a Pullfrog-routed
|
||||
// bedrock run; if the user has set the env var manually for some other
|
||||
// reason (e.g. always-Bedrock org policy), `...process.env` already
|
||||
// carries it through and we don't disturb it.
|
||||
const repoDir = process.cwd();
|
||||
|
||||
// PWD must match the spawn cwd (see opencode_v2.ts for the analogous fix).
|
||||
// claude-code 2.1.x reads `process.env.PWD` and registers it as a "session"
|
||||
// additional-working-directory when it differs from `process.cwd()` (per
|
||||
// the bundled cli.js — `let H=process.env.PWD; if(H && H !== Y7() && ...)
|
||||
// j.set(H, {path: H, source: "session"})`). Inheriting harness PWD via
|
||||
// `...process.env` ends up adding the wrong dir to the agent's allowed
|
||||
// working set under `pnpm runtest` / `pnpm play`, which silently confuses
|
||||
// path-relative tools.
|
||||
const env: Record<string, string | undefined> = {
|
||||
...process.env,
|
||||
...homeEnv,
|
||||
PWD: repoDir,
|
||||
};
|
||||
if (isBedrockRoute) {
|
||||
env.CLAUDE_CODE_USE_BEDROCK = "1";
|
||||
}
|
||||
if (isVertexRoute) {
|
||||
applyClaudeVertexEnv(env);
|
||||
env.ANTHROPIC_MODEL = specifier;
|
||||
}
|
||||
|
||||
const repoDir = process.cwd();
|
||||
// claude-code's `Vw()` resolver prefers ANTHROPIC_API_KEY over the OAuth
|
||||
// token when both are set, so we strip the API key to fall through to the
|
||||
// Max-subscription path. bedrock route uses AWS creds and is excluded.
|
||||
if (env.CLAUDE_CODE_OAUTH_TOKEN && !isBedrockRoute && env.ANTHROPIC_API_KEY) {
|
||||
log.debug(
|
||||
"» CLAUDE_CODE_OAUTH_TOKEN present — stripping ANTHROPIC_API_KEY from Claude Code env so the OAuth subscription is used"
|
||||
);
|
||||
delete env.ANTHROPIC_API_KEY;
|
||||
}
|
||||
|
||||
log.info(`» effort: ${effort}`);
|
||||
log.debug(`» starting Pullfrog (Claude Code): node ${baseArgs.join(" ")}`);
|
||||
@@ -716,12 +1035,13 @@ export const claude = agent({
|
||||
// the run). the reflection prompt fires once after gates go clean, as a
|
||||
// dedicated turn that nudges the agent to persist learnings.
|
||||
return runPostRunRetryLoop({
|
||||
ctx,
|
||||
initialResult: result,
|
||||
initialUsage: result.usage,
|
||||
stopScript: ctx.stopScript,
|
||||
summaryFilePath: ctx.summaryFilePath,
|
||||
summarySeed: ctx.summarySeed,
|
||||
reflectionPrompt: buildLearningsReflectionPrompt("claude"),
|
||||
reflectionPrompt:
|
||||
ctx.toolState.learningsFilePath && shouldRunReflection(ctx.toolState.selectedMode)
|
||||
? buildLearningsReflectionPrompt(ctx.toolState.learningsFilePath)
|
||||
: undefined,
|
||||
canResume: (r) => Boolean(r.sessionId),
|
||||
resume: async (c) => {
|
||||
const sessionId = c.previousResult.sessionId;
|
||||
|
||||
+4
-1
@@ -1,5 +1,8 @@
|
||||
import { claude } from "./claude.ts";
|
||||
import { opencode } from "./opencode.ts";
|
||||
// v2 harness — adapted to opencode-ai >=1.14.x SDK-v2 / Effect-ts CLI rewrite.
|
||||
// The legacy v1 module (`./opencode.ts`) is kept around for reference + fast
|
||||
// revert; the active runner is the v2 module below.
|
||||
import { opencode } from "./opencode_v2.ts";
|
||||
import type { Agent } from "./shared.ts";
|
||||
|
||||
export type { Agent, AgentUsage } from "./shared.ts";
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { modelAliases } from "../models.ts";
|
||||
import { geminiHighThinkingOverrides } from "./opencode.ts";
|
||||
|
||||
describe("geminiHighThinkingOverrides", () => {
|
||||
// Expected truth pulled the same way the helper does — both must derive from
|
||||
// the registry so the test exercises the wiring, not a hand-maintained list.
|
||||
const expectedApiIds = modelAliases
|
||||
.filter((a) => a.provider === "google")
|
||||
.map((a) => a.resolve.replace(/^google\//, ""));
|
||||
const overrides = geminiHighThinkingOverrides();
|
||||
|
||||
it("covers every direct-Google alias in the registry", () => {
|
||||
expect(Object.keys(overrides).sort()).toEqual([...expectedApiIds].sort());
|
||||
});
|
||||
|
||||
it("is non-empty (catches accidental whole-provider removal)", () => {
|
||||
expect(Object.keys(overrides).length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("strips the `google/` prefix from each resolve to get the bare API id", () => {
|
||||
for (const id of Object.keys(overrides)) {
|
||||
expect(id).not.toMatch(/^google\//);
|
||||
}
|
||||
});
|
||||
|
||||
it("pins every entry to thinkingLevel: high", () => {
|
||||
for (const [id, value] of Object.entries(overrides)) {
|
||||
expect(value, `entry for ${id}`).toEqual({
|
||||
options: { thinkingConfig: { thinkingLevel: "high" } },
|
||||
});
|
||||
}
|
||||
});
|
||||
});
|
||||
+460
-188
@@ -11,23 +11,54 @@
|
||||
* the agent process itself gets full env (needs LLM API keys, PATH, etc.).
|
||||
* security is enforced at the tool layer, not the process layer.
|
||||
*/
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { mkdirSync } from "node:fs";
|
||||
import { mkdirSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { performance } from "node:perf_hooks";
|
||||
import * as core from "@actions/core";
|
||||
import { pullfrogMcpName } from "../external.ts";
|
||||
import { modelAliases } from "../models.ts";
|
||||
import { getIdleMs, markActivity } from "../utils/activity.ts";
|
||||
import { BEDROCK_MODEL_ID_ENV } from "../models.ts";
|
||||
import type { ToolState } from "../toolState.ts";
|
||||
import {
|
||||
getIdleMs,
|
||||
isActivitySuspended,
|
||||
markActivity,
|
||||
resumeActivity,
|
||||
suspendActivity,
|
||||
} from "../utils/activity.ts";
|
||||
import { type AgentDiagnostic, formatAgentHangBody } from "../utils/agentHangReport.ts";
|
||||
import { formatJsonValue, log } from "../utils/cli.ts";
|
||||
import { installFromNpmTarball } from "../utils/install.ts";
|
||||
import { detectProviderError } from "../utils/providerErrors.ts";
|
||||
import { installCodexAuth } from "../utils/codexHome.ts";
|
||||
import { findProviderErrorMatch } from "../utils/providerErrors.ts";
|
||||
import { addSkill, installBundledSkills } from "../utils/skills.ts";
|
||||
import { SPAWN_ACTIVITY_TIMEOUT_CODE, SpawnTimeoutError, spawn } from "../utils/subprocess.ts";
|
||||
import {
|
||||
DEFAULT_MAX_RETAINED_BYTES,
|
||||
SPAWN_ACTIVITY_TIMEOUT_CODE,
|
||||
SpawnTimeoutError,
|
||||
spawn,
|
||||
TailBuffer,
|
||||
} from "../utils/subprocess.ts";
|
||||
import { ThinkingTimer } from "../utils/timer.ts";
|
||||
import type { TodoTracker } from "../utils/todoTracking.ts";
|
||||
import { getDevDependencyVersion } from "../utils/version.ts";
|
||||
import { buildLearningsReflectionPrompt, runPostRunRetryLoop } from "./postRun.ts";
|
||||
import { REVIEWER_AGENT_NAME, REVIEWER_SYSTEM_PROMPT } from "./reviewer.ts";
|
||||
import { resolveVertexOpenCodeModel } from "../utils/vertex.ts";
|
||||
import {
|
||||
PULLFROG_BUS_EVENT_TYPE,
|
||||
PULLFROG_OPENCODE_PLUGIN_FILENAME,
|
||||
PULLFROG_OPENCODE_PLUGIN_SOURCE,
|
||||
} from "./opencodePlugin.ts";
|
||||
import {
|
||||
autoSelectModel,
|
||||
buildReviewerAgentConfig,
|
||||
geminiHighThinkingOverrides,
|
||||
installOpencodeCli,
|
||||
type OpenCodeConfig,
|
||||
} from "./opencodeShared.ts";
|
||||
import {
|
||||
buildLearningsReflectionPrompt,
|
||||
runPostRunRetryLoop,
|
||||
shouldRunReflection,
|
||||
} from "./postRun.ts";
|
||||
import { REVIEWER_AGENT_NAME } from "./reviewer.ts";
|
||||
import { formatWithLabel, ORCHESTRATOR_LABEL, SessionLabeler } from "./sessionLabeler.ts";
|
||||
import {
|
||||
type AgentResult,
|
||||
@@ -38,42 +69,32 @@ import {
|
||||
MAX_STDERR_LINES,
|
||||
} from "./shared.ts";
|
||||
|
||||
async function installOpencodeCli(): Promise<string> {
|
||||
return await installFromNpmTarball({
|
||||
packageName: "opencode-ai",
|
||||
version: getDevDependencyVersion("opencode-ai"),
|
||||
executablePath: "bin/opencode",
|
||||
installDependencies: true,
|
||||
});
|
||||
}
|
||||
// re-export for the existing test (`./opencode.test.ts`) — once v1 is
|
||||
// retired this module collapses and the test imports from opencodeShared.
|
||||
export { geminiHighThinkingOverrides } from "./opencodeShared.ts";
|
||||
|
||||
// ── config ─────────────────────────────────────────────────────────────────────
|
||||
// v1.4-era npm package shipped a per-platform binary directly at this path.
|
||||
const installCli = () => installOpencodeCli({ binPath: "bin/opencode" });
|
||||
|
||||
type OpenCodeConfig = {
|
||||
mcp?: Record<string, unknown>;
|
||||
permission?: Record<string, unknown>;
|
||||
provider?: Record<string, unknown>;
|
||||
agent?: Record<string, unknown>;
|
||||
model?: string;
|
||||
enabled_providers?: string[];
|
||||
[key: string]: unknown;
|
||||
};
|
||||
|
||||
/**
|
||||
* Per-inference `max_tokens` reservation the agent sends to the upstream
|
||||
* model. OpenCode's default is 32_000 (sized for long-running TUI sessions
|
||||
* where a human user might want big outputs). Pullfrog runs are headless and
|
||||
* short — typical outputs are 1-3K tokens — so we cap at 5_000. This
|
||||
* drastically reduces the upfront budget reservation OpenRouter requires per
|
||||
* call (~$0.38 vs ~$2.40 for Opus), which is what lets low-wallet runs
|
||||
* actually start.
|
||||
*
|
||||
* Plumbed via `OPENCODE_EXPERIMENTAL_OUTPUT_TOKEN_MAX` env var rather than the
|
||||
* config JSON. OpenCode's `OUTPUT_TOKEN_MAX` (session/llm.ts) is sourced
|
||||
* exclusively from this env var; top-level `limit.output` in the config
|
||||
* has no read site and is silently dropped on merge.
|
||||
*/
|
||||
const PULLFROG_OPENCODE_OUTPUT_LIMIT = 5000;
|
||||
// NOTE: OpenCode's per-call `max_tokens` defaults to 32_000. We previously
|
||||
// overrode this via `OPENCODE_EXPERIMENTAL_OUTPUT_TOKEN_MAX = 5000` in #616
|
||||
// to lower OpenRouter's per-call upfront budget reservation — back when the
|
||||
// `ROUTER_PER_RUN_LIMIT_USD = 25` per-run key cap meant that reservation was
|
||||
// a hard gate that could lock low-balance accounts out of starting a run.
|
||||
//
|
||||
// That gate is gone (see `app/api/proxy-token/route.ts` ~line 422 — "Per-run
|
||||
// key budget … is decoupled from wallet balance"); the router now mints
|
||||
// keys with `keyLimitCents = balance + buffer` ($50 / $5 / $0). The override
|
||||
// no longer materially helps, and as a hard per-call output truncation it
|
||||
// actively hurt: a single `create_pull_request_review` tool_use with many
|
||||
// inline comments would truncate mid-stream past 5K output tokens, the JSON
|
||||
// was unparseable, and the tool never invoked. We hit this on PR #710's
|
||||
// verify-downshift PR. Removed in #710 — using OpenCode's 32K default.
|
||||
//
|
||||
// If you need to re-cap output for some reason, set
|
||||
// `OPENCODE_EXPERIMENTAL_OUTPUT_TOKEN_MAX` in the action env. OpenCode's
|
||||
// top-level `limit.output` config field has no read site (silently dropped
|
||||
// on merge in session/llm.ts), so the env var is the only working knob.
|
||||
|
||||
function buildSecurityConfig(ctx: AgentRunContext, model: string | undefined): string {
|
||||
const config: OpenCodeConfig = {
|
||||
@@ -88,7 +109,28 @@ function buildSecurityConfig(ctx: AgentRunContext, model: string | undefined): s
|
||||
mcp: {
|
||||
[pullfrogMcpName]: { type: "remote", url: ctx.mcpServerUrl },
|
||||
},
|
||||
agent: buildReviewerAgentConfig(),
|
||||
agent: (() => {
|
||||
const cfg = buildReviewerAgentConfig(model);
|
||||
const reviewerModel = (cfg[REVIEWER_AGENT_NAME] as { model?: string })?.model ?? "(inherit)";
|
||||
log.info(`» subagent models: reviewfrog=${reviewerModel}`);
|
||||
return cfg;
|
||||
})(),
|
||||
// NOTE: `experimental.batch_tool` was enabled in #719 to bundle 1-25
|
||||
// independent tool calls into one round trip, but the batch tool rejects
|
||||
// MCP/"external" tools with `"Tool '<name>' not in registry. External
|
||||
// tools (MCP, environment) cannot be batched - call them directly."`
|
||||
// (anomalyco/opencode PR #2983 design). when a model emits parallel
|
||||
// tool_use blocks containing `pullfrog_*` calls, opencode internally
|
||||
// routes them through batch — they all fail, the model misreads the
|
||||
// error as "the tool doesn't exist", and gives up. caught in CI by
|
||||
// `restricted-opencode` after a `lens:` subagent dispatched parallel
|
||||
// `pullfrog_shell` calls and concluded shell was unavailable.
|
||||
// native parallel tool_use (multiple tool_use blocks per assistant
|
||||
// message) still works without batch_tool for both built-in and MCP
|
||||
// tools, so we lose only the batch wrapper, not parallelism.
|
||||
// gemini-3 thinking pinned to high for review depth; gpt and anthropic
|
||||
// effort set elsewhere (gpt: upstream default, anthropic: --effort flag in claude.ts).
|
||||
provider: { google: { models: geminiHighThinkingOverrides() } },
|
||||
};
|
||||
|
||||
if (model) {
|
||||
@@ -103,76 +145,6 @@ function buildSecurityConfig(ctx: AgentRunContext, model: string | undefined): s
|
||||
return JSON.stringify(config);
|
||||
}
|
||||
|
||||
/**
|
||||
* Read-only subagent for self-review and /anneal lens dispatch. The
|
||||
* non-mutative + non-recursive contract is enforced by the prose system
|
||||
* prompt — see action/agents/reviewer.ts for why we no longer wire per-agent
|
||||
* tool/permission denies here.
|
||||
*/
|
||||
function buildReviewerAgentConfig(): Record<string, unknown> {
|
||||
return {
|
||||
[REVIEWER_AGENT_NAME]: {
|
||||
description:
|
||||
"Read-only review subagent for self-review and lens-based code review. " +
|
||||
"Reads only — no writes, no state-changing shell or MCP calls, no nested subagent dispatch.",
|
||||
mode: "subagent",
|
||||
prompt: REVIEWER_SYSTEM_PROMPT,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// ── model auto-select fallback ──────────────────────────────────────────────────
|
||||
//
|
||||
// steps 1–2 of model resolution (PULLFROG_MODEL env, slug resolution) are handled
|
||||
// by resolveModel() in utils/agent.ts before the agent runs. this fallback only
|
||||
// handles step 3: auto-select via `opencode models`.
|
||||
|
||||
function getOpenCodeModels(cliPath: string): string[] {
|
||||
try {
|
||||
const output = execFileSync(cliPath, ["models"], {
|
||||
encoding: "utf-8",
|
||||
timeout: 30_000,
|
||||
env: process.env,
|
||||
});
|
||||
return output
|
||||
.split("\n")
|
||||
.map((line) => line.trim())
|
||||
.filter(Boolean);
|
||||
} catch (error) {
|
||||
log.debug(
|
||||
`» failed to run \`opencode models\`: ${error instanceof Error ? error.message : String(error)}`
|
||||
);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
const AUTO_SELECT_WARNING =
|
||||
"select a model explicitly in the Pullfrog console (https://pullfrog.com/console) to avoid this.";
|
||||
|
||||
function autoSelectModel(cliPath: string): string | undefined {
|
||||
const availableModels = getOpenCodeModels(cliPath);
|
||||
const availableSet = new Set(availableModels);
|
||||
if (availableSet.size > 0) {
|
||||
log.debug(`» opencode models (${availableSet.size}): ${availableModels.join(", ")}`);
|
||||
const match =
|
||||
modelAliases.find((a) => a.preferred && availableSet.has(a.resolve)) ??
|
||||
modelAliases.find((a) => availableSet.has(a.resolve));
|
||||
if (match) {
|
||||
log.info(
|
||||
`» model: ${match.resolve} (auto-selected${match.preferred ? " — preferred" : ""} curated match)`
|
||||
);
|
||||
log.warning(`» model auto-selected. ${AUTO_SELECT_WARNING}`);
|
||||
return match.resolve;
|
||||
}
|
||||
log.info(
|
||||
`» opencode has ${availableSet.size} models but none match curated aliases — letting OpenCode auto-select`
|
||||
);
|
||||
}
|
||||
|
||||
log.warning(`» no model resolved. letting OpenCode auto-select. ${AUTO_SELECT_WARNING}`);
|
||||
return undefined;
|
||||
}
|
||||
|
||||
// ── NDJSON event types ─────────────────────────────────────────────────────────
|
||||
|
||||
interface OpenCodeInitEvent {
|
||||
@@ -228,6 +200,20 @@ interface OpenCodeStepFinishEvent {
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
/**
|
||||
* tool-part state, mirroring opencode's `ToolState` (anomalyco/opencode
|
||||
* `session/message-v2.ts`). error parts carry the reason on `error`,
|
||||
* completed parts on `output` — reading the wrong field is what caused
|
||||
* the silent `(no error message)` log in #662.
|
||||
*
|
||||
* Named `ToolPartState` locally (not `ToolState`) so it doesn't shadow the
|
||||
* action-wide `ToolState` imported above.
|
||||
*/
|
||||
type ToolPartState =
|
||||
| { status: "pending" | "running"; input?: unknown }
|
||||
| { status: "completed"; input?: unknown; output: string }
|
||||
| { status: "error"; input?: unknown; error: string };
|
||||
|
||||
interface OpenCodeToolUseEvent {
|
||||
type: "tool_use";
|
||||
timestamp?: number;
|
||||
@@ -236,7 +222,7 @@ interface OpenCodeToolUseEvent {
|
||||
id?: string;
|
||||
callID?: string;
|
||||
tool?: string;
|
||||
state?: { status?: string; input?: unknown; output?: string };
|
||||
state?: ToolPartState;
|
||||
};
|
||||
[key: string]: unknown;
|
||||
}
|
||||
@@ -245,7 +231,7 @@ interface OpenCodeToolResultEvent {
|
||||
type: "tool_result";
|
||||
timestamp?: number;
|
||||
sessionID?: string;
|
||||
part?: { callID?: string; state?: { status?: string; output?: string } };
|
||||
part?: { callID?: string; state?: ToolPartState };
|
||||
tool_id?: string;
|
||||
status?: "success" | "error";
|
||||
output?: string;
|
||||
@@ -280,6 +266,36 @@ interface OpenCodeErrorEvent {
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
/**
|
||||
* Envelope event emitted by our `.opencode/plugin/pullfrog-events.ts` (the
|
||||
* source lives in `opencodePlugin.ts`). The plugin subscribes to opencode's
|
||||
* bus via `bus.subscribeAll()` and re-emits non-orchestrator
|
||||
* `message.part.updated` events on stdout so subagent activity surfaces here.
|
||||
*
|
||||
* `bus_event.properties.part` matches the same `Part` shape that opencode's
|
||||
* `cli/cmd/run.ts` uses to drive its own emit() calls, so we can route the
|
||||
* inner part through the existing `tool_use` / `step_start` / `step_finish`
|
||||
* / `text` handlers by synthesizing the equivalent OpenCode-style event.
|
||||
*/
|
||||
interface OpenCodeBusEnvelopeEvent {
|
||||
type: "pullfrog_bus_event";
|
||||
bus_event?: {
|
||||
type?: string;
|
||||
properties?: {
|
||||
part?: {
|
||||
sessionID?: string;
|
||||
type?: string;
|
||||
time?: { end?: number | string };
|
||||
state?: { status?: string };
|
||||
[key: string]: unknown;
|
||||
};
|
||||
[key: string]: unknown;
|
||||
};
|
||||
[key: string]: unknown;
|
||||
};
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
type OpenCodeEvent =
|
||||
| OpenCodeInitEvent
|
||||
| OpenCodeMessageEvent
|
||||
@@ -289,7 +305,8 @@ type OpenCodeEvent =
|
||||
| OpenCodeToolUseEvent
|
||||
| OpenCodeToolResultEvent
|
||||
| OpenCodeResultEvent
|
||||
| OpenCodeErrorEvent;
|
||||
| OpenCodeErrorEvent
|
||||
| OpenCodeBusEnvelopeEvent;
|
||||
|
||||
// ── runner ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
@@ -299,6 +316,7 @@ type RunParams = {
|
||||
args: string[];
|
||||
cwd: string;
|
||||
env: Record<string, string | undefined>;
|
||||
toolState: ToolState;
|
||||
todoTracker?: TodoTracker | undefined;
|
||||
onActivityTimeout?: (() => void) | undefined;
|
||||
onToolUse?: ((event: { toolName: string; input: unknown }) => void) | undefined;
|
||||
@@ -307,7 +325,6 @@ type RunParams = {
|
||||
async function runOpenCode(params: RunParams): Promise<AgentResult> {
|
||||
const startTime = performance.now();
|
||||
let eventCount = 0;
|
||||
const thinkingTimer = new ThinkingTimer();
|
||||
|
||||
let finalOutput = "";
|
||||
let accumulatedTokens = { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 };
|
||||
@@ -324,15 +341,12 @@ async function runOpenCode(params: RunParams): Promise<AgentResult> {
|
||||
// per-session labeler so parallel subagent log lines can be differentiated.
|
||||
// the orchestrator's task tool_use events seed the labeler; the next
|
||||
// previously-unseen sessionID consumes the head of the pending-label queue.
|
||||
// NB: opencode's runtime currently encapsulates subagent execution inside
|
||||
// the `task` tool — subagent-internal tool_use/tool_result events do not
|
||||
// surface on the parent's NDJSON stream. The labeler is therefore mostly
|
||||
// dormant in practice for opencode (no per-event session differentiation
|
||||
// is needed because there are no per-subagent events). The orchestrator's
|
||||
// `task` dispatch log (with `description: <lens>`) and the per-task
|
||||
// duration log below are the actual attribution surface available today.
|
||||
// The labeler is kept in place defensively so that if/when opencode begins
|
||||
// streaming subagent sessions, attribution flips on with no further work.
|
||||
// upstream opencode's `cli/cmd/run.ts` filters subagent events out of its
|
||||
// NDJSON stream (`part.sessionID !== sessionID`), so we ship a per-run
|
||||
// plugin (`action/agents/opencodePlugin.ts`, written into the tmpdir at
|
||||
// setup) that re-emits non-orchestrator `message.part.updated` events. those
|
||||
// arrive here as `pullfrog_bus_event` envelopes and feed the labeler with
|
||||
// real data per subagent session.
|
||||
const labeler = new SessionLabeler();
|
||||
function eventLabel(event: Record<string, unknown>): string {
|
||||
const sid = event.sessionID ?? event.session_id;
|
||||
@@ -342,6 +356,23 @@ async function runOpenCode(params: RunParams): Promise<AgentResult> {
|
||||
return label === ORCHESTRATOR_LABEL ? message : formatWithLabel(label, message);
|
||||
}
|
||||
|
||||
// one ThinkingTimer per session — sharing a single timer across sessions
|
||||
// conflated cross-session interleaving (parent thinks → child tool_call,
|
||||
// or child returns → parent dispatches next) as parent thinking time. each
|
||||
// timer formats its log lines through the session label so the "thought
|
||||
// for X" attribution is visible in the merged stream.
|
||||
const thinkingTimers = new Map<string, ThinkingTimer>();
|
||||
function timerFor(label: string): ThinkingTimer {
|
||||
let t = thinkingTimers.get(label);
|
||||
if (!t) {
|
||||
const formatLine = (line: string) =>
|
||||
label === ORCHESTRATOR_LABEL ? line : formatWithLabel(label, line);
|
||||
t = new ThinkingTimer(formatLine);
|
||||
thinkingTimers.set(label, t);
|
||||
}
|
||||
return t;
|
||||
}
|
||||
|
||||
// tracks per-task dispatch metadata so the matching tool_result can log a
|
||||
// labeled "» subagent finished: lens=X duration=Ys" line. this is the most
|
||||
// useful per-lens observability available given that subagent-internal
|
||||
@@ -516,31 +547,52 @@ async function runOpenCode(params: RunParams): Promise<AgentResult> {
|
||||
return;
|
||||
}
|
||||
|
||||
// suspend the activity watchdog across the tool call (issue #760).
|
||||
// for `task` tool dispatches the injected plugin already reverbs
|
||||
// child.stdout chunks, so this is mostly defense-in-depth there;
|
||||
// for non-task MCP tools (checkout_pr, etc.) the suspend is the
|
||||
// only thing keeping a multi-minute fetch from tripping the 300s
|
||||
// spawn-level idle timer. gate by part status: bus-envelope
|
||||
// re-dispatches at line 915 fire only on terminal statuses
|
||||
// (`completed`/`error`) and never produce a paired `tool_result`,
|
||||
// so suspending on those would leak the watchdog open until the
|
||||
// 15min auto-resume — exactly the issue #12 zombie-run window.
|
||||
const status = event.part?.state?.status;
|
||||
if (status !== "completed" && status !== "error") {
|
||||
suspendActivity();
|
||||
}
|
||||
|
||||
// when the orchestrator dispatches a subagent via the `task` tool, push
|
||||
// a label for the upcoming child session so its events are attributable.
|
||||
// record BEFORE label lookup: this event's session is the parent (whose
|
||||
// label is already bound); the dispatch label is for the next new
|
||||
// sessionID that appears.
|
||||
if (toolName === "task") {
|
||||
const taskInput = (event.part?.state?.input ?? {}) as {
|
||||
description?: string;
|
||||
subagent_type?: string;
|
||||
prompt?: string;
|
||||
};
|
||||
const dispatchedLabel = labeler.recordTaskDispatch(taskInput);
|
||||
// dual-index by callID (fast path) AND in a FIFO queue (fallback path
|
||||
// for when opencode's task tool_result carries a different callID).
|
||||
const dispatch: TaskDispatch = {
|
||||
label: dispatchedLabel,
|
||||
startedAt: performance.now(),
|
||||
toolUseCallID: toolId,
|
||||
};
|
||||
taskDispatchByCallID.set(toolId, dispatch);
|
||||
pendingTaskDispatches.push(dispatch);
|
||||
log.info(
|
||||
`» dispatching subagent: ${dispatchedLabel}` +
|
||||
(taskInput.subagent_type ? ` (subagent_type=${taskInput.subagent_type})` : "")
|
||||
);
|
||||
// may have been pre-registered via the plugin's early task-dispatch
|
||||
// announcement (`pullfrog_bus_event` handler). dedupe on callID so
|
||||
// we don't record the same dispatch twice (which would corrupt the
|
||||
// FIFO label queue).
|
||||
if (!taskDispatchByCallID.has(toolId)) {
|
||||
const taskInput = (event.part?.state?.input ?? {}) as {
|
||||
description?: string;
|
||||
subagent_type?: string;
|
||||
prompt?: string;
|
||||
};
|
||||
const dispatchedLabel = labeler.recordTaskDispatch(taskInput);
|
||||
// dual-index by callID (fast path) AND in a FIFO queue (fallback path
|
||||
// for when opencode's task tool_result carries a different callID).
|
||||
const dispatch: TaskDispatch = {
|
||||
label: dispatchedLabel,
|
||||
startedAt: performance.now(),
|
||||
toolUseCallID: toolId,
|
||||
};
|
||||
taskDispatchByCallID.set(toolId, dispatch);
|
||||
pendingTaskDispatches.push(dispatch);
|
||||
log.info(
|
||||
`» dispatching subagent: ${dispatchedLabel}` +
|
||||
(taskInput.subagent_type ? ` (subagent_type=${taskInput.subagent_type})` : "")
|
||||
);
|
||||
}
|
||||
} else {
|
||||
// remember non-task callIDs so a later tool_result with that callID
|
||||
// is correctly identified as not-a-task (and we don't FIFO-pop a
|
||||
@@ -561,7 +613,7 @@ async function runOpenCode(params: RunParams): Promise<AgentResult> {
|
||||
});
|
||||
}
|
||||
|
||||
thinkingTimer.markToolCall();
|
||||
timerFor(label).markToolCall();
|
||||
const inputFormatted = formatJsonValue(event.part?.state?.input || {});
|
||||
const toolCallLine =
|
||||
inputFormatted !== "{}" ? `» ${toolName}(${inputFormatted})` : `» ${toolName}()`;
|
||||
@@ -570,6 +622,14 @@ async function runOpenCode(params: RunParams): Promise<AgentResult> {
|
||||
if (event.part?.state?.status === "completed" && event.part.state.output) {
|
||||
log.debug(withLabel(label, ` output: ${event.part.state.output}`));
|
||||
}
|
||||
// surface tool errors at info level. opencode emits tool parts at
|
||||
// status="error" through the same `tool_use` event the CLI's run-loop
|
||||
// (and our injected plugin for subagent parts) emits — without this
|
||||
// branch the only signal in the user's logs is `» <tool>(...)` with
|
||||
// no indication the call failed.
|
||||
if (event.part?.state?.status === "error") {
|
||||
log.info(withLabel(label, `» tool call failed: ${event.part.state.error}`));
|
||||
}
|
||||
|
||||
// agent's explicit MCP report_progress takes priority over todo tracking
|
||||
if (toolName.includes("report_progress") && params.todoTracker) {
|
||||
@@ -583,12 +643,19 @@ async function runOpenCode(params: RunParams): Promise<AgentResult> {
|
||||
}
|
||||
},
|
||||
tool_result: (event: OpenCodeToolResultEvent) => {
|
||||
resumeActivity();
|
||||
const toolId = event.part?.callID || event.tool_id;
|
||||
const status = event.part?.state?.status || event.status || "unknown";
|
||||
const output = event.part?.state?.output || event.output;
|
||||
const state = event.part?.state;
|
||||
const status = state?.status ?? event.status ?? "unknown";
|
||||
const payload =
|
||||
state?.status === "completed"
|
||||
? state.output
|
||||
: state?.status === "error"
|
||||
? state.error
|
||||
: event.output;
|
||||
const label = eventLabel(event);
|
||||
|
||||
thinkingTimer.markToolResult();
|
||||
timerFor(label).markToolResult();
|
||||
|
||||
// surface subagent completion at info level — opencode otherwise hides
|
||||
// per-task timing in debug-only logs, so a parallel multi-lens fan-out
|
||||
@@ -604,12 +671,12 @@ async function runOpenCode(params: RunParams): Promise<AgentResult> {
|
||||
if (taskDispatchByCallID.size > 0 || pendingTaskDispatches.length > 0) {
|
||||
if (toolId && taskDispatchByCallID.has(toolId)) {
|
||||
const dispatch = taskDispatchByCallID.get(toolId);
|
||||
if (dispatch) emitSubagentFinished(dispatch, status, output, "exact");
|
||||
if (dispatch) emitSubagentFinished(dispatch, status, payload, "exact");
|
||||
} else {
|
||||
const callIDIsKnownNonTask = toolId ? knownNonTaskCallIDs.has(toolId) : false;
|
||||
if (!callIDIsKnownNonTask && pendingTaskDispatches.length > 0) {
|
||||
const dispatch = pendingTaskDispatches[0]!;
|
||||
emitSubagentFinished(dispatch, status, output, "fifo");
|
||||
emitSubagentFinished(dispatch, status, payload, "fifo");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -626,13 +693,8 @@ async function runOpenCode(params: RunParams): Promise<AgentResult> {
|
||||
`» ${params.label} tool_result${stepContext}: id=${toolId}, status=${status}, duration=${Math.round(toolDuration)}ms`
|
||||
)
|
||||
);
|
||||
if (output) {
|
||||
log.debug(
|
||||
withLabel(
|
||||
label,
|
||||
` output: ${typeof output === "string" ? output : JSON.stringify(output)}`
|
||||
)
|
||||
);
|
||||
if (payload) {
|
||||
log.debug(withLabel(label, ` output: ${payload}`));
|
||||
}
|
||||
if (toolDuration > 5000) {
|
||||
log.info(
|
||||
@@ -645,11 +707,9 @@ async function runOpenCode(params: RunParams): Promise<AgentResult> {
|
||||
}
|
||||
}
|
||||
if (status === "error") {
|
||||
const errorMsg = typeof output === "string" ? output : JSON.stringify(output);
|
||||
log.info(withLabel(label, `» tool call failed: ${errorMsg}`));
|
||||
} else if (output) {
|
||||
const outputStr = typeof output === "string" ? output : JSON.stringify(output);
|
||||
log.debug(withLabel(label, `tool output: ${outputStr}`));
|
||||
log.info(withLabel(label, `» tool call failed: ${payload ?? "(no error message)"}`));
|
||||
} else if (payload) {
|
||||
log.debug(withLabel(label, `tool output: ${payload}`));
|
||||
}
|
||||
},
|
||||
error: (event: OpenCodeErrorEvent) => {
|
||||
@@ -690,6 +750,98 @@ async function runOpenCode(params: RunParams): Promise<AgentResult> {
|
||||
}
|
||||
}
|
||||
},
|
||||
[PULLFROG_BUS_EVENT_TYPE]: async (event: OpenCodeBusEnvelopeEvent) => {
|
||||
// surface subagent activity that opencode's CLI run-loop discards (it
|
||||
// filters `part.sessionID !== sessionID`). our injected plugin
|
||||
// (action/agents/opencodePlugin.ts) re-emits non-orchestrator
|
||||
// `message.part.updated` bus events; here we synthesize the equivalent
|
||||
// CLI-style event for each known part type and dispatch through the
|
||||
// existing handlers so labeling, attribution, and logging all reuse the
|
||||
// same code path as the orchestrator's events. mirrors the dispatch
|
||||
// logic in opencode-ai's `cli/cmd/run.ts` `loop()` function.
|
||||
const busEvent = event.bus_event;
|
||||
if (!busEvent || busEvent.type !== "message.part.updated") return;
|
||||
const part = busEvent.properties?.part;
|
||||
if (!part || typeof part.sessionID !== "string") return;
|
||||
const sessionID = part.sessionID;
|
||||
const partType = part.type;
|
||||
|
||||
// early task dispatch: the orchestrator's task tool fires bus events at
|
||||
// status=running BEFORE the subagent's first message.part.updated, but
|
||||
// the CLI's run-loop only emits the matching tool_use NDJSON event at
|
||||
// status=completed (after the subagent finishes). without
|
||||
// pre-registering the dispatch label here, the labeler binds the
|
||||
// subagent's sessionID to a generic `subagent#N` fallback before the
|
||||
// CLI's tool_use ever fires recordTaskDispatch. dedupe against
|
||||
// taskDispatchByCallID so the late tool_use handler doesn't double-add.
|
||||
if (partType === "tool") {
|
||||
const status = part.state?.status;
|
||||
const partWithToolFields = part as {
|
||||
tool?: string;
|
||||
callID?: string;
|
||||
state?: { status?: string; input?: unknown };
|
||||
};
|
||||
// only running (not pending) — at pending state.input is still {}.
|
||||
// by running, the LLM has filled in description/subagent_type/prompt.
|
||||
// mirrors the same check in the plugin source.
|
||||
const isOrchestratorTaskDispatch =
|
||||
partWithToolFields.tool === "task" && status === "running";
|
||||
if (isOrchestratorTaskDispatch) {
|
||||
const callID = partWithToolFields.callID;
|
||||
if (typeof callID === "string" && !taskDispatchByCallID.has(callID)) {
|
||||
const taskInput = (partWithToolFields.state?.input ?? {}) as {
|
||||
description?: string;
|
||||
subagent_type?: string;
|
||||
prompt?: string;
|
||||
};
|
||||
const dispatchedLabel = labeler.recordTaskDispatch(taskInput);
|
||||
const dispatch: TaskDispatch = {
|
||||
label: dispatchedLabel,
|
||||
startedAt: performance.now(),
|
||||
toolUseCallID: callID,
|
||||
};
|
||||
taskDispatchByCallID.set(callID, dispatch);
|
||||
pendingTaskDispatches.push(dispatch);
|
||||
log.info(
|
||||
`» dispatching subagent: ${dispatchedLabel}` +
|
||||
(taskInput.subagent_type ? ` (subagent_type=${taskInput.subagent_type})` : "")
|
||||
);
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (status !== "completed" && status !== "error") return;
|
||||
await handlers.tool_use({
|
||||
type: "tool_use",
|
||||
sessionID,
|
||||
part,
|
||||
} as OpenCodeToolUseEvent);
|
||||
return;
|
||||
}
|
||||
// intentionally NOT routing subagent step_start / step_finish through
|
||||
// the orchestrator's handlers:
|
||||
// - step_finish carries `tokens` and `cost` and the handler folds
|
||||
// them into the run-wide accumulators. surfacing subagent steps
|
||||
// here would inflate the orchestrator's usage telemetry — and
|
||||
// either double-count (if opencode also bills child tokens back
|
||||
// up to the parent session) or just over-report. the existing
|
||||
// init/message/text handlers all gate on ORCHESTRATOR_LABEL for
|
||||
// the same reason.
|
||||
// - step_start mutates `currentStepId` / `currentStepType` /
|
||||
// `stepHistory`, which are orchestrator-scoped — using them to
|
||||
// attribute subagent activity in the orchestrator's tool-use
|
||||
// timing log would be wrong.
|
||||
// the subagent's tool calls and text still surface (handled below)
|
||||
// — that's the user-visible activity.
|
||||
if (partType === "step-start" || partType === "step-finish") return;
|
||||
if (partType === "text" && part.time?.end !== undefined) {
|
||||
await handlers.text({
|
||||
type: "text",
|
||||
sessionID,
|
||||
part,
|
||||
} as OpenCodeTextEvent);
|
||||
return;
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
const recentStderr: string[] = [];
|
||||
@@ -697,7 +849,25 @@ async function runOpenCode(params: RunParams): Promise<AgentResult> {
|
||||
let lastProviderError: string | null = null;
|
||||
let agentErrorEvent: OpenCodeErrorEvent | null = null;
|
||||
|
||||
let output = "";
|
||||
// shared with main.ts via toolState. updated in place as events stream and
|
||||
// stderr accumulates so the outer activity-timeout catch sees the same
|
||||
// context the harness's own catch path uses to format `result.error`.
|
||||
// recentStderr is shared by reference; the scalar fields are mirrored on
|
||||
// each update below.
|
||||
const diagnostic: AgentDiagnostic = {
|
||||
label: params.label,
|
||||
recentStderr,
|
||||
lastProviderError: undefined,
|
||||
eventCount: 0,
|
||||
};
|
||||
params.toolState.agentDiagnostic = diagnostic;
|
||||
|
||||
// capped accumulator for the agent's narration. used as a post-run fallback
|
||||
// when `finalOutput` (the orchestrator's final assistant message) is empty.
|
||||
// unbounded `output += text` previously grew to ~1 GiB on multi-lens Reviews
|
||||
// and contributed to the wrapper-level RangeError. retain:"none" on spawn
|
||||
// skips the duplicate buffer there; this TailBuffer caps the agent layer.
|
||||
const output = new TailBuffer(DEFAULT_MAX_RETAINED_BYTES);
|
||||
let stdoutBuffer = "";
|
||||
|
||||
try {
|
||||
@@ -709,9 +879,30 @@ async function runOpenCode(params: RunParams): Promise<AgentResult> {
|
||||
activityTimeout: 300_000,
|
||||
onActivityTimeout: params.onActivityTimeout,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
// node_modules/opencode-ai/bin/opencode is a Node shim that spawnSyncs
|
||||
// the native opencode-<plat>-<arch> binary with stdio:"inherit". without
|
||||
// a process-group kill, SIGKILL hits only the shim, the native binary
|
||||
// is reparented to PID 1, holds our stdout pipe open, and `child.close`
|
||||
// never fires — producing zombie runs. detached + killGroup nukes the
|
||||
// whole tree.
|
||||
killGroup: true,
|
||||
// we already drain every chunk via onStdout/onStderr (NDJSON parsing
|
||||
// + recentStderr ring buffer). retaining a second copy in the spawn
|
||||
// wrapper would grow unbounded for multi-lens Reviews and previously
|
||||
// crashed the wrapper with RangeError at ~1 GiB. see issue #680.
|
||||
retain: "none",
|
||||
// suspend the spawn-level idle watchdog across MCP tool calls (issue
|
||||
// #760). bracketed by suspendActivity()/resumeActivity() in the
|
||||
// tool_use/tool_result handlers above, bounded by
|
||||
// MAX_TOOL_CALL_SUSPENSION_MS in activity.ts. the injected plugin
|
||||
// (action/agents/opencodePlugin.ts) re-emits subagent
|
||||
// `message.part.updated` events on opencode's stdout, so subagent
|
||||
// dispatches keep marking child.stdout activity as well — defense
|
||||
// in depth (verified empirically in PR #634, ~3.3 plugin events/sec).
|
||||
isPausedExternally: isActivitySuspended,
|
||||
onStdout: async (chunk) => {
|
||||
const text = chunk.toString();
|
||||
output += text;
|
||||
output.append(text);
|
||||
markActivity();
|
||||
|
||||
stdoutBuffer += text;
|
||||
@@ -731,6 +922,7 @@ async function runOpenCode(params: RunParams): Promise<AgentResult> {
|
||||
}
|
||||
|
||||
eventCount++;
|
||||
diagnostic.eventCount = eventCount;
|
||||
log.debug(JSON.stringify(event, null, 2));
|
||||
|
||||
const timeSinceLastActivity = getIdleMs();
|
||||
@@ -769,10 +961,11 @@ async function runOpenCode(params: RunParams): Promise<AgentResult> {
|
||||
recentStderr.push(trimmed);
|
||||
if (recentStderr.length > MAX_STDERR_LINES) recentStderr.shift();
|
||||
|
||||
const providerError = detectProviderError(trimmed);
|
||||
if (providerError) {
|
||||
lastProviderError = providerError;
|
||||
log.info(`» provider error detected (${providerError}): ${trimmed.substring(0, 500)}`);
|
||||
const match = findProviderErrorMatch(trimmed);
|
||||
if (match) {
|
||||
lastProviderError = match.label;
|
||||
diagnostic.lastProviderError = match.label;
|
||||
log.info(`» provider error detected (${match.label}): ${match.excerpt}`);
|
||||
} else {
|
||||
log.debug(trimmed);
|
||||
}
|
||||
@@ -836,22 +1029,31 @@ async function runOpenCode(params: RunParams): Promise<AgentResult> {
|
||||
|
||||
if (result.exitCode !== 0) {
|
||||
const errorContext = lastProviderError ? ` (${lastProviderError})` : "";
|
||||
// result.stdout / result.stderr are empty because we pass retain:"none"
|
||||
// to spawn (see issue #680); use the agent's bounded mirrors instead.
|
||||
const stdoutSnapshot = output.toString();
|
||||
const stderrSnapshot = recentStderr.join("\n");
|
||||
const errorMessage =
|
||||
result.stderr ||
|
||||
result.stdout ||
|
||||
stderrSnapshot ||
|
||||
stdoutSnapshot ||
|
||||
`unknown error - no output from OpenCode CLI${errorContext}`;
|
||||
log.error(
|
||||
`${params.label} exited with code ${result.exitCode}${errorContext}: ${errorMessage}`
|
||||
);
|
||||
log.debug(`stdout: ${result.stdout?.substring(0, 500)}`);
|
||||
log.debug(`stderr: ${result.stderr?.substring(0, 500)}`);
|
||||
return { success: false, output: finalOutput || output, error: errorMessage, usage };
|
||||
log.debug(`stdout: ${stdoutSnapshot.substring(0, 500)}`);
|
||||
log.debug(`stderr: ${stderrSnapshot.substring(0, 500)}`);
|
||||
return {
|
||||
success: false,
|
||||
output: finalOutput || stdoutSnapshot,
|
||||
error: errorMessage,
|
||||
usage,
|
||||
};
|
||||
}
|
||||
|
||||
if (eventCount === 0 && lastProviderError) {
|
||||
return {
|
||||
success: false,
|
||||
output: finalOutput || output,
|
||||
output: finalOutput || output.toString(),
|
||||
error: `provider error: ${lastProviderError}`,
|
||||
usage,
|
||||
};
|
||||
@@ -864,13 +1066,13 @@ async function runOpenCode(params: RunParams): Promise<AgentResult> {
|
||||
errorEvent.error?.data?.message || errorEvent.error?.name || JSON.stringify(errorEvent);
|
||||
return {
|
||||
success: false,
|
||||
output: finalOutput || output,
|
||||
output: finalOutput || output.toString(),
|
||||
error: `${errorName}: ${errorMessage}`,
|
||||
usage,
|
||||
};
|
||||
}
|
||||
|
||||
return { success: true, output: finalOutput || output, usage };
|
||||
return { success: true, output: finalOutput || output.toString(), usage };
|
||||
} catch (error) {
|
||||
params.todoTracker?.cancel();
|
||||
const duration = performance.now() - startTime;
|
||||
@@ -894,10 +1096,11 @@ async function runOpenCode(params: RunParams): Promise<AgentResult> {
|
||||
`» recent stderr (last ${Math.min(recentStderr.length, 10)} lines):\n${stderrContext}`
|
||||
);
|
||||
|
||||
const body = formatAgentHangBody({ diagnostic, isHang: isActivityTimeout, errorMessage });
|
||||
return {
|
||||
success: false,
|
||||
output: finalOutput || output,
|
||||
error: `${errorMessage} [${diagnosis}]`,
|
||||
output: finalOutput || output.toString(),
|
||||
error: body ?? `${errorMessage} [${diagnosis}]`,
|
||||
usage: buildUsage(),
|
||||
};
|
||||
}
|
||||
@@ -907,11 +1110,35 @@ async function runOpenCode(params: RunParams): Promise<AgentResult> {
|
||||
|
||||
export const opencode = agent({
|
||||
name: "opencode",
|
||||
install: installOpencodeCli,
|
||||
install: installCli,
|
||||
run: async (ctx) => {
|
||||
const cliPath = await installOpencodeCli();
|
||||
const cliPath = await installCli();
|
||||
|
||||
const model = ctx.payload.proxyModel ?? ctx.resolvedModel ?? autoSelectModel(cliPath);
|
||||
const rawModel = ctx.payload.proxyModel ?? ctx.resolvedModel ?? autoSelectModel(cliPath);
|
||||
|
||||
// bedrock route: opencode's `amazon-bedrock` provider expects the model
|
||||
// string in `amazon-bedrock/<bedrock-id>` form. the bare AWS model ID
|
||||
// (what the user puts in `BEDROCK_MODEL_ID`) needs the prefix added.
|
||||
// detect via env-var sentinel — same pattern as claude.ts.
|
||||
//
|
||||
// we deliberately do NOT gate on `!isBedrockAnthropicId(rawModel)` here:
|
||||
// Anthropic-on-Bedrock normally routes to claude-code (per `resolveAgent`),
|
||||
// but `PULLFROG_AGENT=opencode` is the documented escape hatch for forcing
|
||||
// opencode regardless. when that override fires, opencode still needs the
|
||||
// `amazon-bedrock/` prefix or the provider lookup fails with
|
||||
// "Model not found: <modelId>/.". the Anthropic-vs-other discriminant
|
||||
// only belongs in `resolveAgent`.
|
||||
const bedrockModelId = process.env[BEDROCK_MODEL_ID_ENV]?.trim();
|
||||
const isBedrockRoute =
|
||||
rawModel !== undefined && bedrockModelId !== undefined && bedrockModelId === rawModel;
|
||||
let model = rawModel;
|
||||
if (isBedrockRoute) {
|
||||
model = `amazon-bedrock/${rawModel}`;
|
||||
}
|
||||
const vertexModel = resolveVertexOpenCodeModel(rawModel);
|
||||
if (vertexModel) {
|
||||
model = vertexModel;
|
||||
}
|
||||
|
||||
const homeEnv = {
|
||||
HOME: ctx.tmpdir,
|
||||
@@ -920,6 +1147,20 @@ export const opencode = agent({
|
||||
|
||||
mkdirSync(join(homeEnv.XDG_CONFIG_HOME, "opencode"), { recursive: true });
|
||||
|
||||
// drop our bus-event surfacing plugin into opencode's global config dir
|
||||
// (which we've redirected to the per-run tmpdir via XDG_CONFIG_HOME).
|
||||
// opencode auto-discovers plugins from `<Global.Path.config>/{plugin,plugins}/*.{ts,js}`
|
||||
// (see `packages/opencode/src/config/config.ts:633` calling
|
||||
// `ConfigPlugin.load(dir)`), so this lands in the loader without any
|
||||
// config wiring. critically: this MUST be inside the tmpdir, never the
|
||||
// user's repo working tree — see AGENTS.md.
|
||||
const opencodePluginDir = join(homeEnv.XDG_CONFIG_HOME, "opencode", "plugin");
|
||||
mkdirSync(opencodePluginDir, { recursive: true });
|
||||
writeFileSync(
|
||||
join(opencodePluginDir, PULLFROG_OPENCODE_PLUGIN_FILENAME),
|
||||
PULLFROG_OPENCODE_PLUGIN_SOURCE
|
||||
);
|
||||
|
||||
const agentBrowserVersion = getDevDependencyVersion("agent-browser");
|
||||
addSkill({
|
||||
ref: `vercel-labs/agent-browser@v${agentBrowserVersion}`,
|
||||
@@ -930,12 +1171,20 @@ export const opencode = agent({
|
||||
|
||||
installBundledSkills({ home: homeEnv.HOME });
|
||||
|
||||
// materialize CODEX_AUTH_JSON (Pullfrog-stored Codex subscription
|
||||
// credential) into the runner's REAL $HOME/.local/share/opencode/auth.json
|
||||
// so OpenCode's CodexAuthPlugin picks it up and routes openai requests
|
||||
// through the ChatGPT subscription instead of needing OPENAI_API_KEY.
|
||||
// see action/utils/codexHome.ts and wiki/codex-auth.md.
|
||||
const codexAuth = installCodexAuth();
|
||||
|
||||
// base args shared between initial run and continue runs
|
||||
const baseArgs = ["run", "--format", "json", "--print-logs"];
|
||||
|
||||
// OPENCODE_PERMISSION has absolute highest precedence (merged after managed/MDM configs).
|
||||
// external_directory gates ALL native filesystem tools (Read, Write, Edit, Glob, Grep, etc.)
|
||||
// for paths outside the project root. last-match-wins: deny everything, then allow /tmp.
|
||||
// auth.json sits under real $HOME (outside /tmp/*), so deny-default protects it.
|
||||
const permissionOverride = JSON.stringify({
|
||||
external_directory: { "*": "deny", "/tmp/*": "allow" },
|
||||
});
|
||||
@@ -945,11 +1194,32 @@ export const opencode = agent({
|
||||
...homeEnv,
|
||||
OPENCODE_CONFIG_CONTENT: buildSecurityConfig(ctx, model),
|
||||
OPENCODE_PERMISSION: permissionOverride,
|
||||
OPENCODE_EXPERIMENTAL_OUTPUT_TOKEN_MAX: PULLFROG_OPENCODE_OUTPUT_LIMIT.toString(),
|
||||
GOOGLE_GENERATIVE_AI_API_KEY:
|
||||
process.env.GOOGLE_GENERATIVE_AI_API_KEY || process.env.GEMINI_API_KEY,
|
||||
};
|
||||
|
||||
if (codexAuth) {
|
||||
// point OpenCode at the real-home XDG dir so it reads auth.json from
|
||||
// where we wrote it (not the tmpdir-redirected default).
|
||||
env.XDG_DATA_HOME = codexAuth.xdgDataHome;
|
||||
// remove OPENAI_API_KEY so OpenCode's provider merge unambiguously
|
||||
// picks the OAuth path. with both set, the merge order in opencode
|
||||
// makes the effective key ambiguous.
|
||||
delete env.OPENAI_API_KEY;
|
||||
// hand the post-hook everything it needs to detect + persist refresh.
|
||||
// post-hook runs in a fresh node process, so we have to ferry apiToken
|
||||
// explicitly — env is preserved across main/post but our run-context
|
||||
// JWT is computed at runtime and not put in env. see action/entryPost.ts.
|
||||
core.saveState(
|
||||
"codex_writeback",
|
||||
JSON.stringify({
|
||||
apiToken: ctx.apiToken,
|
||||
authPath: codexAuth.authPath,
|
||||
originalRefresh: codexAuth.originalRefresh,
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
const repoDir = process.cwd();
|
||||
|
||||
log.debug(`» starting Pullfrog (OpenCode): ${cliPath} ${baseArgs.join(" ")}`);
|
||||
@@ -960,6 +1230,7 @@ export const opencode = agent({
|
||||
cliPath,
|
||||
cwd: repoDir,
|
||||
env,
|
||||
toolState: ctx.toolState,
|
||||
todoTracker: ctx.todoTracker,
|
||||
onActivityTimeout: ctx.onActivityTimeout,
|
||||
onToolUse: ctx.onToolUse,
|
||||
@@ -976,12 +1247,13 @@ export const opencode = agent({
|
||||
// the reflection prompt fires once after gates go clean, as a dedicated
|
||||
// turn that nudges the agent to persist learnings.
|
||||
return runPostRunRetryLoop({
|
||||
ctx,
|
||||
initialResult: result,
|
||||
initialUsage: result.usage,
|
||||
stopScript: ctx.stopScript,
|
||||
summaryFilePath: ctx.summaryFilePath,
|
||||
summarySeed: ctx.summarySeed,
|
||||
reflectionPrompt: buildLearningsReflectionPrompt("opencode"),
|
||||
reflectionPrompt:
|
||||
ctx.toolState.learningsFilePath && shouldRunReflection(ctx.toolState.selectedMode)
|
||||
? buildLearningsReflectionPrompt(ctx.toolState.learningsFilePath)
|
||||
: undefined,
|
||||
resume: async (c) =>
|
||||
runOpenCode({
|
||||
...runParams,
|
||||
|
||||
@@ -0,0 +1,139 @@
|
||||
/**
|
||||
* Source for the opencode plugin we drop into the per-run tmpdir at
|
||||
* `<XDG_CONFIG_HOME>/opencode/plugin/pullfrog-events.ts`. The harness already
|
||||
* redirects `XDG_CONFIG_HOME` to `ctx.tmpdir/.config` (see `opencode.ts`
|
||||
* `homeEnv`), so opencode's auto-discovery scans the tmpdir, never the user's
|
||||
* working tree. opencode's `Global.Path.config` resolves to
|
||||
* `path.join(xdgConfig, "opencode")` and the config layer auto-discovers
|
||||
* plugins from every directory in its scan list — including
|
||||
* `Global.Path.config` — by globbing `{plugin,plugins}/*.{ts,js}` via
|
||||
* `ConfigPlugin.load(dir)`.
|
||||
*
|
||||
* We MUST NOT write into the user's repo working tree. The repo is a checkout
|
||||
* the agent operates on; only the agent's own tools (gated by
|
||||
* `OPENCODE_PERMISSION`) may modify it. The whole reason we redirect HOME and
|
||||
* XDG_CONFIG_HOME is so harness-side files (config, plugins, scratch state)
|
||||
* land in the tmpdir.
|
||||
*
|
||||
* Why this plugin exists: opencode's `task` tool runs subagents in-process and
|
||||
* the CLI's `cli/cmd/run.ts` event loop filters `part.sessionID !== sessionID`,
|
||||
* so subagent-internal `message.part.updated` events are silently discarded
|
||||
* before reaching our parent NDJSON stream. plugins, by contrast, receive
|
||||
* EVERY bus event via `bus.subscribeAll()` regardless of session.
|
||||
*
|
||||
* The plugin re-emits every relevant bus event onto opencode's stdout as a
|
||||
* single JSON line wrapped in a sentinel envelope. our `runOpenCode` parser
|
||||
* recognises the envelope, unpacks it, and routes the inner part through the
|
||||
* existing handlers with a per-session label from `SessionLabeler` so each
|
||||
* subagent's tool calls / text appear inline alongside the orchestrator's.
|
||||
*
|
||||
* Dumb plugin / smart parent split: the plugin emits every part for every
|
||||
* session. the parent dedupes against the orchestrator's own session id (which
|
||||
* it already knows from the `init` event). this keeps the plugin trivial and
|
||||
* keeps the per-session attribution logic on the parent side where the
|
||||
* SessionLabeler already lives.
|
||||
*
|
||||
* Event-name prefixing: the wrapped event-type sentinel is
|
||||
* `pullfrog_bus_event` — picked to be unmistakably ours so a future opencode
|
||||
* release that introduces a coincidentally-named event type won't collide.
|
||||
*/
|
||||
|
||||
export const PULLFROG_BUS_EVENT_TYPE = "pullfrog_bus_event" as const;
|
||||
|
||||
export const PULLFROG_OPENCODE_PLUGIN_FILENAME = "pullfrog-events.ts" as const;
|
||||
|
||||
/**
|
||||
* Source written verbatim to `<XDG_CONFIG_HOME>/opencode/plugin/pullfrog-events.ts`.
|
||||
*
|
||||
* - Structural typing only (no runtime import of `@opencode-ai/plugin`):
|
||||
* opencode installs that dep into the directory containing the plugin
|
||||
* alongside discovery, but a) the dep isn't required for the structural
|
||||
* shape we use, and b) keeping zero imports avoids any module-resolution
|
||||
* coupling to opencode's plugin-loader internals across versions.
|
||||
* - default export is the plugin factory (opencode's plugin loader accepts
|
||||
* default exports as the server entrypoint).
|
||||
* - we only forward `message.part.updated`. that's where the user-visible
|
||||
* subagent activity (tool calls, text, step transitions) lives. add more
|
||||
* event types here if the parent needs them.
|
||||
* - JSON.stringify+single write keeps the line atomic up to PIPE_BUF (4KB on
|
||||
* Linux). longer parts may interleave with concurrent stdout writers; the
|
||||
* parser tolerates non-JSON lines (logs them at debug) so a torn line is a
|
||||
* missed event, not a crash.
|
||||
*/
|
||||
export const PULLFROG_OPENCODE_PLUGIN_SOURCE = `// AUTOGENERATED by Pullfrog. do not edit; it'll be overwritten on the next run.
|
||||
// surfaces opencode subagent activity that the CLI's run-loop discards. see
|
||||
// action/agents/opencodePlugin.ts in pullfrog/app for why this exists. lives
|
||||
// inside the per-run tmpdir (XDG_CONFIG_HOME/opencode/plugin/), never inside
|
||||
// the user's working tree.
|
||||
|
||||
const PULLFROG_BUS_EVENT_TYPE = ${JSON.stringify(PULLFROG_BUS_EVENT_TYPE)};
|
||||
|
||||
// the first sessionID we see on a message.part.updated event is the
|
||||
// orchestrator — opencode's run command creates exactly one top-level session
|
||||
// before any subagent is dispatched, and the user-prompt text part fires
|
||||
// before the first task tool_use. we lock that sessionID in here and use it
|
||||
// to filter: the orchestrator's events are already streamed by the CLI's
|
||||
// run-loop, so we only forward (a) all subagent events, and (b) the
|
||||
// orchestrator's task tool dispatches at status="running". the CLI only
|
||||
// emits task tool_use at status=completed (after the subagent finishes), so
|
||||
// without the early announce the parent's labeler binds subagent sessions
|
||||
// before recordTaskDispatch fires and the lens label is lost.
|
||||
let orchestratorSessionID: string | undefined;
|
||||
|
||||
function isOrchestratorTaskDispatch(part: {
|
||||
type?: string;
|
||||
tool?: string;
|
||||
state?: { status?: string };
|
||||
}): boolean {
|
||||
if (part.type !== "tool") return false;
|
||||
if (part.tool !== "task") return false;
|
||||
// only forward at status="running" (not "pending"). at pending the
|
||||
// state.input is still {} — the orchestrator has emitted the part shell
|
||||
// but the LLM hasn't filled in description/subagent_type/prompt yet. by
|
||||
// running, input is populated and recordTaskDispatch can derive the lens
|
||||
// label correctly.
|
||||
return part.state?.status === "running";
|
||||
}
|
||||
|
||||
export default async function pullfrogEventsPlugin() {
|
||||
return {
|
||||
event: async (input: {
|
||||
event: {
|
||||
type: string;
|
||||
properties?: {
|
||||
part?: {
|
||||
sessionID?: string;
|
||||
type?: string;
|
||||
tool?: string;
|
||||
state?: { status?: string };
|
||||
};
|
||||
};
|
||||
};
|
||||
}) => {
|
||||
const event = input.event;
|
||||
if (!event || typeof event !== "object") return;
|
||||
if (event.type !== "message.part.updated") return;
|
||||
const part = event.properties?.part;
|
||||
const sessionID = part?.sessionID;
|
||||
if (typeof sessionID !== "string" || sessionID.length === 0) return;
|
||||
if (orchestratorSessionID === undefined) orchestratorSessionID = sessionID;
|
||||
|
||||
if (sessionID === orchestratorSessionID) {
|
||||
// skip orchestrator events EXCEPT early task dispatches.
|
||||
if (!part || !isOrchestratorTaskDispatch(part)) return;
|
||||
}
|
||||
|
||||
try {
|
||||
const line = JSON.stringify({
|
||||
type: PULLFROG_BUS_EVENT_TYPE,
|
||||
bus_event: event,
|
||||
});
|
||||
process.stdout.write(line + "\\n");
|
||||
} catch {
|
||||
// a circular reference or BigInt etc. would throw; swallow rather
|
||||
// than letting a single bad event take down the plugin.
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
`;
|
||||
@@ -0,0 +1,144 @@
|
||||
// Shared helpers for the OpenCode agent harnesses (`./opencode.ts` v1 and
|
||||
// `./opencode_v2.ts` v2). Pure config / model-registry / install glue —
|
||||
// nothing here touches the NDJSON event loop, which differs between v1 and v2.
|
||||
//
|
||||
// Once v1 is deleted post-burn-in this module collapses back into v2; until
|
||||
// then it keeps both runners synchronized so a config drift can't make v1 a
|
||||
// silently-broken fallback.
|
||||
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { modelAliases } from "../models.ts";
|
||||
import { log } from "../utils/cli.ts";
|
||||
import { installFromNpmTarball } from "../utils/install.ts";
|
||||
import { getDevDependencyVersion } from "../utils/version.ts";
|
||||
import { REVIEWER_AGENT_NAME, REVIEWER_SYSTEM_PROMPT } from "./reviewer.ts";
|
||||
import { deriveSubagentModels } from "./subagentModels.ts";
|
||||
|
||||
// ── config ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
export type OpenCodeConfig = {
|
||||
mcp?: Record<string, unknown>;
|
||||
permission?: Record<string, unknown>;
|
||||
provider?: Record<string, unknown>;
|
||||
agent?: Record<string, unknown>;
|
||||
experimental?: Record<string, unknown>;
|
||||
model?: string;
|
||||
enabled_providers?: string[];
|
||||
[key: string]: unknown;
|
||||
};
|
||||
|
||||
/**
|
||||
* Build the `provider.google.models[id].options` map that pins every direct-Google
|
||||
* Gemini alias to `thinkingLevel: "high"`. Sourced from the model registry so
|
||||
* adding/renaming a Google alias in `action/models.ts` flows through automatically.
|
||||
*/
|
||||
export function geminiHighThinkingOverrides(): Record<string, { options: object }> {
|
||||
return Object.fromEntries(
|
||||
modelAliases
|
||||
.filter((a) => a.provider === "google")
|
||||
.map((a) => [
|
||||
a.resolve.replace(/^google\//, ""),
|
||||
{ options: { thinkingConfig: { thinkingLevel: "high" } } },
|
||||
])
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Read-only `reviewfrog` subagent for lens-based review. Non-mutative +
|
||||
* non-recursive — enforced by the system prompt in reviewer.ts.
|
||||
*
|
||||
* Per-subagent `model:` override is driven by the registry in
|
||||
* `action/models.ts` via each alias's `subagentModel` field. Currently wired:
|
||||
* Anthropic opus → sonnet, OpenAI gpt-pro → gpt and gpt → gpt-5.4, Google
|
||||
* gemini-pro → gemini-flash. Other providers inherit (no override).
|
||||
*/
|
||||
export function buildReviewerAgentConfig(
|
||||
orchestratorModel: string | undefined
|
||||
): Record<string, unknown> {
|
||||
const overrides = deriveSubagentModels(orchestratorModel);
|
||||
return {
|
||||
[REVIEWER_AGENT_NAME]: {
|
||||
description:
|
||||
"Read-only review subagent for lens-based code review (correctness, security, billing-subsystem, etc.). " +
|
||||
"Reads only — no writes, no state-changing shell or MCP calls, no nested subagent dispatch.",
|
||||
mode: "subagent",
|
||||
prompt: REVIEWER_SYSTEM_PROMPT,
|
||||
...(overrides.reviewer !== undefined ? { model: overrides.reviewer } : {}),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// ── install ────────────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Install the opencode-ai npm tarball and return the path to the executable.
|
||||
*
|
||||
* The bin path differs by version: v1.4.x and earlier shipped `bin/opencode`;
|
||||
* v1.14+ renames the platform-specific binary to `bin/opencode.exe` for every
|
||||
* OS via the postinstall script. Callers pass the binPath that matches their
|
||||
* pinned version so a v1↔v2 swap can't silently install the wrong file.
|
||||
*/
|
||||
export async function installOpencodeCli(params: { binPath: string }): Promise<string> {
|
||||
return await installFromNpmTarball({
|
||||
packageName: "opencode-ai",
|
||||
version: getDevDependencyVersion("opencode-ai"),
|
||||
executablePath: params.binPath,
|
||||
installDependencies: true,
|
||||
});
|
||||
}
|
||||
|
||||
// ── model auto-select fallback ──────────────────────────────────────────────────
|
||||
//
|
||||
// steps 1–2 of model resolution (PULLFROG_MODEL env, slug resolution) happen
|
||||
// in resolveModel() in utils/agent.ts before the agent runs. this is step 3:
|
||||
// auto-select via `opencode models`.
|
||||
|
||||
const AUTO_SELECT_WARNING =
|
||||
"select a model explicitly in the Pullfrog console (https://pullfrog.com/console) to avoid this.";
|
||||
|
||||
function getOpenCodeModels(cliPath: string): string[] {
|
||||
try {
|
||||
const output = execFileSync(cliPath, ["models"], {
|
||||
encoding: "utf-8",
|
||||
timeout: 30_000,
|
||||
env: process.env,
|
||||
});
|
||||
return output
|
||||
.split("\n")
|
||||
.map((line) => line.trim())
|
||||
.filter(Boolean);
|
||||
} catch (error) {
|
||||
log.debug(
|
||||
`» failed to run \`opencode models\`: ${error instanceof Error ? error.message : String(error)}`
|
||||
);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
export function autoSelectModel(cliPath: string): string | undefined {
|
||||
const availableModels = getOpenCodeModels(cliPath);
|
||||
const availableSet = new Set(availableModels);
|
||||
if (availableSet.size > 0) {
|
||||
log.debug(`» opencode models (${availableSet.size}): ${availableModels.join(", ")}`);
|
||||
// skip hidden aliases (internal subagent-tier targets like
|
||||
// opencode/gpt-5.4) — they should never surface as a user-facing
|
||||
// orchestrator pick. mirrors the selectable-list filter in
|
||||
// components/ModelSelector.tsx and action/commands/init.ts.
|
||||
const match =
|
||||
modelAliases.find((a) => !a.hidden && a.preferred && availableSet.has(a.resolve)) ??
|
||||
modelAliases.find((a) => !a.hidden && availableSet.has(a.resolve));
|
||||
if (match) {
|
||||
log.info(
|
||||
`» model: ${match.resolve} (auto-selected${match.preferred ? " — preferred" : ""} curated match)`
|
||||
);
|
||||
log.warning(`» model auto-selected. ${AUTO_SELECT_WARNING}`);
|
||||
return match.resolve;
|
||||
}
|
||||
log.info(
|
||||
`» opencode has ${availableSet.size} models but none match curated aliases — letting OpenCode auto-select`
|
||||
);
|
||||
}
|
||||
|
||||
log.warning(`» no model resolved. letting OpenCode auto-select. ${AUTO_SELECT_WARNING}`);
|
||||
return undefined;
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
+47
-412
@@ -1,429 +1,64 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { SPAWN_TIMEOUT_CODE, SpawnTimeoutError } from "../utils/subprocess.ts";
|
||||
import type { AgentResult } from "./shared.ts";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { ToolState } from "../toolState.ts";
|
||||
import { getUnsubmittedReview } from "./postRun.ts";
|
||||
|
||||
vi.mock("./shared.ts", async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import("./shared.ts")>();
|
||||
function makeToolState(overrides: Partial<ToolState> = {}): ToolState {
|
||||
return {
|
||||
...actual,
|
||||
getGitStatus: vi.fn(() => ""),
|
||||
progressComment: undefined,
|
||||
hadProgressComment: true,
|
||||
prepushFailureCount: 0,
|
||||
backgroundProcesses: new Map(),
|
||||
usageEntries: [],
|
||||
...overrides,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
vi.mock("../utils/subprocess.ts", async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import("../utils/subprocess.ts")>();
|
||||
return {
|
||||
...actual,
|
||||
spawn: vi.fn(),
|
||||
};
|
||||
});
|
||||
|
||||
const { runPostRunRetryLoop, executeStopHook } = await import("./postRun.ts");
|
||||
const { getGitStatus } = await import("./shared.ts");
|
||||
const { spawn } = await import("../utils/subprocess.ts");
|
||||
const mockedGetGitStatus = vi.mocked(getGitStatus);
|
||||
const mockedSpawn = vi.mocked(spawn);
|
||||
|
||||
const successResult = (overrides: Partial<AgentResult> = {}): AgentResult => ({
|
||||
success: true,
|
||||
output: "ok",
|
||||
...overrides,
|
||||
});
|
||||
|
||||
describe("runPostRunRetryLoop — reflection turn", () => {
|
||||
beforeEach(() => {
|
||||
mockedGetGitStatus.mockReset();
|
||||
mockedGetGitStatus.mockReturnValue("");
|
||||
mockedSpawn.mockReset();
|
||||
describe("getUnsubmittedReview", () => {
|
||||
it("returns null when mode is not a review mode", () => {
|
||||
expect(getUnsubmittedReview(makeToolState({ selectedMode: "Build" }))).toBeNull();
|
||||
expect(getUnsubmittedReview(makeToolState())).toBeNull();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
it("returns null when a review was already submitted", () => {
|
||||
expect(
|
||||
getUnsubmittedReview(
|
||||
makeToolState({
|
||||
selectedMode: "Review",
|
||||
review: { id: 1, nodeId: "n", reviewedSha: undefined },
|
||||
})
|
||||
)
|
||||
).toBeNull();
|
||||
});
|
||||
|
||||
it("does not flip a successful run to failed when reflection returns success:false", async () => {
|
||||
// the reflection turn is a best-effort nudge (update_learnings). if it
|
||||
// fails — e.g. the model API errors mid-turn — the underlying task has
|
||||
// already completed and been gated cleanly, so the run as a whole must
|
||||
// still be reported as successful.
|
||||
const initial = successResult({ output: "task done" });
|
||||
const resume = vi
|
||||
.fn<(ctx: { prompt: string; previousResult: AgentResult }) => Promise<AgentResult>>()
|
||||
.mockResolvedValue({ success: false, error: "model API transient failure" });
|
||||
|
||||
const result = await runPostRunRetryLoop({
|
||||
initialResult: initial,
|
||||
initialUsage: undefined,
|
||||
stopScript: null,
|
||||
resume,
|
||||
reflectionPrompt: "REFLECTION: call update_learnings if anything is worth saving",
|
||||
});
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.output).toBe("task done");
|
||||
expect(result.error).toBeUndefined();
|
||||
expect(resume).toHaveBeenCalledTimes(1);
|
||||
expect(resume.mock.calls[0]?.[0].prompt).toMatch(/REFLECTION/);
|
||||
it("fires for Review even when report_progress wrote a final summary", () => {
|
||||
// Review's only valid exit is `create_pull_request_review`. a summary
|
||||
// comment is not a substitute, and accepting it here previously let
|
||||
// subagent-flipped `finalSummaryWritten` silence the gate.
|
||||
expect(
|
||||
getUnsubmittedReview(makeToolState({ selectedMode: "Review", finalSummaryWritten: true }))
|
||||
).toBe("Review");
|
||||
});
|
||||
|
||||
it("still aggregates usage from a failed reflection turn", async () => {
|
||||
// the reflection consumed tokens even if it didn't produce useful output;
|
||||
// the run total must reflect that so billing/reporting stays accurate.
|
||||
const initial = successResult({
|
||||
usage: { agent: "claude", inputTokens: 100, outputTokens: 50 },
|
||||
});
|
||||
const resume = vi
|
||||
.fn<(ctx: { prompt: string; previousResult: AgentResult }) => Promise<AgentResult>>()
|
||||
.mockResolvedValue({
|
||||
success: false,
|
||||
error: "model API transient failure",
|
||||
usage: { agent: "claude", inputTokens: 10, outputTokens: 5 },
|
||||
});
|
||||
|
||||
const result = await runPostRunRetryLoop({
|
||||
initialResult: initial,
|
||||
initialUsage: initial.usage,
|
||||
stopScript: null,
|
||||
resume,
|
||||
reflectionPrompt: "reflect",
|
||||
});
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.usage?.inputTokens).toBe(110);
|
||||
expect(result.usage?.outputTokens).toBe(55);
|
||||
it("returns null for IncrementalReview when report_progress wrote a final summary", () => {
|
||||
// IncrementalReview treats `report_progress` as a legitimate
|
||||
// "no review warranted" exit, matching the post-failure error message.
|
||||
expect(
|
||||
getUnsubmittedReview(
|
||||
makeToolState({ selectedMode: "IncrementalReview", finalSummaryWritten: true })
|
||||
)
|
||||
).toBeNull();
|
||||
});
|
||||
|
||||
it("falls back to the reflection's output when the pre-reflection output is empty", async () => {
|
||||
// the preservation fix must only kick in when the task actually produced
|
||||
// meaningful output. runs that communicate exclusively through MCP tools
|
||||
// (e.g. report_progress) leave result.output = "" — using `??` here kept
|
||||
// the empty string and dropped the reflection's reply, leaving the
|
||||
// fallback `handleAgentResult` path with nothing to show. prefer the
|
||||
// reflection's output (even a trivial "done") over no output at all.
|
||||
const initial = successResult({ output: "" });
|
||||
const resume = vi
|
||||
.fn<(ctx: { prompt: string; previousResult: AgentResult }) => Promise<AgentResult>>()
|
||||
.mockResolvedValue(successResult({ output: "done" }));
|
||||
|
||||
const result = await runPostRunRetryLoop({
|
||||
initialResult: initial,
|
||||
initialUsage: undefined,
|
||||
stopScript: null,
|
||||
resume,
|
||||
reflectionPrompt: "REFLECTION: consider update_learnings",
|
||||
});
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.output).toBe("done");
|
||||
it("returns null when there is no progress comment to anchor the failure to", () => {
|
||||
expect(
|
||||
getUnsubmittedReview(makeToolState({ selectedMode: "Review", hadProgressComment: false }))
|
||||
).toBeNull();
|
||||
});
|
||||
|
||||
it("preserves the pre-reflection task output when a trivial reflection ('done') succeeds", async () => {
|
||||
// the reflection turn is a meta-ask — its literal reply ("done" or a
|
||||
// short "updated learnings with N bullets") is not the task summary the
|
||||
// caller wants to see. before this fix, `result = reflectionResult`
|
||||
// clobbered the task's output on the returned AgentResult, so downstream
|
||||
// consumers (handleAgentResult's fallback path when toolState is empty,
|
||||
// programmatic callers of main()) saw "done" instead of the real
|
||||
// summary. assert the task's output survives a successful reflection.
|
||||
const initial = successResult({ output: "Implemented feature X; tests pass; pushed PR #42" });
|
||||
const resume = vi
|
||||
.fn<(ctx: { prompt: string; previousResult: AgentResult }) => Promise<AgentResult>>()
|
||||
.mockResolvedValue(successResult({ output: "done" }));
|
||||
|
||||
const result = await runPostRunRetryLoop({
|
||||
initialResult: initial,
|
||||
initialUsage: undefined,
|
||||
stopScript: null,
|
||||
resume,
|
||||
reflectionPrompt: "REFLECTION: consider update_learnings",
|
||||
});
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.output).toBe("Implemented feature X; tests pass; pushed PR #42");
|
||||
});
|
||||
|
||||
it("skips reflection entirely when canResume returns false", async () => {
|
||||
const initial = successResult();
|
||||
const resume = vi
|
||||
.fn<(ctx: { prompt: string; previousResult: AgentResult }) => Promise<AgentResult>>()
|
||||
.mockResolvedValue(successResult());
|
||||
|
||||
const result = await runPostRunRetryLoop({
|
||||
initialResult: initial,
|
||||
initialUsage: undefined,
|
||||
stopScript: null,
|
||||
resume,
|
||||
canResume: () => false,
|
||||
reflectionPrompt: "reflect",
|
||||
});
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(resume).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("catches a reflection turn that dirties the tree via the dirty-tree gate on the next iteration", async () => {
|
||||
// PR claims: "if the reflection turn dirties the tree, the loop picks
|
||||
// that up on the next iteration via the normal dirty-tree gate." lock
|
||||
// it in — without this invariant the reflection prompt could bypass the
|
||||
// commit-before-you-finish contract whenever the agent misbehaves.
|
||||
//
|
||||
// three getGitStatus calls in sequence:
|
||||
// 1. clean (triggers reflection)
|
||||
// 2. reflection left the tree dirty
|
||||
// 3. retry committed the changes — now clean, loop exits
|
||||
mockedGetGitStatus
|
||||
.mockReturnValueOnce("")
|
||||
.mockReturnValueOnce(" M scratch/notes.md")
|
||||
.mockReturnValueOnce("");
|
||||
|
||||
const initial = successResult();
|
||||
const resume = vi
|
||||
.fn<(ctx: { prompt: string; previousResult: AgentResult }) => Promise<AgentResult>>()
|
||||
.mockResolvedValue(successResult({ output: "resumed" }));
|
||||
|
||||
const result = await runPostRunRetryLoop({
|
||||
initialResult: initial,
|
||||
initialUsage: undefined,
|
||||
stopScript: null,
|
||||
resume,
|
||||
reflectionPrompt: "REFLECTION: consider update_learnings",
|
||||
});
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
// call 0: reflection; call 1: dirty-tree retry
|
||||
expect(resume).toHaveBeenCalledTimes(2);
|
||||
expect(resume.mock.calls[0]?.[0].prompt).toContain("REFLECTION");
|
||||
expect(resume.mock.calls[1]?.[0].prompt).toContain("UNCOMMITTED CHANGES");
|
||||
expect(resume.mock.calls[1]?.[0].prompt).toContain("scratch/notes.md");
|
||||
});
|
||||
|
||||
it("surfaces a persistent stop hook failure as AgentResult.error after MAX_POST_RUN_RETRIES", async () => {
|
||||
// PR test plan item #1: "confirm the agent is resumed with the hook
|
||||
// output and the run fails after 3 attempts if never resolved."
|
||||
//
|
||||
// stop the hook from passing on every invocation, have `resume` always
|
||||
// return success (the agent tried but couldn't fix the issue), and
|
||||
// verify: (a) the loop exhausts all retries, (b) the final result is
|
||||
// success=false, (c) the error mentions the retry count and the hook
|
||||
// output verbatim so the GitHub comment surfaces what actually failed.
|
||||
const hookFailure = {
|
||||
stdout: "lint: 3 issues in src/foo.ts",
|
||||
stderr: "",
|
||||
exitCode: 7,
|
||||
durationMs: 5,
|
||||
};
|
||||
mockedSpawn.mockResolvedValue(hookFailure);
|
||||
|
||||
const initial = successResult({ output: "agent done" });
|
||||
const resume = vi
|
||||
.fn<(ctx: { prompt: string; previousResult: AgentResult }) => Promise<AgentResult>>()
|
||||
.mockResolvedValue(successResult({ output: "retry done" }));
|
||||
|
||||
const result = await runPostRunRetryLoop({
|
||||
initialResult: initial,
|
||||
initialUsage: undefined,
|
||||
stopScript: "pnpm lint",
|
||||
resume,
|
||||
reflectionPrompt: undefined,
|
||||
});
|
||||
|
||||
expect(result.success).toBe(false);
|
||||
expect(result.error).toContain("stop hook failed");
|
||||
expect(result.error).toContain("exit code 7");
|
||||
expect(result.error).toContain("3 retry attempts");
|
||||
expect(result.error).toContain("lint: 3 issues in src/foo.ts");
|
||||
// each retry feeds the hook output back into the agent as the resume prompt
|
||||
expect(resume).toHaveBeenCalledTimes(3);
|
||||
for (const call of resume.mock.calls) {
|
||||
expect(call[0].prompt).toContain("STOP HOOK FAILED");
|
||||
expect(call[0].prompt).toContain("lint: 3 issues in src/foo.ts");
|
||||
}
|
||||
});
|
||||
|
||||
it("treats a persistently dirty tree (no stop hook failure) as a soft-fail", async () => {
|
||||
// the PR documents: "dirty-tree-only failures preserve prior behavior:
|
||||
// they're logged but don't fail the run." a regression that started
|
||||
// surfacing dirty-tree as AgentResult.error would make every run that
|
||||
// leaves untracked test fixtures around fail spuriously. guard it.
|
||||
mockedGetGitStatus.mockReturnValue(" M src/foo.ts");
|
||||
const initial = successResult();
|
||||
const resume = vi
|
||||
.fn<(ctx: { prompt: string; previousResult: AgentResult }) => Promise<AgentResult>>()
|
||||
.mockResolvedValue(successResult({ output: "tried but tree still dirty" }));
|
||||
|
||||
const result = await runPostRunRetryLoop({
|
||||
initialResult: initial,
|
||||
initialUsage: undefined,
|
||||
stopScript: null,
|
||||
resume,
|
||||
});
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.error).toBeUndefined();
|
||||
// retries were attempted (the loop fed the dirty-tree prompt back to the agent)
|
||||
expect(resume).toHaveBeenCalledTimes(3);
|
||||
for (const call of resume.mock.calls) {
|
||||
expect(call[0].prompt).toContain("UNCOMMITTED CHANGES");
|
||||
}
|
||||
});
|
||||
|
||||
it("surfaces a stop hook failure even when canResume is false (no retry budget, still fails the run)", async () => {
|
||||
// the retry loop is best-effort. when canResume says no (e.g. claude
|
||||
// without a sessionId), we still need the failure gate to fire so the
|
||||
// user sees WHY the run failed instead of an opaque success. covers the
|
||||
// "checks still ran even if we can't resume" comment in postRun.ts.
|
||||
mockedSpawn.mockResolvedValue({
|
||||
stdout: "typecheck: 2 errors",
|
||||
stderr: "",
|
||||
exitCode: 1,
|
||||
durationMs: 1,
|
||||
});
|
||||
const initial = successResult();
|
||||
const resume = vi
|
||||
.fn<(ctx: { prompt: string; previousResult: AgentResult }) => Promise<AgentResult>>()
|
||||
.mockResolvedValue(successResult());
|
||||
|
||||
const result = await runPostRunRetryLoop({
|
||||
initialResult: initial,
|
||||
initialUsage: undefined,
|
||||
stopScript: "pnpm typecheck",
|
||||
resume,
|
||||
canResume: () => false,
|
||||
});
|
||||
|
||||
expect(result.success).toBe(false);
|
||||
expect(result.error).toContain("stop hook failed");
|
||||
expect(result.error).toContain("typecheck: 2 errors");
|
||||
// no retries were attempted because canResume said no — error lists no
|
||||
// retry count (that would be a lie).
|
||||
expect(result.error).not.toContain("retry attempt");
|
||||
expect(resume).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("short-circuits the loop when the initial result is already failed", async () => {
|
||||
// if the agent already failed (timeout, model error) there's no point
|
||||
// running gates or a reflection — the run is toast. preserve the original
|
||||
// error verbatim so triage is straightforward.
|
||||
const initial: AgentResult = {
|
||||
success: false,
|
||||
error: "agent died mid-turn",
|
||||
output: "partial",
|
||||
};
|
||||
const resume = vi
|
||||
.fn<(ctx: { prompt: string; previousResult: AgentResult }) => Promise<AgentResult>>()
|
||||
.mockResolvedValue(successResult());
|
||||
|
||||
const result = await runPostRunRetryLoop({
|
||||
initialResult: initial,
|
||||
initialUsage: undefined,
|
||||
stopScript: "pnpm lint",
|
||||
resume,
|
||||
reflectionPrompt: "reflect",
|
||||
});
|
||||
|
||||
expect(result.success).toBe(false);
|
||||
expect(result.error).toBe("agent died mid-turn");
|
||||
expect(resume).not.toHaveBeenCalled();
|
||||
expect(mockedSpawn).not.toHaveBeenCalled();
|
||||
expect(mockedGetGitStatus).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("aggregates usage across every gate retry", async () => {
|
||||
// billing/reporting rely on the usage summary reflecting the full run,
|
||||
// not just the final retry's slice. regression gate.
|
||||
mockedSpawn.mockResolvedValue({
|
||||
stdout: "fail",
|
||||
stderr: "",
|
||||
exitCode: 1,
|
||||
durationMs: 1,
|
||||
});
|
||||
const initial = successResult({
|
||||
usage: { agent: "claude", inputTokens: 100, outputTokens: 50 },
|
||||
});
|
||||
const resume = vi
|
||||
.fn<(ctx: { prompt: string; previousResult: AgentResult }) => Promise<AgentResult>>()
|
||||
.mockResolvedValue({
|
||||
success: true,
|
||||
output: "retry",
|
||||
usage: { agent: "claude", inputTokens: 10, outputTokens: 5 },
|
||||
});
|
||||
|
||||
const result = await runPostRunRetryLoop({
|
||||
initialResult: initial,
|
||||
initialUsage: initial.usage,
|
||||
stopScript: "flaky",
|
||||
resume,
|
||||
});
|
||||
|
||||
// 100 initial + 10 * 3 retries = 130
|
||||
expect(result.usage?.inputTokens).toBe(130);
|
||||
expect(result.usage?.outputTokens).toBe(65);
|
||||
});
|
||||
});
|
||||
|
||||
describe("executeStopHook — output capture", () => {
|
||||
beforeEach(() => {
|
||||
mockedSpawn.mockReset();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it("includes both stdout and stderr in the failure output when both are populated", async () => {
|
||||
// hooks that wrap other tools commonly emit a benign warning to stderr
|
||||
// (e.g. "config file not found, using defaults") and the actionable error
|
||||
// to stdout. a `(stderr || stdout)` heuristic drops stdout entirely
|
||||
// whenever stderr is non-empty, starving the agent of the information it
|
||||
// needs to fix the issue.
|
||||
mockedSpawn.mockResolvedValue({
|
||||
stdout: "ERROR: lint check failed at path/to/file.ts:42",
|
||||
stderr: "Warning: config file not found, using defaults",
|
||||
exitCode: 1,
|
||||
durationMs: 5,
|
||||
});
|
||||
const failure = await executeStopHook("run-lint");
|
||||
expect(failure).not.toBeNull();
|
||||
expect(failure?.output).toContain("ERROR: lint check failed at path/to/file.ts:42");
|
||||
expect(failure?.output).toContain("Warning: config file not found, using defaults");
|
||||
});
|
||||
|
||||
it("returns null (treated as passed) when spawn throws a timeout", async () => {
|
||||
// infra-level failures can't be fixed by the agent. surfacing them as a
|
||||
// hook failure would put the loop into a retry cycle that never
|
||||
// terminates. soft-fail and let the run succeed.
|
||||
mockedSpawn.mockRejectedValue(
|
||||
new SpawnTimeoutError("hook exceeded 10 minutes", SPAWN_TIMEOUT_CODE)
|
||||
it("returns the selected mode when the gate should fire", () => {
|
||||
expect(getUnsubmittedReview(makeToolState({ selectedMode: "Review" }))).toBe("Review");
|
||||
expect(getUnsubmittedReview(makeToolState({ selectedMode: "IncrementalReview" }))).toBe(
|
||||
"IncrementalReview"
|
||||
);
|
||||
const failure = await executeStopHook("slow-hook");
|
||||
expect(failure).toBeNull();
|
||||
});
|
||||
|
||||
it("returns null (treated as passed) on spawn ENOENT (command not found)", async () => {
|
||||
// if the user misconfigures the hook (wrong binary, typo), the spawn
|
||||
// itself throws. same rationale as timeouts: soft-fail, don't retry.
|
||||
mockedSpawn.mockRejectedValue(
|
||||
Object.assign(new Error("spawn nosuchbin ENOENT"), { code: "ENOENT" })
|
||||
);
|
||||
const failure = await executeStopHook("nosuchbin");
|
||||
expect(failure).toBeNull();
|
||||
});
|
||||
|
||||
it("truncates oversize output, keeping the tail", async () => {
|
||||
// the error is embedded in AgentResult.error and flows into GitHub
|
||||
// comments (65535-char cap). the 4096-char truncation is our guardrail;
|
||||
// lock it in so a well-meaning refactor can't blow the comment budget.
|
||||
const longTail = "LAST_LINE_IS_ACTIONABLE";
|
||||
const longOutput = "x".repeat(10_000) + longTail;
|
||||
mockedSpawn.mockResolvedValue({
|
||||
stdout: longOutput,
|
||||
stderr: "",
|
||||
exitCode: 1,
|
||||
durationMs: 1,
|
||||
});
|
||||
const failure = await executeStopHook("noisy");
|
||||
expect(failure?.output).toContain(longTail);
|
||||
expect(failure?.output).toContain("truncated");
|
||||
expect(failure?.output.length).toBeLessThan(longOutput.length);
|
||||
});
|
||||
});
|
||||
|
||||
+207
-59
@@ -1,6 +1,7 @@
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { type AgentId, formatMcpToolRef } from "../external.ts";
|
||||
import { LIFECYCLE_HOOK_TIMEOUT_MS } from "../lifecycle.ts";
|
||||
import { NON_COMMITTING_MODES } from "../modes.ts";
|
||||
import type { ToolState } from "../toolState.ts";
|
||||
import { log } from "../utils/cli.ts";
|
||||
import {
|
||||
SPAWN_ACTIVITY_TIMEOUT_CODE,
|
||||
@@ -10,6 +11,7 @@ import {
|
||||
} from "../utils/subprocess.ts";
|
||||
import {
|
||||
type AgentResult,
|
||||
type AgentRunContext,
|
||||
type AgentUsage,
|
||||
buildCommitPrompt,
|
||||
getGitStatus,
|
||||
@@ -20,6 +22,36 @@ import {
|
||||
type StopHookFailure,
|
||||
} from "./shared.ts";
|
||||
|
||||
/**
|
||||
* derive "agent picked a review mode but never produced visible output" from
|
||||
* the literal facts on `toolState`. returns the selected mode when the gate
|
||||
* should fire, `null` otherwise — pure read, no side effects, safe to invoke
|
||||
* after every agent attempt.
|
||||
*
|
||||
* the gate is anchored to `hadProgressComment` so silent runs (non-issue
|
||||
* events, dispatcher skipped seeding) don't fire a nudge there's no UI for.
|
||||
*
|
||||
* `Review` and `IncrementalReview` have different valid exits:
|
||||
* - Review: only `create_pull_request_review` counts. `report_progress` is
|
||||
* not a substitute — a Review run that exits with just a summary comment
|
||||
* has produced nothing reviewable on the PR. matches the hard-fail
|
||||
* message at `expected = "create_pull_request_review"` below.
|
||||
* - IncrementalReview: `report_progress` is a legitimate "no review
|
||||
* warranted" exit, so either toolState flag short-circuits.
|
||||
* splitting per mode also closes the bypass where a subagent (e.g. a
|
||||
* `task`-dispatched `reviewfrog` lens) calls `report_progress` and silences
|
||||
* the gate even though the orchestrator never submitted a review.
|
||||
*/
|
||||
export function getUnsubmittedReview(toolState: ToolState): "Review" | "IncrementalReview" | null {
|
||||
const mode = toolState.selectedMode;
|
||||
if (!toolState.hadProgressComment) return null;
|
||||
if (mode === "Review") return toolState.review ? null : "Review";
|
||||
if (mode === "IncrementalReview") {
|
||||
return toolState.review || toolState.finalSummaryWritten ? null : "IncrementalReview";
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* hook output can flow into two size-sensitive places: the LLM resume prompt
|
||||
* (context window) and AgentResult.error (surfaced in GitHub comments capped
|
||||
@@ -116,67 +148,173 @@ export function buildSummaryStalePrompt(filePath: string): string {
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
export function buildUnsubmittedReviewPrompt(mode: "Review" | "IncrementalReview"): string {
|
||||
// mode-aware: Review mode's contract is "always submit one review" — its
|
||||
// mode prompt forbids `report_progress`, so the nudge here must not offer
|
||||
// it as an exit. IncrementalReview legitimately allows a report_progress
|
||||
// exit when there are no new issues since the last review (mode prompt
|
||||
// step 8), so the nudge mirrors that contract.
|
||||
if (mode === "Review") {
|
||||
return [
|
||||
`MISSING REVIEW OUTPUT — you selected Review mode but stopped without calling \`create_pull_request_review\`. the user has no visible signal that this run produced anything; the progress comment will be deleted on exit and no review will appear on the PR.`,
|
||||
"",
|
||||
"call `create_pull_request_review` now with your aggregated review (body + inline comments). pick the tier per the mode prompt — Review mode has no no-submit exit, so even informational `> ✅ No new issues found.` reviews must be submitted (with `approved: true`). the first call may error once with a diff-coverage nudge — retry the same call to proceed.",
|
||||
"",
|
||||
"do NOT stop again until `create_pull_request_review` has been called successfully.",
|
||||
].join("\n");
|
||||
}
|
||||
return [
|
||||
`MISSING REVIEW OUTPUT — you selected IncrementalReview mode but stopped without calling \`create_pull_request_review\` or \`report_progress\`. the user has no visible signal that this run produced anything; the progress comment will be deleted on exit and no review will appear on the PR.`,
|
||||
"",
|
||||
"do exactly one of:",
|
||||
"- if you have findings: call `create_pull_request_review` now with your aggregated review (body + inline comments). the first call may error once with a diff-coverage nudge — retry the same call to proceed.",
|
||||
"- if there are genuinely no actionable findings since the last review (e.g. only formatting / comment / lockfile changes): call `report_progress` with a 1-2 sentence summary explaining that no review was warranted.",
|
||||
"",
|
||||
"do NOT stop again until one of those tools has been called successfully.",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
/**
|
||||
* check the post-run gates: did the stop hook pass, is the working tree
|
||||
* clean, and (when applicable) did the agent touch the rolling PR summary
|
||||
* snapshot? returns everything that still needs nudging so the caller can
|
||||
* render a single combined resume prompt.
|
||||
* snapshot or produce review output? returns everything that still needs
|
||||
* nudging so the caller can render a single combined resume prompt.
|
||||
*
|
||||
* the summary-stale check is skipped when `summaryFilePath` / `summarySeed`
|
||||
* are not provided; this is the common case (non-PR runs, runs where the
|
||||
* dispatcher didn't request snapshot generation, runs where the seed step
|
||||
* failed). loop callers also pass these as undefined after the agent has
|
||||
* already been nudged once, to avoid burning the retry budget on a soft
|
||||
* non-blocking gate.
|
||||
* reads run state directly off `ctx.toolState` so each invocation sees the
|
||||
* latest mutations from MCP tool calls. `skipSummaryStale` lets the loop
|
||||
* suppress the summary-stale check after the one-shot nudge has been
|
||||
* delivered (re-firing it would burn the retry budget on a soft gate the
|
||||
* agent has already decided not to act on).
|
||||
*/
|
||||
export async function collectPostRunIssues(params: {
|
||||
stopScript: string | null | undefined;
|
||||
summaryFilePath?: string | undefined;
|
||||
summarySeed?: string | undefined;
|
||||
}): Promise<PostRunIssues> {
|
||||
export async function collectPostRunIssues(
|
||||
ctx: AgentRunContext,
|
||||
options: { skipSummaryStale?: boolean } = {}
|
||||
): Promise<PostRunIssues> {
|
||||
const issues: PostRunIssues = {};
|
||||
if (params.stopScript) {
|
||||
const failure = await executeStopHook(params.stopScript);
|
||||
if (failure) issues.stopHook = failure;
|
||||
}
|
||||
// stop hook is disabled — production audit (May 2026) showed 8/9 configured
|
||||
// scripts are foot-guns (duplicates of prepushScript, run on non-committing
|
||||
// modes against unchanged trees) burning the retry budget on un-fixable
|
||||
// gates. re-enable here + the dashboard block in `AgentSettings.tsx` once
|
||||
// we've decided on the right semantics (mode-gating vs. HEAD-changed gating
|
||||
// vs. deletion). see issue #714.
|
||||
// if (ctx.stopScript) {
|
||||
// const failure = await executeStopHook(ctx.stopScript);
|
||||
// if (failure) issues.stopHook = failure;
|
||||
// }
|
||||
// dirty-tree gate fires only in modes that legitimately commit. Review /
|
||||
// IncrementalReview / Plan complete via review submission or a Plan
|
||||
// comment, not by touching files — any tree dirt is incidental (e.g. a
|
||||
// tool-installed `node_modules/`) and the worktree is ephemeral, so
|
||||
// nudging the agent to commit it would produce a spurious PR. see
|
||||
// `NON_COMMITTING_MODES` in `action/modes.ts`.
|
||||
const status = getGitStatus();
|
||||
if (status) issues.dirtyTree = status;
|
||||
if (params.summaryFilePath && params.summarySeed !== undefined) {
|
||||
const stale = await isSummaryUnchanged(params.summaryFilePath, params.summarySeed);
|
||||
if (stale) issues.summaryStale = { filePath: params.summaryFilePath };
|
||||
const mode = ctx.toolState.selectedMode;
|
||||
if (status) {
|
||||
if (mode && NON_COMMITTING_MODES.has(mode)) {
|
||||
log.info(`» dirty-tree gate suppressed: mode \`${mode}\` does not commit`);
|
||||
} else {
|
||||
issues.dirtyTree = status;
|
||||
}
|
||||
}
|
||||
const summaryFilePath = ctx.toolState.summaryFilePath;
|
||||
const summarySeed = ctx.toolState.summarySeed;
|
||||
if (!options.skipSummaryStale && summaryFilePath && summarySeed !== undefined) {
|
||||
const stale = await isSummaryUnchanged(summaryFilePath, summarySeed);
|
||||
if (stale) issues.summaryStale = { filePath: summaryFilePath };
|
||||
}
|
||||
const unsubmittedMode = getUnsubmittedReview(ctx.toolState);
|
||||
if (unsubmittedMode) issues.unsubmittedReview = unsubmittedMode;
|
||||
return issues;
|
||||
}
|
||||
|
||||
export function buildPostRunPrompt(issues: PostRunIssues): string {
|
||||
// order matches the terminal hard-fail order in `runPostRunRetryLoop` so
|
||||
// the prompt's emphasis (which gate the agent should fix first) lines up
|
||||
// with the user-visible failure message reported when retries exhaust.
|
||||
// both hard-fail gates first (`stopHook` → `unsubmittedReview`), then the
|
||||
// soft gates (`dirtyTree` → `summaryStale`).
|
||||
const parts: string[] = [];
|
||||
if (issues.stopHook) parts.push(buildStopHookPrompt(issues.stopHook));
|
||||
if (issues.unsubmittedReview) {
|
||||
parts.push(buildUnsubmittedReviewPrompt(issues.unsubmittedReview));
|
||||
}
|
||||
if (issues.dirtyTree) parts.push(buildCommitPrompt(issues.dirtyTree));
|
||||
if (issues.summaryStale) parts.push(buildSummaryStalePrompt(issues.summaryStale.filePath));
|
||||
return parts.join("\n\n---\n\n");
|
||||
}
|
||||
|
||||
/**
|
||||
* prompt for a dedicated post-run reflection turn nudging the agent to call
|
||||
* `update_learnings` if it discovered anything worth persisting.
|
||||
* modes for which the post-run reflection turn is skipped. reflection costs a
|
||||
* full resume turn (~$0.50-0.80 per run on Opus, mostly cache-write) and only
|
||||
* pays for itself when the run actually produced novel, durable findings.
|
||||
*
|
||||
* this exists because the learnings step baked into mode checklists is
|
||||
* frequently ignored — the agent stays focused on the task and the meta-ask
|
||||
* falls through. delivering it as its own resume turn, with nothing competing
|
||||
* for attention, raises the fire rate substantially.
|
||||
* `IncrementalReview` is the lowest-novelty mode — it's a tight delta review
|
||||
* against an existing PR with the prior summary already loaded as context.
|
||||
* the agent rarely discovers anything generalizable to next runs, so the
|
||||
* reflection turn is dead weight. initial `Review` still touches fresh PR
|
||||
* territory and benefits; `Build` / `Fix` / `AddressReviews` definitely do.
|
||||
*/
|
||||
export function buildLearningsReflectionPrompt(agentId: AgentId): string {
|
||||
const t = (name: string) => formatMcpToolRef(agentId, name);
|
||||
const REFLECTION_SKIP_MODES: ReadonlySet<string> = new Set(["IncrementalReview"]);
|
||||
|
||||
export function shouldRunReflection(mode: string | undefined): boolean {
|
||||
if (!mode) return true;
|
||||
return !REFLECTION_SKIP_MODES.has(mode);
|
||||
}
|
||||
|
||||
/**
|
||||
* prompt for a dedicated post-run reflection turn nudging the agent to edit
|
||||
* the rolling learnings file if it discovered anything worth persisting.
|
||||
*
|
||||
* this exists because passive "if you learned something, write it down"
|
||||
* instructions baked into mode checklists are frequently ignored — the agent
|
||||
* stays focused on the task and the meta-ask falls through. delivering it
|
||||
* as its own resume turn, with nothing competing for attention, raises the
|
||||
* fire rate substantially.
|
||||
*
|
||||
* the file is the single source of truth — there is no separate MCP tool
|
||||
* call. the server reads the file at end-of-run and persists any edits to
|
||||
* `Repo.learnings`.
|
||||
*
|
||||
* the prompt copy is shaped by repo-wide audits of the actual content the
|
||||
* agent has been writing (issue #619 in pullfrog/app). recurring failure
|
||||
* modes the framing pushes back on:
|
||||
* - massive multi-paragraph "bullets" that are really mini-articles
|
||||
* - facts anchored to moving repo state (PR / review / commit / branch
|
||||
* refs, dates, version pins, line numbers) that decay within weeks
|
||||
* - sections growing into giant flat lists with no internal structure,
|
||||
* forcing future runs to read kilobytes to find one fact
|
||||
*
|
||||
* single litmus delivered in the prompt: "would a future run on this repo
|
||||
* do its work better because this bullet exists?". tool-quirk workarounds
|
||||
* are explicitly allowed when the agent burned calls discovering the
|
||||
* quirk this run — recording the workaround prevents next run from
|
||||
* repeating the waste. tradeoff: the same quirk gets duplicated across
|
||||
* repos, so when a quirk is fixed upstream in tool descriptions the
|
||||
* per-repo bullets go stale and we have no batch-invalidation path.
|
||||
*/
|
||||
export function buildLearningsReflectionPrompt(filePath: string): string {
|
||||
return [
|
||||
`REFLECTION — before you finish, think back over this task: did you discover anything about this repo's setup, test commands, conventions, or patterns that you are confident is correct and would reliably help future runs?`,
|
||||
`REFLECTION — before you finish, think back over this task: did you discover anything about this repo's setup, test commands, conventions, or patterns that is high-confidence and would reliably help future runs?`,
|
||||
"",
|
||||
`if so, call \`${t("update_learnings")}\` to persist it.`,
|
||||
`the rolling learnings file is at \`${filePath}\`. read it first if you haven't already, then edit it in place using your native file tools. the server reads this file at end-of-run and persists any changes — there is no tool to call.`,
|
||||
"",
|
||||
`rules:`,
|
||||
`- only call \`${t("update_learnings")}\` when the finding is high-confidence and broadly useful. skip if unsure, speculative, or one-off.`,
|
||||
`- pass the FULL merged list: existing learnings from the original prompt + your new discoveries. one fact per bullet, lines starting with \`- \`.`,
|
||||
`- deduplicate, and drop bullets that are clearly wrong or no longer relevant to the current codebase.`,
|
||||
`- if you already called \`${t("update_learnings")}\` earlier in this run, or nothing new is worth capturing, just reply "done" and stop — do not edit the repo for this reflection.`,
|
||||
`structure:`,
|
||||
`- markdown hierarchy: \`## \` for top-level themes, \`### \` and deeper for sub-themes when a section grows. there is no fixed taxonomy — choose headings that fit THIS repo (e.g. for one repo \`## Migrations\` / \`## Local dev\` may make sense; for another, \`## API quirks\` / \`## Failure modes\`).`,
|
||||
`- **no section over ~300 lines.** when a section is approaching that, split it: introduce \`### \` subsections grouping related bullets, or hoist a coherent group into a new top-level \`## \` section. granular sections mean future runs read targeted line ranges instead of slurping the whole file. this is the most important hygiene rule on long-lived repos.`,
|
||||
`- if you find a flat unstructured list (legacy content from before this format), restructure it: read it, group related bullets, rewrite the file with \`## \` / \`### \` headings around them. don't preserve bad structure — fix it.`,
|
||||
"",
|
||||
`the only test: would a future run on this repo do its work better because this bullet exists? useful for future runs in this repo — prevent wasted tool calls, rabbit holes, and mistakes.`,
|
||||
"",
|
||||
`bullet hygiene:`,
|
||||
`- one fact per line starting with \`- \`, ≤ 240 chars.`,
|
||||
`- only add when high-confidence, broadly useful, evergreen.`,
|
||||
`- prune wrong or low-signal bullets; merge overlaps; dedupe across sections.`,
|
||||
"",
|
||||
`don't anchor facts to repo state that will move: PR / review / commit / branch refs, dates, version pins, line numbers. state the rule directly. if it needs the anchor to be load-bearing, it isn't evergreen.`,
|
||||
"",
|
||||
`tool-quirk bullets are fine when you burned calls discovering the quirk and a future run would repeat them. write the workaround, not the war story.`,
|
||||
"",
|
||||
`if you have nothing substantively new to add AND the existing entries still look healthy and well-structured, leave the file alone — just reply "done" and stop. silence is a valid outcome.`,
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
@@ -200,17 +338,9 @@ export function buildLearningsReflectionPrompt(agentId: AgentId): string {
|
||||
* behavior: they're logged but don't fail the run.
|
||||
*/
|
||||
export async function runPostRunRetryLoop<R extends AgentResult>(params: {
|
||||
ctx: AgentRunContext;
|
||||
initialResult: R;
|
||||
initialUsage: AgentUsage | undefined;
|
||||
stopScript: string | null | undefined;
|
||||
/** absolute path to the seeded PR summary file. when set together with
|
||||
* `summarySeed`, the loop checks after each agent attempt whether the
|
||||
* file has been edited; if not, it nudges the agent ONCE via a resume
|
||||
* turn (subsequent iterations skip the check so we don't keep burning
|
||||
* retries on a soft gate when the agent has decided no edit is warranted). */
|
||||
summaryFilePath?: string | undefined;
|
||||
/** exact bytes of the seeded summary file used for the unchanged-check. */
|
||||
summarySeed?: string | undefined;
|
||||
resume: (context: { prompt: string; previousResult: R }) => Promise<R>;
|
||||
canResume?: ((result: R) => boolean) | undefined;
|
||||
reflectionPrompt?: string | undefined;
|
||||
@@ -220,19 +350,16 @@ export async function runPostRunRetryLoop<R extends AgentResult>(params: {
|
||||
let finalIssues: PostRunIssues = {};
|
||||
let gateResumeCount = 0;
|
||||
let pendingReflection = params.reflectionPrompt;
|
||||
// nudge for an untouched summary file fires AT MOST ONCE per run. after
|
||||
// we've delivered the prompt, subsequent gate checks pass undefined so
|
||||
// the loop doesn't keep flagging the same condition — the agent may have
|
||||
// legitimately decided no edit is warranted, and re-prompting would
|
||||
// burn the retry budget without adding signal.
|
||||
// nudge for an untouched summary file fires AT MOST ONCE per run. once
|
||||
// delivered, subsequent collectPostRunIssues calls skip the check — the
|
||||
// agent may have legitimately decided no edit is warranted, and
|
||||
// re-prompting would burn the retry budget without adding signal.
|
||||
let summaryStaleNudged = false;
|
||||
|
||||
while (gateResumeCount < MAX_POST_RUN_RETRIES) {
|
||||
if (!result.success) break;
|
||||
const issues = await collectPostRunIssues({
|
||||
stopScript: params.stopScript,
|
||||
summaryFilePath: summaryStaleNudged ? undefined : params.summaryFilePath,
|
||||
summarySeed: summaryStaleNudged ? undefined : params.summarySeed,
|
||||
const issues = await collectPostRunIssues(params.ctx, {
|
||||
skipSummaryStale: summaryStaleNudged,
|
||||
});
|
||||
if (issues.summaryStale) summaryStaleNudged = true;
|
||||
finalIssues = issues;
|
||||
@@ -318,10 +445,11 @@ export async function runPostRunRetryLoop<R extends AgentResult>(params: {
|
||||
// false-positive failures right after it just passed.
|
||||
if (gateResumeCount > 0 && result.success && hasPostRunIssues(finalIssues)) {
|
||||
// re-check the gates that can actually fail the run (stop hook /
|
||||
// dirty tree). summary-stale is intentionally NOT re-checked here:
|
||||
// we already delivered the one-shot nudge, and a still-unchanged
|
||||
// file at this point is the agent's deliberate choice.
|
||||
finalIssues = await collectPostRunIssues({ stopScript: params.stopScript });
|
||||
// dirty tree / unsubmitted review). summary-stale is intentionally
|
||||
// NOT re-checked here: we already delivered the one-shot nudge, and
|
||||
// a still-unchanged file at this point is the agent's deliberate
|
||||
// choice.
|
||||
finalIssues = await collectPostRunIssues(params.ctx, { skipSummaryStale: true });
|
||||
}
|
||||
|
||||
if (result.success && finalIssues.stopHook) {
|
||||
@@ -337,5 +465,25 @@ export async function runPostRunRetryLoop<R extends AgentResult>(params: {
|
||||
};
|
||||
}
|
||||
|
||||
if (result.success && finalIssues.unsubmittedReview) {
|
||||
const retryNote =
|
||||
gateResumeCount > 0
|
||||
? ` after ${gateResumeCount} retry ${gateResumeCount === 1 ? "attempt" : "attempts"}`
|
||||
: "";
|
||||
// mode-aware: Review's contract requires a review submission; only
|
||||
// IncrementalReview accepts `report_progress` as an exit. mirroring
|
||||
// the nudge prompt avoids contradicting the agent-facing copy.
|
||||
const expected =
|
||||
finalIssues.unsubmittedReview === "Review"
|
||||
? "create_pull_request_review"
|
||||
: "create_pull_request_review or report_progress";
|
||||
return {
|
||||
...result,
|
||||
success: false,
|
||||
error: `${finalIssues.unsubmittedReview} mode finished without calling ${expected}${retryNote}`,
|
||||
usage: aggregatedUsage,
|
||||
};
|
||||
}
|
||||
|
||||
return { ...result, usage: aggregatedUsage };
|
||||
}
|
||||
|
||||
@@ -146,6 +146,40 @@ describe("SessionLabeler", () => {
|
||||
]);
|
||||
});
|
||||
|
||||
test("Claude path: parent_tool_use_id resolves directly without consuming FIFO", () => {
|
||||
// Claude runs subagents inside the orchestrator's session — they share
|
||||
// session_id — and stamps subagent messages with parent_tool_use_id.
|
||||
// recording dispatch with the Agent tool_use id binds it directly so
|
||||
// future events resolve regardless of session_id.
|
||||
const labeler = new SessionLabeler();
|
||||
expect(labeler.labelFor("shared-session", null)).toBe(ORCHESTRATOR_LABEL);
|
||||
|
||||
labeler.recordTaskDispatch({ description: "correctness" }, "toolu_01");
|
||||
labeler.recordTaskDispatch({ description: "security" }, "toolu_02");
|
||||
|
||||
// subagent events come through with shared session_id but distinct
|
||||
// parent_tool_use_id — direct mapping wins
|
||||
expect(labeler.labelFor("shared-session", "toolu_01")).toBe("lens:correctness");
|
||||
expect(labeler.labelFor("shared-session", "toolu_02")).toBe("lens:security");
|
||||
|
||||
// orchestrator events on the same session still resolve correctly
|
||||
expect(labeler.labelFor("shared-session", null)).toBe(ORCHESTRATOR_LABEL);
|
||||
|
||||
// pendingLabels is unused on the Claude path — FIFO never consumed
|
||||
expect(labeler.pendingDispatchCount()).toBe(2);
|
||||
expect(labeler.size()).toBe(1);
|
||||
});
|
||||
|
||||
test("Claude path: unknown parent_tool_use_id falls through to sessionID/FIFO logic", () => {
|
||||
// defensive: if a subagent event arrives with a parent_tool_use_id we
|
||||
// never recorded (e.g. orchestrator dispatched off-stream, or a tool we
|
||||
// didn't track), the labeler shouldn't crash — it should fall through
|
||||
// to the sessionID-keyed path.
|
||||
const labeler = new SessionLabeler();
|
||||
labeler.labelFor("shared", null);
|
||||
expect(labeler.labelFor("shared", "unknown-tool-id")).toBe(ORCHESTRATOR_LABEL);
|
||||
});
|
||||
|
||||
test("realistic four-lens parallel fan-out — interleaved tool_use stream", () => {
|
||||
// simulates the event order we'd see when the orchestrator dispatches
|
||||
// 4 lens subagents in a single assistant turn and they all start emitting
|
||||
|
||||
+48
-18
@@ -67,38 +67,68 @@ export function deriveLabelFromTaskInput(input: TaskDispatchInput): string {
|
||||
}
|
||||
|
||||
/**
|
||||
* Stateful tracker mapping sessionIDs to human labels.
|
||||
* Stateful tracker mapping subagent activity back to human-readable labels.
|
||||
*
|
||||
* Lifecycle:
|
||||
* - First call to `labelFor()` returns ORCHESTRATOR_LABEL and binds that
|
||||
* sessionID to it. Every subsequent event from that session gets the
|
||||
* same label.
|
||||
* - When the orchestrator emits a Task tool_use, the harness calls
|
||||
* `recordTaskDispatch()` to push the dispatch's derived label onto a
|
||||
* pending FIFO queue.
|
||||
* - The next previously-unseen sessionID consumes the head of the queue.
|
||||
* - If `labelFor()` is called for a new session with an empty queue
|
||||
* (e.g. a subagent emitted events before the parent's tool_use was
|
||||
* parsed, or the runtime spawned a session we didn't expect), the
|
||||
* labeler falls back to `subagent#N` so log lines remain attributable.
|
||||
* Two attribution channels are supported because the runtimes differ:
|
||||
*
|
||||
* - **OpenCode** spawns each subagent as its own opencode `Session` with
|
||||
* a distinct `sessionID`. The harness records each Task dispatch into a
|
||||
* pending FIFO queue; the next previously-unseen sessionID consumes the
|
||||
* head of the queue and binds it to that label.
|
||||
*
|
||||
* - **Claude Code** runs subagents inside the orchestrator's session — they
|
||||
* all share `session_id` — and instead stamps every subagent message with
|
||||
* `parent_tool_use_id` pointing at the Agent tool_use id that spawned them.
|
||||
* The harness binds each Agent tool_use id to its dispatched label up
|
||||
* front, then `labelFor` looks the label up directly when an event arrives
|
||||
* carrying that `parent_tool_use_id`.
|
||||
*
|
||||
* `labelFor(sessionID, parentToolUseId?)` accepts both: when
|
||||
* `parentToolUseId` is set and known it short-circuits to the direct mapping;
|
||||
* otherwise it falls through to the FIFO/sessionID path.
|
||||
*/
|
||||
export class SessionLabeler {
|
||||
private readonly labels = new Map<string, string>();
|
||||
private readonly labelsByToolUseId = new Map<string, string>();
|
||||
private readonly pendingLabels: string[] = [];
|
||||
private fallbackCounter = 0;
|
||||
|
||||
recordTaskDispatch(input: TaskDispatchInput): string {
|
||||
/**
|
||||
* Record a Task/Agent tool dispatch.
|
||||
*
|
||||
* @param input Task tool input — used to derive the lens label.
|
||||
* @param toolUseId Optional Agent tool_use id. When provided, future events
|
||||
* carrying `parent_tool_use_id === toolUseId` resolve
|
||||
* directly to this label without consuming the FIFO queue
|
||||
* (Claude path). Always also pushed to the FIFO queue so
|
||||
* the OpenCode path still works when toolUseId is absent.
|
||||
*/
|
||||
recordTaskDispatch(input: TaskDispatchInput, toolUseId?: string | null): string {
|
||||
const label = deriveLabelFromTaskInput(input);
|
||||
this.pendingLabels.push(label);
|
||||
if (toolUseId) this.labelsByToolUseId.set(toolUseId, label);
|
||||
return label;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return a label for the given sessionID. Binds on first call.
|
||||
* Pass undefined/empty for events that lack a session id — the caller
|
||||
* gets ORCHESTRATOR_LABEL so the line is still attributable.
|
||||
* Return a label for the given event.
|
||||
*
|
||||
* @param sessionID Session id from the event (OpenCode: per-session;
|
||||
* Claude: shared across orchestrator + subagents).
|
||||
* @param parentToolUseId Claude's `parent_tool_use_id` — non-null on
|
||||
* subagent messages. When set and known, takes
|
||||
* priority over the FIFO/sessionID path.
|
||||
*/
|
||||
labelFor(sessionID: string | undefined | null): string {
|
||||
labelFor(sessionID: string | undefined | null, parentToolUseId?: string | null): string {
|
||||
// Claude path: subagent messages carry parent_tool_use_id pointing at
|
||||
// the Agent tool_use that spawned them. resolve directly without
|
||||
// touching the sessionID-keyed map (which is bound to the orchestrator
|
||||
// for the shared session_id and would otherwise misattribute).
|
||||
if (parentToolUseId) {
|
||||
const direct = this.labelsByToolUseId.get(parentToolUseId);
|
||||
if (direct) return direct;
|
||||
}
|
||||
|
||||
if (!sessionID) return ORCHESTRATOR_LABEL;
|
||||
const existing = this.labels.get(sessionID);
|
||||
if (existing) return existing;
|
||||
|
||||
+37
-14
@@ -1,5 +1,6 @@
|
||||
import { execFileSync } from "node:child_process";
|
||||
import type { AgentId } from "../external.ts";
|
||||
import type { ToolState } from "../toolState.ts";
|
||||
import { log } from "../utils/cli.ts";
|
||||
import type { ResolvedInstructions } from "../utils/instructions.ts";
|
||||
import type { ResolvedPayload } from "../utils/payload.ts";
|
||||
@@ -54,13 +55,25 @@ export interface PostRunIssues {
|
||||
* seed, i.e. the agent never touched it. soft gate — nudges once via a
|
||||
* resume turn but never fails the run, parallel to dirtyTree semantics. */
|
||||
summaryStale?: SummaryStale;
|
||||
/**
|
||||
* populated when the agent selected a review mode but the post-run check
|
||||
* over toolState shows neither a `create_pull_request_review` submission
|
||||
* nor a final `report_progress` write happened. derived inline from
|
||||
* `toolState.selectedMode` + `toolState.review` + `toolState.finalSummaryWritten`
|
||||
* via {@link getUnsubmittedReview} — no parallel toolState flag is stored.
|
||||
* carries the mode name so the resume prompt can reference it. handled like
|
||||
* `stopHook`: nudge via resume, hard-fail if still unsatisfied after
|
||||
* `MAX_POST_RUN_RETRIES`.
|
||||
*/
|
||||
unsubmittedReview?: "Review" | "IncrementalReview";
|
||||
}
|
||||
|
||||
export function hasPostRunIssues(issues: PostRunIssues): boolean {
|
||||
return (
|
||||
issues.stopHook !== undefined ||
|
||||
issues.dirtyTree !== undefined ||
|
||||
issues.summaryStale !== undefined
|
||||
issues.summaryStale !== undefined ||
|
||||
issues.unsubmittedReview !== undefined
|
||||
);
|
||||
}
|
||||
|
||||
@@ -104,13 +117,22 @@ export interface AgentResult {
|
||||
}
|
||||
|
||||
/**
|
||||
* Minimal context passed to agent.run()
|
||||
* Context passed to agent.run() and threaded through the post-run loop.
|
||||
*
|
||||
* design rule: this is the single object that flows through the harness and
|
||||
* downstream utilities by reference. derived predicates (e.g.
|
||||
* `getUnsubmittedReview`), tmpfile paths, and seed bytes live on
|
||||
* `toolState` — read them at the call site, do not duplicate them onto this
|
||||
* interface. utilities that need run state should accept `ctx` whole, not
|
||||
* destructure a narrow subset.
|
||||
*/
|
||||
export interface AgentRunContext {
|
||||
payload: ResolvedPayload;
|
||||
resolvedModel?: string | undefined;
|
||||
mcpServerUrl: string;
|
||||
tmpdir: string;
|
||||
/** harness-owned secret paths that agent filesystem tools must never read. */
|
||||
secretDenyPaths?: string[] | undefined;
|
||||
instructions: ResolvedInstructions;
|
||||
todoTracker?: TodoTracker | undefined;
|
||||
/**
|
||||
@@ -120,19 +142,13 @@ export interface AgentRunContext {
|
||||
*/
|
||||
stopScript?: string | null | undefined;
|
||||
/**
|
||||
* absolute path to the rolling PR summary tmpfile, when one was seeded
|
||||
* for this run (Review / IncrementalReview / pr-summary Task). enables
|
||||
* a post-run sanity nudge that prompts the agent if the file is still
|
||||
* byte-identical to its seed.
|
||||
* mutable per-run state shared with the MCP server (by reference). post-run
|
||||
* gates read fresh values from it after each agent attempt — `summaryFilePath`,
|
||||
* `summarySeed`, `selectedMode`, `review`, `finalSummaryWritten`,
|
||||
* `hadProgressComment` are all consulted by `collectPostRunIssues`. see
|
||||
* `action/toolState.ts` for the literal-state design rule.
|
||||
*/
|
||||
summaryFilePath?: string | undefined;
|
||||
/**
|
||||
* exact bytes of the seeded summary file. compared against the current
|
||||
* file content after each agent attempt to detect "agent forgot to edit
|
||||
* the summary" — particularly common with smaller models that lose
|
||||
* track of multi-step instructions.
|
||||
*/
|
||||
summarySeed?: string | undefined;
|
||||
toolState: ToolState;
|
||||
/**
|
||||
* called synchronously when the agent subprocess is killed for inner
|
||||
* activity timeout. lets main.ts tear down shared resources (MCP HTTP
|
||||
@@ -140,6 +156,13 @@ export interface AgentRunContext {
|
||||
*/
|
||||
onActivityTimeout?: (() => void) | undefined;
|
||||
onToolUse?: ((event: AgentToolUseEvent) => void) | undefined;
|
||||
/**
|
||||
* Pullfrog API JWT scoped to this run. agents only need this when they
|
||||
* have to write state back to Pullfrog mid-run (today: opencode.ts uses
|
||||
* it to seed the post-hook's writeback envelope for Codex auth refresh).
|
||||
* empty string when the run wasn't context-resolved (e.g. local dry-runs).
|
||||
*/
|
||||
apiToken: string;
|
||||
}
|
||||
|
||||
export interface Agent {
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { deriveSubagentModels } from "./subagentModels.ts";
|
||||
|
||||
describe("deriveSubagentModels", () => {
|
||||
it("returns no override when orchestrator is undefined", () => {
|
||||
expect(deriveSubagentModels(undefined)).toEqual({ reviewer: undefined });
|
||||
});
|
||||
|
||||
it("returns no override when orchestrator slug isn't registered", () => {
|
||||
expect(deriveSubagentModels("nonexistent/model")).toEqual({ reviewer: undefined });
|
||||
});
|
||||
|
||||
describe("anthropic family — opus → sonnet", () => {
|
||||
it("direct anthropic opus", () => {
|
||||
expect(deriveSubagentModels("anthropic/claude-opus-4-7")).toEqual({
|
||||
reviewer: "anthropic/claude-sonnet-4-6",
|
||||
});
|
||||
});
|
||||
it("opencode-vendored opus stays on opencode prefix", () => {
|
||||
expect(deriveSubagentModels("opencode/claude-opus-4-7")).toEqual({
|
||||
reviewer: "opencode/claude-sonnet-4-6",
|
||||
});
|
||||
});
|
||||
it("openrouter-anthropic-opus-via-anthropic-direct hits anthropic alias's openRouterResolve", () => {
|
||||
// both the anthropic alias and the opencode alias have the same
|
||||
// openRouterResolve. first-match-wins by alias declaration order
|
||||
// (anthropic declared first in providers).
|
||||
expect(deriveSubagentModels("openrouter/anthropic/claude-opus-4.7")).toEqual({
|
||||
reviewer: "openrouter/anthropic/claude-sonnet-4.6",
|
||||
});
|
||||
});
|
||||
it("sonnet has no further downshift", () => {
|
||||
expect(deriveSubagentModels("anthropic/claude-sonnet-4-6")).toEqual({ reviewer: undefined });
|
||||
expect(deriveSubagentModels("opencode/claude-sonnet-4-6")).toEqual({ reviewer: undefined });
|
||||
});
|
||||
it("haiku has no downshift", () => {
|
||||
expect(deriveSubagentModels("anthropic/claude-haiku-4-5")).toEqual({ reviewer: undefined });
|
||||
});
|
||||
});
|
||||
|
||||
describe("openai family", () => {
|
||||
it("gpt-pro → gpt (direct)", () => {
|
||||
expect(deriveSubagentModels("openai/gpt-5.5-pro")).toEqual({ reviewer: "openai/gpt-5.5" });
|
||||
});
|
||||
it("gpt → gpt-5.4 (direct)", () => {
|
||||
expect(deriveSubagentModels("openai/gpt-5.5")).toEqual({ reviewer: "openai/gpt-5.4" });
|
||||
});
|
||||
it("gpt → gpt-5.4 (opencode-vendored)", () => {
|
||||
expect(deriveSubagentModels("opencode/gpt-5.5")).toEqual({ reviewer: "opencode/gpt-5.4" });
|
||||
});
|
||||
it("gpt-pro → gpt (openrouter)", () => {
|
||||
expect(deriveSubagentModels("openrouter/openai/gpt-5.5-pro")).toEqual({
|
||||
reviewer: "openrouter/openai/gpt-5.5",
|
||||
});
|
||||
});
|
||||
it("gpt → gpt-5.4 (openrouter)", () => {
|
||||
expect(deriveSubagentModels("openrouter/openai/gpt-5.5")).toEqual({
|
||||
reviewer: "openrouter/openai/gpt-5.4",
|
||||
});
|
||||
});
|
||||
it("gpt-5.4 itself (the hidden subagent target) has no further downshift", () => {
|
||||
expect(deriveSubagentModels("openai/gpt-5.4")).toEqual({ reviewer: undefined });
|
||||
});
|
||||
it("gpt-mini has no downshift", () => {
|
||||
expect(deriveSubagentModels("openai/gpt-5.4-mini")).toEqual({ reviewer: undefined });
|
||||
});
|
||||
});
|
||||
|
||||
describe("google (gemini) — inherit (Pro for both orchestrator and lenses)", () => {
|
||||
// pro → flash was a meaningful capability cliff (Flash missed catastrophic
|
||||
// cross-file bugs the v4 e2e test surfaced); Pro is cost-effective enough
|
||||
// to keep on for lenses too. Google has no in-between tier.
|
||||
it("direct google pro inherits", () => {
|
||||
expect(deriveSubagentModels("google/gemini-3.1-pro-preview")).toEqual({
|
||||
reviewer: undefined,
|
||||
});
|
||||
});
|
||||
it("opencode-vendored gemini-pro inherits", () => {
|
||||
expect(deriveSubagentModels("opencode/gemini-3.1-pro")).toEqual({
|
||||
reviewer: undefined,
|
||||
});
|
||||
});
|
||||
it("openrouter gemini-pro inherits", () => {
|
||||
expect(deriveSubagentModels("openrouter/google/gemini-3.1-pro-preview")).toEqual({
|
||||
reviewer: undefined,
|
||||
});
|
||||
});
|
||||
it("flash has no downshift", () => {
|
||||
expect(deriveSubagentModels("google/gemini-3-flash-preview")).toEqual({
|
||||
reviewer: undefined,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("providers / models without a subagentModel — inherit", () => {
|
||||
it("xai grok (already cheap flagship)", () => {
|
||||
expect(deriveSubagentModels("xai/grok-4.3")).toEqual({ reviewer: undefined });
|
||||
});
|
||||
it("deepseek", () => {
|
||||
expect(deriveSubagentModels("deepseek/deepseek-v4-pro")).toEqual({ reviewer: undefined });
|
||||
});
|
||||
it("moonshot kimi", () => {
|
||||
expect(deriveSubagentModels("moonshotai/kimi-k2.6")).toEqual({ reviewer: undefined });
|
||||
});
|
||||
it("opencode big-pickle", () => {
|
||||
expect(deriveSubagentModels("opencode/big-pickle")).toEqual({ reviewer: undefined });
|
||||
});
|
||||
it("legacy fallback aliases (gpt-codex, deepseek-reasoner)", () => {
|
||||
expect(deriveSubagentModels("openai/gpt-5.3-codex")).toEqual({ reviewer: undefined });
|
||||
expect(deriveSubagentModels("deepseek/deepseek-reasoner")).toEqual({ reviewer: undefined });
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,40 @@
|
||||
import { modelAliases } from "../models.ts";
|
||||
|
||||
/**
|
||||
* Derive a cheaper subagent model override from the orchestrator's resolved
|
||||
* model spec.
|
||||
*
|
||||
* This is a pure registry lookup: every alias in `action/models.ts` declares
|
||||
* its own `subagentModel` (alias key in the same provider). At runtime we
|
||||
* reverse-lookup the orchestrator's resolved slug to find the alias that
|
||||
* produced it, follow the `subagentModel` pointer, and return the target
|
||||
* alias's resolve / openRouterResolve depending on which route the
|
||||
* orchestrator was using.
|
||||
*
|
||||
* Returns `{ reviewer: undefined }` when the orchestrator's alias has no
|
||||
* `subagentModel` (e.g. it's already at a sufficiently cheap tier, or its
|
||||
* provider doesn't have a clean cheaper-but-capable sibling). See models.ts
|
||||
* for the wiring + per-provider rationale.
|
||||
*/
|
||||
export function deriveSubagentModels(orchestratorSpec: string | undefined): {
|
||||
reviewer: string | undefined;
|
||||
} {
|
||||
if (!orchestratorSpec) return { reviewer: undefined };
|
||||
|
||||
// Reverse-lookup. The same resolve string appears in only one alias
|
||||
// (within its provider), so first match wins. We track which field
|
||||
// matched (resolve vs openRouterResolve) so we can pick the same field
|
||||
// off the subagent target — keeping the orchestrator's route consistent.
|
||||
for (const source of modelAliases) {
|
||||
const matchedDirect = source.resolve === orchestratorSpec;
|
||||
const matchedOR = source.openRouterResolve === orchestratorSpec;
|
||||
if (!matchedDirect && !matchedOR) continue;
|
||||
if (!source.subagentModel) return { reviewer: undefined };
|
||||
const target = modelAliases.find((a) => a.slug === source.subagentModel);
|
||||
if (!target) return { reviewer: undefined };
|
||||
const reviewer = matchedOR ? target.openRouterResolve : target.resolve;
|
||||
return { reviewer };
|
||||
}
|
||||
|
||||
return { reviewer: undefined };
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
const claudeSource = readFileSync(join(__dirname, "claude.ts"), "utf-8");
|
||||
const opencodeSharedSource = readFileSync(join(__dirname, "opencodeShared.ts"), "utf-8");
|
||||
const opencodeV2Source = readFileSync(join(__dirname, "opencode_v2.ts"), "utf-8");
|
||||
|
||||
/**
|
||||
* The Claude Code `--agents` JSON and OpenCode `agent` config block are the
|
||||
* only places where per-subagent model overrides take effect. They're built
|
||||
* by string-only helpers we don't export, so this test reads the source and
|
||||
* asserts the literal model strings + agent names are wired in. A regression
|
||||
* here means the next review run silently runs lenses on Opus instead of
|
||||
* Sonnet.
|
||||
*/
|
||||
describe("subagent registration source asserts", () => {
|
||||
describe("claude.ts buildAgentsJson", () => {
|
||||
it("registers reviewfrog with sonnet model", () => {
|
||||
expect(claudeSource).toMatch(
|
||||
/\[REVIEWER_AGENT_NAME\]:\s*\{[^}]*model:\s*"claude-sonnet-4-6"/s
|
||||
);
|
||||
});
|
||||
it("imports the reviewer name constant", () => {
|
||||
expect(claudeSource).toMatch(/REVIEWER_AGENT_NAME/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("opencodeShared.ts buildReviewerAgentConfig", () => {
|
||||
it("registers reviewfrog with mode: subagent", () => {
|
||||
expect(opencodeSharedSource).toMatch(/\[REVIEWER_AGENT_NAME\]:[^}]*mode:\s*"subagent"/s);
|
||||
});
|
||||
it("uses deriveSubagentModels for the reviewer model override", () => {
|
||||
expect(opencodeSharedSource).toMatch(/deriveSubagentModels\(/);
|
||||
expect(opencodeSharedSource).toMatch(/overrides\.reviewer/);
|
||||
});
|
||||
it("v2 runner passes orchestrator model to buildReviewerAgentConfig", () => {
|
||||
expect(opencodeV2Source).toMatch(/buildReviewerAgentConfig\(model\)/);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,6 +1,7 @@
|
||||
import { basename } from "node:path";
|
||||
import arg from "arg";
|
||||
import pc from "picocolors";
|
||||
import { runCli as runAuthCli } from "./commands/auth.ts";
|
||||
import { runCli as runGhaCli } from "./commands/gha.ts";
|
||||
import { runCli as runInitCli } from "./commands/init.ts";
|
||||
|
||||
@@ -13,6 +14,7 @@ function printMainUsage(stream: typeof console.log): void {
|
||||
stream(`usage: ${PROG} <command>\n`);
|
||||
stream("commands:");
|
||||
stream(" init set up pullfrog on the current repository");
|
||||
stream(" auth manage provider credentials for the current repository");
|
||||
stream("");
|
||||
stream("global options:");
|
||||
stream(" -h, --help show help");
|
||||
@@ -85,6 +87,15 @@ async function run(): Promise<void> {
|
||||
return;
|
||||
}
|
||||
|
||||
if (command === "auth") {
|
||||
await runAuthCli({
|
||||
args: commandArgs,
|
||||
prog: PROG,
|
||||
showHelp: globalParsed["--help"] === true,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (globalParsed["--help"]) {
|
||||
printMainUsage(console.log);
|
||||
process.exit(0);
|
||||
|
||||
@@ -0,0 +1,229 @@
|
||||
// shared helpers used by `init` and `auth` subcommands. these were originally
|
||||
// inlined in `init.ts`; pulled out so `auth.ts` can reuse them without
|
||||
// duplicating gh-auth/pullfrog-api/secret-save logic.
|
||||
|
||||
import { execFileSync } from "node:child_process";
|
||||
import * as p from "@clack/prompts";
|
||||
import pc from "picocolors";
|
||||
|
||||
export const PULLFROG_API_URL = (process.env.PULLFROG_API_URL || "https://pullfrog.com").replace(
|
||||
/\/+$/,
|
||||
""
|
||||
);
|
||||
|
||||
// active spinner reference so bail/cancel can stop it before exiting. shared
|
||||
// across init/auth subcommands via this module's singleton scope; whichever
|
||||
// command starts a spinner sets this so handleCancel/bail can clean up.
|
||||
let activeSpin: ReturnType<typeof p.spinner> | null = null;
|
||||
|
||||
export function setActiveSpin(spin: ReturnType<typeof p.spinner> | null): void {
|
||||
activeSpin = spin;
|
||||
}
|
||||
|
||||
export function bail(msg: string): never {
|
||||
if (activeSpin) {
|
||||
activeSpin.stop(pc.red("failed"));
|
||||
activeSpin = null;
|
||||
}
|
||||
p.cancel(msg);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
export function handleCancel<T>(value: T | symbol): asserts value is T {
|
||||
if (p.isCancel(value)) {
|
||||
if (activeSpin) {
|
||||
activeSpin.stop(pc.red("canceled."));
|
||||
activeSpin = null;
|
||||
}
|
||||
p.cancel("canceled.");
|
||||
process.exit(0);
|
||||
}
|
||||
}
|
||||
|
||||
export function getGhToken(): string {
|
||||
let token: string;
|
||||
try {
|
||||
token = execFileSync("gh", ["auth", "token"], { encoding: "utf-8" }).trim();
|
||||
} catch {
|
||||
bail(
|
||||
`gh cli not found or not authenticated.\n` +
|
||||
` ${pc.dim("install:")} https://cli.github.com\n` +
|
||||
` ${pc.dim("then:")} gh auth login`
|
||||
);
|
||||
}
|
||||
if (!token) {
|
||||
bail(
|
||||
`gh cli returned an empty token. try re-authenticating:\n` +
|
||||
` ${pc.dim("run:")} gh auth login`
|
||||
);
|
||||
}
|
||||
return token;
|
||||
}
|
||||
|
||||
export function parseGitRemote(): { owner: string; repo: string } {
|
||||
let url: string;
|
||||
try {
|
||||
url = execFileSync("git", ["remote", "get-url", "origin"], { encoding: "utf-8" }).trim();
|
||||
} catch {
|
||||
bail("not a git repository or no 'origin' remote found.");
|
||||
}
|
||||
|
||||
const match = url.match(/github\.com(?::\d+)?[:/]+([^/]+)\/(.+?)(?:\.git)?(?:\/)?$/);
|
||||
if (!match) bail(`could not parse github owner/repo from remote: ${url}`);
|
||||
return { owner: match[1], repo: match[2] };
|
||||
}
|
||||
|
||||
// ── Pullfrog API ──
|
||||
|
||||
type SecretsApiData = {
|
||||
error?: string;
|
||||
appSlug?: string;
|
||||
installationId?: number | null;
|
||||
repositorySelection?: string | null;
|
||||
isOrg?: boolean;
|
||||
accessible?: boolean;
|
||||
repoSecrets?: string[];
|
||||
orgSecrets?: string[];
|
||||
pullfrogSecrets?: string[];
|
||||
repoStatus?: string | null;
|
||||
repoModel?: string | null;
|
||||
hasRuns?: boolean;
|
||||
};
|
||||
|
||||
type SecretsInfo = {
|
||||
isOrg: boolean;
|
||||
installationId: number | null;
|
||||
secretsAccessible: boolean;
|
||||
repoSecrets: string[];
|
||||
orgSecrets: string[];
|
||||
pullfrogSecrets: string[];
|
||||
model: string | null;
|
||||
hasRuns: boolean;
|
||||
};
|
||||
|
||||
type InstallationNotFound = {
|
||||
appSlug: string;
|
||||
installationId: number | null;
|
||||
repositorySelection: "all" | "selected" | null;
|
||||
isOrg: boolean;
|
||||
};
|
||||
|
||||
type StatusResult =
|
||||
| ({ installed: true } & SecretsInfo)
|
||||
| ({ installed: false } & InstallationNotFound);
|
||||
|
||||
type ApiResult<T = Record<string, unknown>> = {
|
||||
ok: boolean;
|
||||
status: number;
|
||||
data: T;
|
||||
};
|
||||
|
||||
async function pullfrogApi<T = Record<string, unknown>>(ctx: {
|
||||
path: string;
|
||||
token: string;
|
||||
method?: string;
|
||||
body?: Record<string, unknown>;
|
||||
}): Promise<ApiResult<T>> {
|
||||
const headers: Record<string, string> = { authorization: `Bearer ${ctx.token}` };
|
||||
if (ctx.body) headers["content-type"] = "application/json";
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), 30_000);
|
||||
try {
|
||||
const response = await fetch(`${PULLFROG_API_URL}${ctx.path}`, {
|
||||
method: ctx.method || "GET",
|
||||
headers,
|
||||
body: ctx.body ? JSON.stringify(ctx.body) : null,
|
||||
signal: controller.signal,
|
||||
});
|
||||
const data = (await response.json().catch(() => ({}))) as T;
|
||||
return { ok: response.ok, status: response.status, data };
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
|
||||
export async function fetchStatus(ctx: {
|
||||
token: string;
|
||||
owner: string;
|
||||
repo: string;
|
||||
}): Promise<StatusResult> {
|
||||
const result = await pullfrogApi<SecretsApiData>({
|
||||
path: `/api/cli/secrets?owner=${encodeURIComponent(ctx.owner)}&repo=${encodeURIComponent(ctx.repo)}`,
|
||||
token: ctx.token,
|
||||
});
|
||||
|
||||
if (!result.ok) {
|
||||
const errorMsg = result.data.error || "";
|
||||
if (result.status === 401) bail("invalid or expired github token.");
|
||||
if (result.status === 404) {
|
||||
const sel = result.data.repositorySelection;
|
||||
if (!result.data.appSlug) bail("server did not return appSlug");
|
||||
return {
|
||||
installed: false,
|
||||
appSlug: result.data.appSlug,
|
||||
installationId:
|
||||
typeof result.data.installationId === "number" ? result.data.installationId : null,
|
||||
repositorySelection: sel === "all" || sel === "selected" ? sel : null,
|
||||
isOrg: result.data.isOrg === true,
|
||||
};
|
||||
}
|
||||
bail(errorMsg || `secrets check failed (${result.status})`);
|
||||
}
|
||||
|
||||
return {
|
||||
installed: true,
|
||||
isOrg: result.data.isOrg === true,
|
||||
installationId:
|
||||
typeof result.data.installationId === "number" ? result.data.installationId : null,
|
||||
secretsAccessible: result.data.accessible !== false,
|
||||
repoSecrets: result.data.repoSecrets || [],
|
||||
orgSecrets: result.data.orgSecrets || [],
|
||||
pullfrogSecrets: result.data.pullfrogSecrets || [],
|
||||
model: result.data.repoModel ?? null,
|
||||
hasRuns: result.data.hasRuns === true,
|
||||
};
|
||||
}
|
||||
|
||||
// ── secret save ──
|
||||
|
||||
export type SecretScope = "account" | "repo";
|
||||
|
||||
type PullfrogSecretResult = { saved: boolean; error: string };
|
||||
|
||||
export async function setPullfrogSecret(ctx: {
|
||||
token: string;
|
||||
owner: string;
|
||||
repo: string;
|
||||
name: string;
|
||||
value: string;
|
||||
scope: SecretScope;
|
||||
}): Promise<PullfrogSecretResult> {
|
||||
const result = await pullfrogApi<{ success?: boolean; error?: string }>({
|
||||
path: "/api/cli/secrets",
|
||||
token: ctx.token,
|
||||
method: "POST",
|
||||
body: {
|
||||
owner: ctx.owner,
|
||||
repo: ctx.repo,
|
||||
name: ctx.name,
|
||||
value: ctx.value,
|
||||
scope: ctx.scope,
|
||||
},
|
||||
});
|
||||
if (result.ok && result.data.success === true) {
|
||||
return { saved: true, error: "" };
|
||||
}
|
||||
return { saved: false, error: result.data.error || `api returned ${result.status}` };
|
||||
}
|
||||
|
||||
export async function promptScope(ctx: { owner: string; repo: string }): Promise<SecretScope> {
|
||||
const scope = await p.select<SecretScope>({
|
||||
message: "secret scope",
|
||||
options: [
|
||||
{ value: "account", label: `${ctx.owner} organization`, hint: "shared across repos" },
|
||||
{ value: "repo", label: `${ctx.owner}/${ctx.repo} only` },
|
||||
],
|
||||
});
|
||||
handleCancel(scope);
|
||||
return scope;
|
||||
}
|
||||
@@ -0,0 +1,324 @@
|
||||
// `pullfrog auth <provider>` — manage credentials for a configured repo
|
||||
// without going through the full `init` flow. currently supports:
|
||||
//
|
||||
// pullfrog auth codex mint a Codex subscription credential and save it
|
||||
// as the `CODEX_AUTH_JSON` Pullfrog secret
|
||||
//
|
||||
// the `codex` subcommand runs `codex login --device-auth` against an
|
||||
// isolated `CODEX_HOME` (so the user's existing ~/.codex/auth.json is never
|
||||
// touched), validates the resulting auth.json, and posts it to the Pullfrog
|
||||
// secrets API. used both for first-time setup of a Codex subscription on a
|
||||
// repo and for rotating a stale credential.
|
||||
|
||||
import { spawn } from "node:child_process";
|
||||
import * as p from "@clack/prompts";
|
||||
import arg from "arg";
|
||||
import pc from "picocolors";
|
||||
import { mintCodexAuth, refreshCodexAuth } from "../utils/codexAuth.ts";
|
||||
import {
|
||||
bail,
|
||||
fetchStatus,
|
||||
getGhToken,
|
||||
handleCancel,
|
||||
PULLFROG_API_URL,
|
||||
parseGitRemote,
|
||||
promptScope,
|
||||
setActiveSpin,
|
||||
setPullfrogSecret,
|
||||
} from "./_shared.ts";
|
||||
|
||||
const CODEX_AUTH_SECRET = "CODEX_AUTH_JSON";
|
||||
|
||||
/** strip CSI ANSI escapes (color, cursor) from a string so callers can re-style
|
||||
* the visible text without inheriting the source's formatting. covers what
|
||||
* Codex emits during device auth (mostly `\x1b[<digits>m` color codes).
|
||||
*/
|
||||
function stripAnsi(s: string): string {
|
||||
// biome-ignore lint/suspicious/noControlCharactersInRegex: ANSI escapes are control chars by design
|
||||
return s.replace(/\x1b\[[0-9;]*[a-zA-Z]/g, "");
|
||||
}
|
||||
|
||||
/** matches the Codex device-auth verification URL printed by `codex login
|
||||
* --device-auth`. captures the full URL (with query string) up to whitespace.
|
||||
*/
|
||||
const CODEX_DEVICE_URL_RE = /https:\/\/auth\.openai\.com\/codex\/device\S*/;
|
||||
|
||||
/** best-effort cross-platform "open URL in default browser". swallows
|
||||
* spawn errors and non-zero exits — the user can always copy-paste the URL
|
||||
* Codex already printed. on Linux, falls back to `wslview` when `xdg-open`
|
||||
* is missing (covers WSL where xdg-open isn't installed by default).
|
||||
*/
|
||||
function openInBrowser(url: string): void {
|
||||
const platform = process.platform;
|
||||
let cmd: string;
|
||||
let args: string[];
|
||||
if (platform === "darwin") {
|
||||
cmd = "open";
|
||||
args = [url];
|
||||
} else if (platform === "win32") {
|
||||
// `start` is a cmd.exe builtin. the empty "" is the window title
|
||||
// (required when the next argument is quoted, which happens for
|
||||
// URLs with `&`).
|
||||
cmd = "cmd.exe";
|
||||
args = ["/c", "start", "", url];
|
||||
} else {
|
||||
cmd = "xdg-open";
|
||||
args = [url];
|
||||
}
|
||||
const child = spawn(cmd, args, { stdio: "ignore", detached: true });
|
||||
child.on("error", () => {
|
||||
if (platform !== "linux") return;
|
||||
const fallback = spawn("wslview", [url], { stdio: "ignore", detached: true });
|
||||
fallback.on("error", () => {});
|
||||
fallback.unref();
|
||||
});
|
||||
child.unref();
|
||||
}
|
||||
|
||||
interface AuthCliParams {
|
||||
args: string[];
|
||||
prog: string;
|
||||
showHelp?: boolean;
|
||||
}
|
||||
|
||||
function printAuthUsage(params: { stream: typeof console.log; prog: string }): void {
|
||||
params.stream(`usage: ${params.prog} auth <provider>\n`);
|
||||
params.stream("manage provider credentials for the current repository.");
|
||||
params.stream("");
|
||||
params.stream("providers:");
|
||||
params.stream(" codex mint a Codex (ChatGPT) subscription credential");
|
||||
params.stream("");
|
||||
params.stream("options:");
|
||||
params.stream(" -h, --help show help");
|
||||
}
|
||||
|
||||
function printCodexUsage(params: { stream: typeof console.log; prog: string }): void {
|
||||
params.stream(`usage: ${params.prog} auth codex [options]\n`);
|
||||
params.stream("mint a Codex subscription credential and save it as CODEX_AUTH_JSON.");
|
||||
params.stream("");
|
||||
params.stream("options:");
|
||||
params.stream(" -h, --help show help");
|
||||
}
|
||||
|
||||
export async function runCli(params: AuthCliParams): Promise<void> {
|
||||
// route `auth --help` (no subcommand) to top-level usage. when the user
|
||||
// passes `auth codex --help`, we leave the flag in the rest args so the
|
||||
// subcommand's own parser handles it.
|
||||
const firstArg = params.args[0];
|
||||
const helpAtTopLevel =
|
||||
params.showHelp ||
|
||||
params.args.length === 0 ||
|
||||
(params.args.length === 1 && (firstArg === "--help" || firstArg === "-h"));
|
||||
if (helpAtTopLevel) {
|
||||
printAuthUsage({ stream: console.log, prog: params.prog });
|
||||
return;
|
||||
}
|
||||
|
||||
const subcommand = firstArg;
|
||||
const rest = params.args.slice(1);
|
||||
|
||||
if (subcommand === "codex") {
|
||||
await runCodex({ args: rest, prog: params.prog });
|
||||
return;
|
||||
}
|
||||
|
||||
console.error(`unknown auth provider: ${pc.bold(subcommand)}\n`);
|
||||
printAuthUsage({ stream: console.error, prog: params.prog });
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
interface CodexCliParams {
|
||||
args: string[];
|
||||
prog: string;
|
||||
}
|
||||
|
||||
function parseCodexArgs(args: string[]) {
|
||||
return arg(
|
||||
{
|
||||
"--help": Boolean,
|
||||
"-h": "--help",
|
||||
},
|
||||
{ argv: args }
|
||||
);
|
||||
}
|
||||
|
||||
async function runCodex(params: CodexCliParams): Promise<void> {
|
||||
let parsed: ReturnType<typeof parseCodexArgs>;
|
||||
try {
|
||||
parsed = parseCodexArgs(params.args);
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
console.error(`${message}\n`);
|
||||
printCodexUsage({ stream: console.error, prog: params.prog });
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
if (parsed["--help"]) {
|
||||
printCodexUsage({ stream: console.log, prog: params.prog });
|
||||
return;
|
||||
}
|
||||
|
||||
await runCodexAuth();
|
||||
}
|
||||
|
||||
async function runCodexAuth(): Promise<void> {
|
||||
p.intro(pc.bgGreen(pc.black(" pullfrog auth codex ")));
|
||||
|
||||
const spin = p.spinner();
|
||||
setActiveSpin(spin);
|
||||
|
||||
try {
|
||||
spin.start("authenticating with github");
|
||||
const token = getGhToken();
|
||||
spin.stop("github authenticated");
|
||||
|
||||
spin.start("detecting repository");
|
||||
const remote = parseGitRemote();
|
||||
spin.stop(`detected repo ${pc.cyan(`${remote.owner}/${remote.repo}`)}`);
|
||||
|
||||
spin.start("checking pullfrog app installation");
|
||||
const status = await fetchStatus({ token, owner: remote.owner, repo: remote.repo });
|
||||
if (!status.installed) {
|
||||
spin.stop(pc.red("pullfrog app not installed on this repo"));
|
||||
bail(
|
||||
`install pullfrog on ${pc.bold(`${remote.owner}/${remote.repo}`)} before configuring auth.\n` +
|
||||
` ${pc.dim("run:")} ${pc.cyan(`npx pullfrog init`)}`
|
||||
);
|
||||
}
|
||||
spin.stop(`pullfrog app is installed on ${pc.cyan(`@${remote.owner}`)}`);
|
||||
|
||||
if (status.pullfrogSecrets.includes(CODEX_AUTH_SECRET)) {
|
||||
const overwrite = await p.select({
|
||||
message: `${pc.cyan(CODEX_AUTH_SECRET)} is already configured — overwrite?`,
|
||||
options: [
|
||||
{ value: true, label: "overwrite", hint: "rotate to a freshly minted credential" },
|
||||
{ value: false, label: "cancel" },
|
||||
],
|
||||
});
|
||||
handleCancel(overwrite);
|
||||
if (!overwrite) {
|
||||
p.cancel("canceled.");
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// user-owned repos can only ever be "account" (Pullfrog has no per-repo
|
||||
// store for user accounts), so we never bother prompting. on org-owned
|
||||
// repos, prompt interactively — matches `init`'s behavior.
|
||||
const scope = status.isOrg
|
||||
? await promptScope({ owner: remote.owner, repo: remote.repo })
|
||||
: "account";
|
||||
|
||||
p.log.info(
|
||||
[
|
||||
`signing in via Codex device authorization. open the URL Codex prints`,
|
||||
`below, enter the one-time code, and approve in your browser.`,
|
||||
``,
|
||||
`${pc.dim("note:")} if your ChatGPT account doesn't have device-code auth enabled,`,
|
||||
`Codex will exit early. enable it at ${pc.cyan(`https://chatgpt.com/#settings/Security`)}`,
|
||||
`then re-run ${pc.cyan(`${process.env.PULLFROG_BIN_NAME || "pullfrog"} auth codex`)}.`,
|
||||
].join("\n")
|
||||
);
|
||||
|
||||
// tracks the most recent exit so the retry prompt can tell the user
|
||||
// *why* no auth.json was written (timeout vs. early-exit).
|
||||
let lastTimedOut = false;
|
||||
// gate so we don't re-launch the browser if Codex prints the URL
|
||||
// more than once (e.g. on a retry attempt within the same flow).
|
||||
let hasOpenedDeviceUrl = false;
|
||||
const auth = await mintCodexAuth({
|
||||
childStdio: "pipe",
|
||||
onChildLine: (line) => {
|
||||
// dim Codex's own colored output (URL/code in cyan, boilerplate in
|
||||
// gray) so the user reads it as sub-process noise, not Pullfrog's
|
||||
// own prompts. the rail char matches @clack/prompts so the column
|
||||
// reads as one continuous flow.
|
||||
const stripped = stripAnsi(line);
|
||||
process.stdout.write(`${pc.gray(p.S_BAR)} ${pc.dim(stripped)}\n`);
|
||||
if (hasOpenedDeviceUrl) return;
|
||||
const match = stripped.match(CODEX_DEVICE_URL_RE);
|
||||
if (!match) return;
|
||||
hasOpenedDeviceUrl = true;
|
||||
const url = match[0];
|
||||
openInBrowser(url);
|
||||
process.stdout.write(
|
||||
`${pc.gray(p.S_BAR)} ${pc.dim(`» opened ${url} in browser (paste manually if it didn't open)`)}\n`
|
||||
);
|
||||
},
|
||||
onProgress: (event) => {
|
||||
if (event.kind === "start") {
|
||||
lastTimedOut = false;
|
||||
if (event.attempt > 1) p.log.info(`retry attempt ${event.attempt}`);
|
||||
// shell-prompt style header so the user sees what Pullfrog is
|
||||
// about to spawn, with the rail to keep the visual column.
|
||||
process.stdout.write(`${pc.gray(p.S_BAR)}\n`);
|
||||
process.stdout.write(`${pc.gray(p.S_BAR)} $ codex login --device-auth\n`);
|
||||
}
|
||||
if (event.kind === "exit") {
|
||||
if (event.timedOut) lastTimedOut = true;
|
||||
// trailing blank rail so the next clack prompt isn't crammed
|
||||
// against the last codex output line.
|
||||
process.stdout.write(`${pc.gray(p.S_BAR)}\n`);
|
||||
}
|
||||
},
|
||||
shouldRetry: async () => {
|
||||
const message = lastTimedOut
|
||||
? "device authorization timed out — retry?"
|
||||
: "no auth.json was written — retry?";
|
||||
const retry = await p.select({
|
||||
message,
|
||||
options: [
|
||||
{ value: true, label: "retry", hint: "after enabling device-code auth" },
|
||||
{ value: false, label: "cancel" },
|
||||
],
|
||||
});
|
||||
handleCancel(retry);
|
||||
return retry;
|
||||
},
|
||||
});
|
||||
|
||||
// eager refresh: bump the OAuth chain once before persisting so the
|
||||
// saved token is one Pullfrog has used. otherwise the user's laptop's
|
||||
// codex CLI could refresh first and strand our copy.
|
||||
spin.start("refreshing token");
|
||||
let savable: typeof auth;
|
||||
try {
|
||||
savable = await refreshCodexAuth(auth);
|
||||
spin.stop("refreshed");
|
||||
} catch (err) {
|
||||
spin.stop(pc.yellow("refresh failed — saving minted token as-is"));
|
||||
p.log.warn(err instanceof Error ? err.message : String(err));
|
||||
savable = auth;
|
||||
}
|
||||
|
||||
spin.start(`saving ${pc.cyan(CODEX_AUTH_SECRET)} to Pullfrog`);
|
||||
const result = await setPullfrogSecret({
|
||||
token,
|
||||
owner: remote.owner,
|
||||
repo: remote.repo,
|
||||
name: CODEX_AUTH_SECRET,
|
||||
value: savable.json,
|
||||
scope,
|
||||
});
|
||||
if (!result.saved) {
|
||||
spin.stop(pc.red("could not save secret"));
|
||||
p.log.warn(
|
||||
`${result.error}\n ${pc.dim("set it manually at:")} ${PULLFROG_API_URL}/console/${remote.owner}`
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
spin.stop(`saved ${pc.cyan(CODEX_AUTH_SECRET)} to Pullfrog (${scope})`);
|
||||
|
||||
setActiveSpin(null);
|
||||
p.outro("done.");
|
||||
} catch (error) {
|
||||
// mirror what `bail` does: stop the spinner with a red "failed" glyph
|
||||
// before clearing it, otherwise an in-flight spinner keeps animating
|
||||
// above the error message we're about to print.
|
||||
spin.stop(pc.red("failed"));
|
||||
setActiveSpin(null);
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
p.log.error(message);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
+9
-2
@@ -22,9 +22,16 @@ type CliProvider = {
|
||||
|
||||
function buildProviders(): CliProvider[] {
|
||||
return Object.entries(providers)
|
||||
.filter(([key]) => key !== "opencode" && key !== "openrouter")
|
||||
.filter(([key]) => key !== "opencode" && key !== "openrouter" && key !== "bedrock")
|
||||
.map(([key, config]: [string, ProviderConfig]) => {
|
||||
const aliases = modelAliases.filter((a) => a.provider === key && !a.fallback);
|
||||
// bedrock requires multi-secret setup (auth + region + model id) that
|
||||
// doesn't fit the single-paste flow below — direct users to
|
||||
// https://docs.pullfrog.com/bedrock instead. revisit once the init flow
|
||||
// supports multi-value setup. `hidden` excludes internal-only subagent
|
||||
// targets (e.g. openai/gpt-5.4) per #710.
|
||||
const aliases = modelAliases.filter(
|
||||
(a) => a.provider === key && !a.fallback && !a.routing && !a.hidden
|
||||
);
|
||||
const recommended = aliases.find((a) => a.preferred);
|
||||
const sorted = [...aliases].sort((a, b) => {
|
||||
if (a.preferred && !b.preferred) return -1;
|
||||
|
||||
Executable
+63
@@ -0,0 +1,63 @@
|
||||
#!/bin/bash
|
||||
# entrypoint for the pullfrog GHA-like container (see Dockerfile).
|
||||
#
|
||||
# - remaps `testuser` to the host uid/gid so bind-mounted files keep correct
|
||||
# ownership after writes inside the container
|
||||
# - on linux hosts, copies host ssh keys into testuser's $HOME (darwin hosts
|
||||
# forward the ssh-agent socket instead, no copy needed)
|
||||
# - installs action workspace deps (volume-cached, ~1.5s warm)
|
||||
# - exec's the requested command as testuser; argv is preserved (no nested
|
||||
# `bash -c`, no shell quoting hazards)
|
||||
set -euo pipefail
|
||||
|
||||
HOST_UID="${HOST_UID:-1000}"
|
||||
HOST_GID="${HOST_GID:-1000}"
|
||||
|
||||
if [ "$HOST_UID" != "1000" ] || [ "$HOST_GID" != "1000" ]; then
|
||||
groupmod -g "$HOST_GID" testuser 2>/dev/null || true
|
||||
usermod -u "$HOST_UID" -g "$HOST_GID" testuser 2>/dev/null || true
|
||||
# chown top-level dirs only — recursive chown would fail on `:ro` bind
|
||||
# mounts (e.g. macOS known_hosts mounted directly into /tmp/home/.ssh).
|
||||
chown "$HOST_UID:$HOST_GID" /tmp/home /tmp/home/.config /tmp/home/.cache 2>/dev/null || true
|
||||
chown "$HOST_UID:$HOST_GID" /app /app/action /app/action/node_modules 2>/dev/null || true
|
||||
fi
|
||||
|
||||
# linux hosts: copy host ssh keys into testuser's $HOME (we own this dir,
|
||||
# safe to chown). darwin hosts forward the ssh-agent socket instead and
|
||||
# bind-mount known_hosts read-only — nothing to do here.
|
||||
if [ -d /tmp/.ssh-host ]; then
|
||||
mkdir -p /tmp/home/.ssh
|
||||
cp /tmp/.ssh-host/id_* /tmp/home/.ssh/ 2>/dev/null || true
|
||||
chmod 600 /tmp/home/.ssh/id_* 2>/dev/null || true
|
||||
ssh-keyscan -t ed25519,rsa github.com >> /tmp/home/.ssh/known_hosts 2>/dev/null || true
|
||||
chmod 644 /tmp/home/.ssh/known_hosts 2>/dev/null || true
|
||||
chown -R "$HOST_UID:$HOST_GID" /tmp/home/.ssh 2>/dev/null || true
|
||||
# set GIT_SSH_COMMAND if any private key got copied. don't pin a
|
||||
# specific key with -i — let ssh pick whatever's in /tmp/home/.ssh
|
||||
# (covers id_rsa, id_ed25519, id_ecdsa, etc.).
|
||||
if ls /tmp/home/.ssh/id_* 2>/dev/null | grep -qv '\.pub$'; then
|
||||
export GIT_SSH_COMMAND="ssh -o UserKnownHostsFile=/tmp/home/.ssh/known_hosts -o StrictHostKeyChecking=no"
|
||||
fi
|
||||
fi
|
||||
|
||||
# warm the volume-cached node_modules. frozen-lockfile + ignore-scripts keeps
|
||||
# this idempotent and fast (~1.5s when nothing changed).
|
||||
#
|
||||
# the lockfile lives IN the shared node_modules volume so concurrent
|
||||
# `pnpm docker` invocations (e.g. `pnpm play:docker` in one terminal and
|
||||
# `pnpm runtest:docker` in another) serialize their install instead of racing.
|
||||
# `flock -w 120` waits up to 2min before giving up — well under any
|
||||
# real-world install time but short enough to surface true deadlocks.
|
||||
mkdir -p /app/action/node_modules
|
||||
flock -w 120 /app/action/node_modules/.gha-install.lock \
|
||||
sudo -u testuser -E env HOME=/tmp/home \
|
||||
corepack pnpm install --frozen-lockfile --ignore-scripts >/dev/null
|
||||
|
||||
# `--shell` drops into an interactive bash for debugging the container.
|
||||
if [ "${1:-}" = "--shell" ]; then
|
||||
exec sudo -u testuser -E env HOME=/tmp/home bash
|
||||
fi
|
||||
|
||||
# exec the command as testuser, preserving env. argv passes through unchanged
|
||||
# — no `bash -c` nesting, no quoting required by callers.
|
||||
exec sudo -u testuser -E env HOME=/tmp/home "$@"
|
||||
@@ -0,0 +1,532 @@
|
||||
// run any node script inside the pullfrog local docker container that
|
||||
// mocks the GHA `ubuntu-24.04` runner environment. NOT a real GitHub
|
||||
// Actions runner — for the real thing, see `.github/workflows/*.yml`
|
||||
// and `action/commands/gha.ts` (the action's GHA entry point).
|
||||
//
|
||||
// usage:
|
||||
// pnpm docker <script> [args…] # run script in container
|
||||
// pnpm docker --shell # interactive bash (requires TTY)
|
||||
// pnpm docker --build [--no-cache] # force-rebuild image
|
||||
// pnpm docker --clean # prune orphan images/volumes
|
||||
// pnpm docker --doctor # versions of every baked tool
|
||||
//
|
||||
// the action's two main entrypoints default to the host (fast iteration).
|
||||
// `:docker` suffix wraps this script:
|
||||
// pnpm play [args…] # host (this is the fast default)
|
||||
// pnpm play:docker [args…] # === pnpm docker play.ts [args…]
|
||||
// pnpm runtest [filters…] # host
|
||||
// pnpm runtest:docker [filters…] # === pnpm docker test/run.ts [filters…]
|
||||
//
|
||||
// the container is a baked ubuntu:24.04 image (see Dockerfile) with the
|
||||
// same toolset as GHA `ubuntu-24.04` runners. host env passes through
|
||||
// verbatim — no allowlist. multi-line values (RSA keys) handled via -e
|
||||
// fallback; everything else flows through `--env-file` for cleanliness.
|
||||
//
|
||||
// host services are reachable at `host.docker.internal:<port>` (works on
|
||||
// both linux and macOS — see --add-host below).
|
||||
//
|
||||
// rebuild is content-hash gated on Dockerfile + docker-entrypoint.sh.
|
||||
//
|
||||
// design rationale + gaps: wiki/docker.md.
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import { existsSync, mkdirSync, readFileSync, unlinkSync, writeFileSync } from "node:fs";
|
||||
import { platform, tmpdir } from "node:os";
|
||||
import { dirname, join, resolve } from "node:path";
|
||||
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
import { config } from "dotenv";
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
const actionDir = __dirname;
|
||||
const repoRoot = join(actionDir, "..");
|
||||
|
||||
config({ path: join(actionDir, ".env") });
|
||||
config({ path: join(repoRoot, ".env") });
|
||||
|
||||
// host env vars that would actively conflict with the container's own
|
||||
// configuration (paths, identity, shell, and outer-CI workflow-run identifiers
|
||||
// that don't apply to whatever repo the harness is acting against). everything
|
||||
// else passes through.
|
||||
const HOST_ONLY_VARS = new Set([
|
||||
// paths / identity / shell — would clobber the container's testuser setup
|
||||
"PATH",
|
||||
"HOME",
|
||||
"USER",
|
||||
"LOGNAME",
|
||||
"SHELL",
|
||||
"PWD",
|
||||
"OLDPWD",
|
||||
"TMPDIR",
|
||||
"TMP",
|
||||
"TEMP",
|
||||
"DOCKER_HOST",
|
||||
"DOCKER_CONFIG",
|
||||
"_",
|
||||
"SHLVL",
|
||||
"PS1",
|
||||
"PS2",
|
||||
"TERM_PROGRAM",
|
||||
"TERM_PROGRAM_VERSION",
|
||||
"TERM_SESSION_ID",
|
||||
"__CF_USER_TEXT_ENCODING",
|
||||
"XPC_SERVICE_NAME",
|
||||
"XPC_FLAGS",
|
||||
"Apple_PubSub_Socket_Render",
|
||||
"COMMAND_MODE",
|
||||
"COLORTERM",
|
||||
"ITERM_PROFILE",
|
||||
"ITERM_SESSION_ID",
|
||||
// outer-CI workflow-run identifiers — when the test suite runs inside
|
||||
// pullfrog/app's CI, these refer to pullfrog/app's run, NOT the test repo
|
||||
// the harness is acting against (e.g. pullfrog/test-repo). Anything inside
|
||||
// the action that uses them as keys to look up state on the test repo (most
|
||||
// notably `resolveRun()`'s `actions.listJobsForWorkflowRun(...)` call) will
|
||||
// 404. Filtering them here means the action sees them as undefined and
|
||||
// skips the lookup, instead of misdirecting it. `GITHUB_REPOSITORY` and
|
||||
// `GITHUB_TOKEN` are NOT filtered — those are genuinely needed inside.
|
||||
"GITHUB_RUN_ID",
|
||||
"GITHUB_RUN_NUMBER",
|
||||
"GITHUB_RUN_ATTEMPT",
|
||||
"GITHUB_JOB",
|
||||
"GITHUB_WORKFLOW",
|
||||
"GITHUB_ACTION",
|
||||
"GITHUB_REF",
|
||||
"GITHUB_SHA",
|
||||
"GITHUB_HEAD_REF",
|
||||
"GITHUB_BASE_REF",
|
||||
"GITHUB_TRIGGERING_ACTOR",
|
||||
]);
|
||||
|
||||
type Args = {
|
||||
forceBuild: boolean;
|
||||
noCache: boolean;
|
||||
shell: boolean;
|
||||
clean: boolean;
|
||||
doctor: boolean;
|
||||
passthrough: string[];
|
||||
};
|
||||
|
||||
/**
|
||||
* parses docker-level flags up to (but not including) the first positional
|
||||
* argument. anything after the first positional, or after a literal `--`,
|
||||
* passes through verbatim to the inner script. this prevents
|
||||
* `pnpm docker test/run.ts --build` from intercepting `--build` as a
|
||||
* docker flag.
|
||||
*/
|
||||
function parseArgs(argv: string[]): Args {
|
||||
const out: Args = {
|
||||
forceBuild: false,
|
||||
noCache: false,
|
||||
shell: false,
|
||||
clean: false,
|
||||
doctor: false,
|
||||
passthrough: [],
|
||||
};
|
||||
let i = 0;
|
||||
while (i < argv.length) {
|
||||
const a = argv[i];
|
||||
if (a === "--") {
|
||||
out.passthrough.push(...argv.slice(i + 1));
|
||||
return out;
|
||||
}
|
||||
if (a === "--build") out.forceBuild = true;
|
||||
else if (a === "--no-cache") {
|
||||
out.forceBuild = true;
|
||||
out.noCache = true;
|
||||
} else if (a === "--shell") out.shell = true;
|
||||
else if (a === "--clean") out.clean = true;
|
||||
else if (a === "--doctor") out.doctor = true;
|
||||
else if (a === "--help" || a === "-h") {
|
||||
showHelp();
|
||||
process.exit(0);
|
||||
} else {
|
||||
// first positional — script name and everything after passes through.
|
||||
out.passthrough.push(...argv.slice(i));
|
||||
return out;
|
||||
}
|
||||
i++;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function showHelp(): void {
|
||||
process.stdout.write(`Usage: pnpm docker <script> [args…]
|
||||
pnpm docker --shell
|
||||
pnpm docker --build [--no-cache]
|
||||
pnpm docker --clean
|
||||
pnpm docker --doctor
|
||||
|
||||
Run a node script inside the pullfrog local docker container that mocks
|
||||
the GHA ubuntu-24.04 runner toolset (gh, jq, python3, sudo, +
|
||||
build-essential / wget / xz / file). Host env passes through verbatim.
|
||||
The host is reachable from inside the container at host.docker.internal
|
||||
(useful for scripts that hit your local dev server).
|
||||
|
||||
The action's two main entrypoints have host (fast) and docker variants:
|
||||
pnpm play [args…] # host — the fast default
|
||||
pnpm play:docker [args…] # === pnpm docker play.ts [args…]
|
||||
pnpm runtest [filters…] # host
|
||||
pnpm runtest:docker [filters…] # === pnpm docker test/run.ts [filters…]
|
||||
|
||||
Options:
|
||||
--build rebuild the current image (otherwise rebuilt automatically
|
||||
when Dockerfile or docker-entrypoint.sh content changes).
|
||||
on its own, builds and exits.
|
||||
--no-cache pair with --build to also bust docker's layer cache;
|
||||
useful when an apt mirror or base image changed.
|
||||
--shell drop into an interactive bash inside the container.
|
||||
requires a TTY.
|
||||
--clean prune orphaned pullfrog-docker:* images and node_modules
|
||||
volumes whose hash doesn't match the current Dockerfile.
|
||||
--doctor print version info for tools inside the container (node,
|
||||
pnpm, gh, jq, git, python3, ssh, …). useful for diagnosing
|
||||
"works in CI fails locally" or vice versa.
|
||||
-h, --help show this message.
|
||||
|
||||
Pass-through:
|
||||
Anything after the first positional argument (or after a literal \`--\`)
|
||||
goes to the inner script verbatim. so \`pnpm docker test/run.ts --build\`
|
||||
passes \`--build\` to test/run.ts, not to docker.
|
||||
|
||||
Examples:
|
||||
pnpm docker play.ts
|
||||
pnpm docker play.ts --raw '{"prompt":"hi"}'
|
||||
pnpm docker test/run.ts smoke
|
||||
pnpm docker --shell
|
||||
pnpm docker --build # build image, then exit
|
||||
pnpm docker --build --no-cache # rebuild from scratch
|
||||
pnpm docker --clean # reclaim disk from old image hashes
|
||||
pnpm docker --doctor # fidelity audit
|
||||
`);
|
||||
}
|
||||
|
||||
function ensureDocker(): void {
|
||||
if (platform() === "win32") {
|
||||
fail("pnpm docker is not supported on native windows. use wsl2.");
|
||||
}
|
||||
const probe = spawnSync("docker", ["info"], { stdio: "ignore" });
|
||||
if (probe.status !== 0) {
|
||||
fail("docker is not running. start docker desktop and retry.");
|
||||
}
|
||||
}
|
||||
|
||||
function fail(msg: string): never {
|
||||
process.stderr.write(`error: ${msg}\n`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
type ImageRef = { tag: string; volumeName: string };
|
||||
|
||||
function imageRefFor(ctx: { dockerfile: string; entrypoint: string }): ImageRef {
|
||||
const hash = createHash("sha256")
|
||||
.update(readFileSync(ctx.dockerfile))
|
||||
.update(readFileSync(ctx.entrypoint))
|
||||
.digest("hex")
|
||||
.slice(0, 12);
|
||||
return {
|
||||
tag: `pullfrog-docker:${hash}`,
|
||||
// version the volume by image hash so a stale node_modules cache from
|
||||
// an old image (e.g. different node major) can't poison a new image.
|
||||
volumeName: `pullfrog-docker-node-modules-${hash}`,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* remove pullfrog-docker:* images and pullfrog-docker-node-modules-* volumes
|
||||
* whose hash doesn't match the current Dockerfile + entrypoint. each
|
||||
* Dockerfile/entrypoint edit creates a fresh hash and orphans the prior
|
||||
* pair; without periodic cleanup these accumulate (~600MB image + ~200MB
|
||||
* node_modules each).
|
||||
*/
|
||||
function cleanOrphans(currentRef: ImageRef): void {
|
||||
const imgList = spawnSync("docker", ["image", "ls", "--format", "{{.Repository}}:{{.Tag}}"], {
|
||||
encoding: "utf8",
|
||||
});
|
||||
const images = (imgList.stdout ?? "")
|
||||
.split("\n")
|
||||
.filter((s) => s.startsWith("pullfrog-docker:") && s !== currentRef.tag);
|
||||
if (images.length > 0) {
|
||||
process.stderr.write(`» removing ${images.length} orphan image(s): ${images.join(", ")}\n`);
|
||||
spawnSync("docker", ["image", "rm", "-f", ...images], { stdio: "inherit" });
|
||||
}
|
||||
const volList = spawnSync("docker", ["volume", "ls", "-q"], { encoding: "utf8" });
|
||||
const volumes = (volList.stdout ?? "")
|
||||
.split("\n")
|
||||
.filter((s) => s.startsWith("pullfrog-docker-node-modules-") && s !== currentRef.volumeName);
|
||||
if (volumes.length > 0) {
|
||||
process.stderr.write(`» removing ${volumes.length} orphan volume(s): ${volumes.join(", ")}\n`);
|
||||
spawnSync("docker", ["volume", "rm", ...volumes], { stdio: "inherit" });
|
||||
}
|
||||
if (images.length === 0 && volumes.length === 0) {
|
||||
process.stderr.write("» no orphans to clean (all matching current image hash)\n");
|
||||
}
|
||||
}
|
||||
|
||||
function buildImageIfNeeded(ctx: {
|
||||
ref: ImageRef;
|
||||
force: boolean;
|
||||
noCache: boolean;
|
||||
dockerfile: string;
|
||||
}): void {
|
||||
if (!ctx.force) {
|
||||
const inspect = spawnSync("docker", ["image", "inspect", ctx.ref.tag], { stdio: "ignore" });
|
||||
if (inspect.status === 0) return;
|
||||
}
|
||||
process.stderr.write(
|
||||
`» building ${ctx.ref.tag}${ctx.noCache ? " (--no-cache)" : ""} (one-time, ~30-60s)…\n`
|
||||
);
|
||||
const buildArgs = ["build", "-t", ctx.ref.tag, "-f", ctx.dockerfile];
|
||||
if (ctx.noCache) buildArgs.push("--no-cache");
|
||||
buildArgs.push(actionDir);
|
||||
const build = spawnSync("docker", buildArgs, { stdio: "inherit" });
|
||||
if (build.status !== 0) {
|
||||
fail("image build failed");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* print versions of every tool we expect to be available, so contributors
|
||||
* can sanity-check fidelity with the GHA `ubuntu-24.04` runner when a test
|
||||
* passes locally but fails in CI (or vice versa).
|
||||
*/
|
||||
function runDoctor(ref: ImageRef): void {
|
||||
// multi-line bash script; spawnSync passes the whole thing as one argv
|
||||
// entry so there's no nested-shell quoting to worry about, and `do` is
|
||||
// not followed by a stray semicolon.
|
||||
const script = `set +e
|
||||
echo '--- container ---'
|
||||
grep -E '^(NAME|VERSION)=' /etc/os-release
|
||||
echo "arch=$(uname -m)"
|
||||
|
||||
echo
|
||||
echo '--- runtimes ---'
|
||||
echo "node $(node --version)"
|
||||
if cd /app/action 2>/dev/null; then
|
||||
echo "pnpm $(corepack pnpm --version) (corepack-resolved from packageManager)"
|
||||
else
|
||||
echo "pnpm $(pnpm --version) (system fallback — /app/action not mounted?)"
|
||||
fi
|
||||
python3 --version
|
||||
|
||||
echo
|
||||
echo '--- tools ---'
|
||||
for t in gh jq git ssh curl wget tar gzip xz unzip file make gcc g++ sudo unshare awk sed grep find xargs; do
|
||||
if ! command -v "$t" >/dev/null 2>&1; then
|
||||
printf ' %-10s MISSING\\n' "$t"
|
||||
continue
|
||||
fi
|
||||
case "$t" in
|
||||
ssh|unzip) v=$("$t" -V 2>&1 | head -1) ;;
|
||||
*) v=$("$t" --version 2>&1 | head -1) ;;
|
||||
esac
|
||||
printf ' %-10s %s\\n' "$t" "$v"
|
||||
done
|
||||
|
||||
echo
|
||||
echo '--- env ---'
|
||||
echo "CI=$CI HOME=$HOME TMPDIR=$TMPDIR"
|
||||
echo "doctor runs as: $(whoami) (uid=$(id -u) gid=$(id -g))"
|
||||
echo "tests run as: testuser (uid remapped to host uid at entrypoint)"
|
||||
echo "host.docker.internal -> $(getent hosts host.docker.internal | awk '{print $1}' || echo UNRESOLVED)"
|
||||
`;
|
||||
const result = spawnSync(
|
||||
"docker",
|
||||
[
|
||||
"run",
|
||||
"--rm",
|
||||
"-v",
|
||||
`${actionDir}:/app/action:cached`,
|
||||
"--add-host=host.docker.internal:host-gateway",
|
||||
"--entrypoint",
|
||||
"/bin/bash",
|
||||
ref.tag,
|
||||
"-c",
|
||||
script,
|
||||
],
|
||||
{ stdio: "inherit" }
|
||||
);
|
||||
process.exit(result.status ?? 1);
|
||||
}
|
||||
|
||||
function volumeExists(name: string): boolean {
|
||||
return spawnSync("docker", ["volume", "inspect", name], { stdio: "ignore" }).status === 0;
|
||||
}
|
||||
|
||||
function initVolumeOwnership(ctx: { ref: ImageRef; uid: number; gid: number }): void {
|
||||
// a fresh named volume is owned by root; chown once on creation. on warm
|
||||
// runs the volume already has the right ownership and `docker run … chown`
|
||||
// is sub-second pure overhead — skip it.
|
||||
if (volumeExists(ctx.ref.volumeName)) return;
|
||||
spawnSync(
|
||||
"docker",
|
||||
[
|
||||
"run",
|
||||
"--rm",
|
||||
"--entrypoint",
|
||||
"chown",
|
||||
"-v",
|
||||
`${ctx.ref.volumeName}:/app/action/node_modules`,
|
||||
ctx.ref.tag,
|
||||
"-R",
|
||||
`${ctx.uid}:${ctx.gid}`,
|
||||
"/app/action/node_modules",
|
||||
],
|
||||
{ stdio: "ignore" }
|
||||
);
|
||||
}
|
||||
|
||||
type EnvParts = { envFile: string; multiLineFlags: string[] };
|
||||
|
||||
function buildEnvParts(env: NodeJS.ProcessEnv): EnvParts {
|
||||
const dir = join(tmpdir(), "pullfrog-docker");
|
||||
mkdirSync(dir, { recursive: true });
|
||||
const envFile = join(dir, `env-${process.pid}-${Date.now()}.list`);
|
||||
const lines: string[] = [];
|
||||
const multiLineFlags: string[] = [];
|
||||
for (const key of Object.keys(env)) {
|
||||
if (HOST_ONLY_VARS.has(key)) continue;
|
||||
const value = env[key];
|
||||
if (value === undefined) continue;
|
||||
// docker --env-file is line-oriented and does not support multi-line
|
||||
// values. fall back to -e for those (RSA keys, multi-line PEMs, etc.).
|
||||
if (value.includes("\n") || value.includes("\r")) {
|
||||
multiLineFlags.push("-e", `${key}=${value}`);
|
||||
} else {
|
||||
lines.push(`${key}=${value}`);
|
||||
}
|
||||
}
|
||||
writeFileSync(envFile, `${lines.join("\n")}\n`, { mode: 0o600 });
|
||||
return { envFile, multiLineFlags };
|
||||
}
|
||||
|
||||
function buildSshFlags(home: string | undefined): string[] {
|
||||
const flags: string[] = [];
|
||||
if (!home) return flags;
|
||||
if (platform() === "darwin") {
|
||||
const knownHosts = join(home, ".ssh", "known_hosts");
|
||||
if (existsSync(knownHosts)) {
|
||||
flags.push("-v", `${knownHosts}:/tmp/home/.ssh/known_hosts:ro`);
|
||||
}
|
||||
flags.push(
|
||||
"-v",
|
||||
"/run/host-services/ssh-auth.sock:/run/host-services/ssh-auth.sock",
|
||||
"-e",
|
||||
"SSH_AUTH_SOCK=/run/host-services/ssh-auth.sock"
|
||||
);
|
||||
} else {
|
||||
const sshDir = join(home, ".ssh");
|
||||
if (existsSync(sshDir)) {
|
||||
flags.push("-v", `${sshDir}:/tmp/.ssh-host:ro`);
|
||||
}
|
||||
}
|
||||
return flags;
|
||||
}
|
||||
|
||||
function main(): void {
|
||||
const args = parseArgs(process.argv.slice(2));
|
||||
ensureDocker();
|
||||
|
||||
const dockerfile = join(actionDir, "Dockerfile");
|
||||
const entrypoint = join(actionDir, "docker-entrypoint.sh");
|
||||
const ref = imageRefFor({ dockerfile, entrypoint });
|
||||
|
||||
if (args.clean) {
|
||||
cleanOrphans(ref);
|
||||
if (!args.shell && !args.doctor && args.passthrough.length === 0 && !args.forceBuild) {
|
||||
process.exit(0);
|
||||
}
|
||||
}
|
||||
|
||||
buildImageIfNeeded({ ref, force: args.forceBuild, noCache: args.noCache, dockerfile });
|
||||
|
||||
if (args.doctor) {
|
||||
runDoctor(ref);
|
||||
// runDoctor exits; unreachable.
|
||||
}
|
||||
|
||||
// standalone `--build`: image's done, nothing to run.
|
||||
if (!args.shell && args.passthrough.length === 0) {
|
||||
if (!args.forceBuild) {
|
||||
showHelp();
|
||||
process.exit(1);
|
||||
}
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
// node sets isTTY to `true` for a terminal stdin, `undefined` otherwise
|
||||
// (never `false`). check truthiness, not equality.
|
||||
if (args.shell && !process.stdin.isTTY) {
|
||||
fail("--shell needs a TTY (stdin is not a terminal). run from an interactive shell.");
|
||||
}
|
||||
|
||||
const uid = process.getuid?.() ?? 1000;
|
||||
const gid = process.getgid?.() ?? 1000;
|
||||
initVolumeOwnership({ ref, uid, gid });
|
||||
|
||||
const envParts = buildEnvParts(process.env);
|
||||
const sshFlags = buildSshFlags(process.env.HOME);
|
||||
|
||||
const runArgs: string[] = [
|
||||
"run",
|
||||
"--rm",
|
||||
// `--init` uses tini as PID 1, which forwards signals (SIGINT/SIGTERM)
|
||||
// to our entrypoint and reaps zombies. Without it, bash-as-PID-1
|
||||
// swallows Ctrl-C during the pre-exec warmup phase.
|
||||
"--init",
|
||||
args.shell ? "-it" : "-t",
|
||||
"--privileged",
|
||||
// make the host reachable from inside the container at a stable name
|
||||
// (macOS Docker Desktop bakes this in; the flag makes Linux match,
|
||||
// matters when scripts hit local dev servers like API_URL=
|
||||
// http://host.docker.internal:3100).
|
||||
"--add-host=host.docker.internal:host-gateway",
|
||||
"-v",
|
||||
`${actionDir}:/app/action:cached`,
|
||||
"-v",
|
||||
`${ref.volumeName}:/app/action/node_modules`,
|
||||
"-w",
|
||||
"/app/action",
|
||||
"--env-file",
|
||||
envParts.envFile,
|
||||
"-e",
|
||||
`HOST_UID=${uid}`,
|
||||
"-e",
|
||||
`HOST_GID=${gid}`,
|
||||
...envParts.multiLineFlags,
|
||||
...sshFlags,
|
||||
ref.tag,
|
||||
];
|
||||
|
||||
if (args.shell) {
|
||||
runArgs.push("--shell");
|
||||
} else {
|
||||
// resolve script paths relative to actionDir (matches `pnpm -C action`
|
||||
// mental model). absolute paths and bare flags pass through unchanged.
|
||||
const [script, ...rest] = args.passthrough;
|
||||
if (script === undefined) {
|
||||
fail("internal: passthrough empty");
|
||||
}
|
||||
runArgs.push("node", script, ...rest);
|
||||
}
|
||||
|
||||
let exitCode = 1;
|
||||
try {
|
||||
const result = spawnSync("docker", runArgs, { stdio: "inherit" });
|
||||
exitCode = result.status ?? 1;
|
||||
} finally {
|
||||
try {
|
||||
unlinkSync(envParts.envFile);
|
||||
} catch {
|
||||
// best-effort; tmpdir is GC'd by the OS regardless.
|
||||
}
|
||||
}
|
||||
process.exit(exitCode);
|
||||
}
|
||||
|
||||
const isDirectExecution = process.argv[1]
|
||||
? import.meta.url === pathToFileURL(resolve(process.argv[1])).href
|
||||
: false;
|
||||
|
||||
if (isDirectExecution) {
|
||||
main();
|
||||
}
|
||||
+102
@@ -0,0 +1,102 @@
|
||||
#!/usr/bin/env node
|
||||
//
|
||||
// GitHub Actions `post:` entry point. Runs after the main step regardless of
|
||||
// exit status (cancellation, timeout, unhandled error) — that's the contract
|
||||
// we need for credential persistence: if OpenCode refreshed the Codex
|
||||
// auth.json during the run, the refreshed token must land back in Pullfrog
|
||||
// even when the main step died unexpectedly.
|
||||
//
|
||||
// THIS IS WHY `CODEX_AUTH_JSON` HAS TO LIVE IN PULLFROG'S OWN SECRET STORE,
|
||||
// NOT IN GITHUB ACTIONS SECRETS. The refresh chain rotates on every use; this
|
||||
// hook PUTs the rotated chain back to Pullfrog Postgres so the next run starts
|
||||
// from a fresh token. GH Actions secrets are read-only at runtime — there is
|
||||
// no API to write them back from inside a job — so a token stashed there
|
||||
// silently goes stale on the first refresh and the next run fails. See
|
||||
// wiki/codex-auth.md.
|
||||
//
|
||||
// Today's only job: detect a Codex auth refresh by diffing the on-disk
|
||||
// auth.json against the original refresh token (saved to GH Actions state
|
||||
// by action/agents/opencode_v2.ts — see also the legacy v1 file kept as
|
||||
// reference at action/agents/opencode.ts), convert OpenCode's auth shape
|
||||
// back to Codex CLI shape, and PUT it to /api/runtime/secret.
|
||||
//
|
||||
// Silent no-op when the main step didn't materialize Codex auth (no state
|
||||
// saved). Best-effort: failures are logged but never throw — the workflow
|
||||
// is already done, and a missed refresh write-back means the user re-runs
|
||||
// `pullfrog auth codex` next time the chain breaks.
|
||||
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
import * as core from "@actions/core";
|
||||
import { apiFetch } from "./utils/apiFetch.ts";
|
||||
import { detectCodexRefresh } from "./utils/codexHome.ts";
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const raw = core.getState("codex_writeback");
|
||||
if (!raw) {
|
||||
core.info("codex post-hook: no writeback state — skipping");
|
||||
return;
|
||||
}
|
||||
|
||||
let state: { apiToken: string; authPath: string; originalRefresh: string };
|
||||
try {
|
||||
state = JSON.parse(raw) as typeof state;
|
||||
} catch (err) {
|
||||
core.warning(`codex post-hook: malformed writeback state — ${err}`);
|
||||
return;
|
||||
}
|
||||
if (!state.apiToken || !state.authPath || !state.originalRefresh) {
|
||||
core.warning("codex post-hook: incomplete writeback state — skipping");
|
||||
return;
|
||||
}
|
||||
|
||||
if (!existsSync(state.authPath)) {
|
||||
core.info(`codex post-hook: ${state.authPath} not found — nothing to write back`);
|
||||
return;
|
||||
}
|
||||
|
||||
let authFileContent: string;
|
||||
try {
|
||||
authFileContent = readFileSync(state.authPath, "utf8");
|
||||
} catch (err) {
|
||||
core.warning(`codex post-hook: cannot read ${state.authPath} — ${err}`);
|
||||
return;
|
||||
}
|
||||
|
||||
const refreshedCodexJson = detectCodexRefresh({
|
||||
authFileContent,
|
||||
originalRefresh: state.originalRefresh,
|
||||
});
|
||||
if (!refreshedCodexJson) {
|
||||
core.info("codex post-hook: refresh chain unchanged — no writeback needed");
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
// route through apiFetch so the Vercel preview-deployment SSO gate gets
|
||||
// the `x-vercel-protection-bypass` header/query (raw fetch silently 401s
|
||||
// against preview envs — production is unaffected but every preview-run
|
||||
// refresh would be lost). see action/utils/apiFetch.ts.
|
||||
const response = await apiFetch({
|
||||
path: "/api/runtime/secret",
|
||||
method: "PUT",
|
||||
headers: {
|
||||
authorization: `Bearer ${state.apiToken}`,
|
||||
"content-type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({ name: "CODEX_AUTH_JSON", value: refreshedCodexJson }),
|
||||
});
|
||||
if (!response.ok) {
|
||||
const body = await response.text().catch(() => "");
|
||||
core.warning(`codex post-hook: writeback returned ${response.status}: ${body}`);
|
||||
return;
|
||||
}
|
||||
core.info("codex post-hook: refreshed CODEX_AUTH_JSON persisted to Pullfrog");
|
||||
} catch (err) {
|
||||
core.warning(`codex post-hook: writeback failed — ${err}`);
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
// never throw — post-hook failure must not fail the workflow
|
||||
core.warning(`codex post-hook: unexpected error — ${err}`);
|
||||
});
|
||||
@@ -30,6 +30,7 @@ export function formatMcpToolRef(agentId: AgentId, toolName: string): string {
|
||||
export type { ModelAlias, ModelProvider, ProviderConfig } from "./models.ts";
|
||||
export {
|
||||
getModelEnvVars,
|
||||
getModelManagedCredentials,
|
||||
getModelProvider,
|
||||
getProviderDisplayName,
|
||||
modelAliases,
|
||||
@@ -273,6 +274,13 @@ export interface WriteablePayload {
|
||||
triggerer?: string | undefined;
|
||||
/** event-level instructions for this trigger type (flag-expanded server-side) */
|
||||
eventInstructions?: string | undefined;
|
||||
/**
|
||||
* system-injected note about prior superseded runs (e.g. when the
|
||||
* triggering @pullfrog comment is edited). rendered alongside the user's
|
||||
* prompt rather than via eventInstructions so it survives user-prompt
|
||||
* precedence.
|
||||
*/
|
||||
previousRunsNote?: string | undefined;
|
||||
/** event data from webhook payload - discriminated union based on trigger field */
|
||||
event: PayloadEvent;
|
||||
/** timeout for agent run (e.g., "10m", "1h30m") - defaults to "1h" */
|
||||
|
||||
@@ -17,6 +17,7 @@ export type {
|
||||
} from "../external.ts";
|
||||
export {
|
||||
getModelEnvVars,
|
||||
getModelManagedCredentials,
|
||||
getModelProvider,
|
||||
getProviderDisplayName,
|
||||
modelAliases,
|
||||
@@ -44,6 +45,7 @@ export {
|
||||
isLeapingIntoActionCommentBody,
|
||||
LEAPING_INTO_ACTION_PREFIX,
|
||||
} from "../utils/leapingComment.ts";
|
||||
export { MAX_LEARNINGS_LENGTH, truncateAtLineBoundary } from "../utils/learningsTruncate.ts";
|
||||
export type {
|
||||
CreateProgressCommentTarget,
|
||||
ProgressComment,
|
||||
|
||||
@@ -3,16 +3,11 @@
|
||||
import { existsSync, readdirSync } from "node:fs";
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import * as core from "@actions/core";
|
||||
import { deleteProgressComment, reportProgress } from "./mcp/comment.ts";
|
||||
import { reportProgress } from "./mcp/comment.ts";
|
||||
import { startInstallation } from "./mcp/dependencies.ts";
|
||||
import {
|
||||
initToolState,
|
||||
startMcpHttpServer,
|
||||
type ToolContext,
|
||||
type ToolState,
|
||||
} from "./mcp/server.ts";
|
||||
import { startMcpHttpServer, type ToolContext } from "./mcp/server.ts";
|
||||
import { computeModes } from "./modes.ts";
|
||||
import { initToolState } from "./toolState.ts";
|
||||
import {
|
||||
type ActivityTimeout,
|
||||
createProcessOutputActivityTimeout,
|
||||
@@ -20,27 +15,33 @@ import {
|
||||
DEFAULT_ACTIVITY_TIMEOUT_MS,
|
||||
} from "./utils/activity.ts";
|
||||
import { resolveAgent, resolveModel } from "./utils/agent.ts";
|
||||
import { apiFetch } from "./utils/apiFetch.ts";
|
||||
import { validateAgentApiKey } from "./utils/apiKeys.ts";
|
||||
import { resolveBody } from "./utils/body.ts";
|
||||
import { formatUsageSummary, log, writeSummary } from "./utils/cli.ts";
|
||||
import { selectFallbackModelIfNeeded } from "./utils/byokFallback.ts";
|
||||
import { log } from "./utils/cli.ts";
|
||||
import { recordDiffReadFromToolUse } from "./utils/diffCoverage.ts";
|
||||
import { reportErrorToComment } from "./utils/errorReport.ts";
|
||||
import { onExitSignal } from "./utils/exitHandler.ts";
|
||||
import { resolveGit, setGitAuthServer } from "./utils/gitAuth.ts";
|
||||
import { startGitAuthServer } from "./utils/gitAuthServer.ts";
|
||||
import { createOctokit, writeGitHubUsageSummaryToFile } from "./utils/github.ts";
|
||||
import { resolveInstructions } from "./utils/instructions.ts";
|
||||
import { persistLearnings, seedLearningsFile } from "./utils/learnings.ts";
|
||||
import { executeLifecycleHook } from "./utils/lifecycle.ts";
|
||||
import { normalizeEnv } from "./utils/normalizeEnv.ts";
|
||||
import { normalizeEnv, sanitizeSecret } from "./utils/normalizeEnv.ts";
|
||||
import { applyOverrides } from "./utils/overrides.ts";
|
||||
import { aggregateUsage, patchWorkflowRunFields } from "./utils/patchWorkflowRunFields.ts";
|
||||
import { resolvePayload, resolvePromptInput } from "./utils/payload.ts";
|
||||
import { isRouterKeylimitExhaustedError } from "./utils/providerErrors.ts";
|
||||
import { readSummaryFile, seedSummaryFile } from "./utils/prSummary.ts";
|
||||
import { postReviewCleanup } from "./utils/reviewCleanup.ts";
|
||||
import { resolveOutputSchema, resolvePayload, resolvePromptInput } from "./utils/payload.ts";
|
||||
import { type OidcCredentials, runProxyResolution } from "./utils/proxy.ts";
|
||||
import { fetchPreviousSnapshot, persistSummary, seedSummaryFile } from "./utils/prSummary.ts";
|
||||
import { handleAgentResult } from "./utils/run.ts";
|
||||
import { type AccountPlan, isInfraCovered } from "./utils/runContext.ts";
|
||||
import { resolveRunContextData } from "./utils/runContextData.ts";
|
||||
import { renderRunError } from "./utils/runErrorRenderer.ts";
|
||||
import {
|
||||
finalizeSuccessRun,
|
||||
persistRunArtifacts,
|
||||
writeRunErrorOutputs,
|
||||
} from "./utils/runLifecycle.ts";
|
||||
import { logRunStartup } from "./utils/runStartupLog.ts";
|
||||
import { setEnvAllowlist } from "./utils/secrets.ts";
|
||||
import { createTempDirectory, setupGit } from "./utils/setup.ts";
|
||||
import { killTrackedChildren } from "./utils/subprocess.ts";
|
||||
@@ -48,6 +49,11 @@ import { resolveTimeoutMs, TIMEOUT_DISABLED } from "./utils/time.ts";
|
||||
import { Timer } from "./utils/timer.ts";
|
||||
import { createTodoTracker } from "./utils/todoTracking.ts";
|
||||
import { getJobToken, resolveTokens } from "./utils/token.ts";
|
||||
import {
|
||||
cleanupVertexCredentials,
|
||||
materializeVertexCredentials,
|
||||
type VertexCredentials,
|
||||
} from "./utils/vertex.ts";
|
||||
import { resolveRun } from "./utils/workflow.ts";
|
||||
|
||||
export { Inputs } from "./utils/payload.ts";
|
||||
@@ -59,384 +65,29 @@ export interface MainResult {
|
||||
result?: string | undefined;
|
||||
}
|
||||
|
||||
function resolveOutputSchema(): Record<string, unknown> | undefined {
|
||||
const raw = core.getInput("output_schema");
|
||||
if (!raw) return undefined;
|
||||
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(raw);
|
||||
} catch {
|
||||
throw new Error(`invalid output_schema: not valid JSON`);
|
||||
}
|
||||
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
|
||||
throw new Error(`invalid output_schema: must be a JSON object`);
|
||||
}
|
||||
log.info("» structured output schema provided — output will be required");
|
||||
return parsed as Record<string, unknown>;
|
||||
}
|
||||
|
||||
function resolveTimeoutForLog(timeout: string | undefined): string {
|
||||
if (!timeout) return "1h (default)";
|
||||
if (timeout === TIMEOUT_DISABLED) return "none (disabled)";
|
||||
return timeout;
|
||||
}
|
||||
|
||||
function resolveModelForLog(ctx: {
|
||||
payload: ResolvedPayload;
|
||||
resolvedModel: string | undefined;
|
||||
}): string {
|
||||
const envModel = process.env.PULLFROG_MODEL?.trim();
|
||||
if (envModel) return `${envModel} (override via PULLFROG_MODEL)`;
|
||||
if (ctx.payload.proxyModel) return `${ctx.payload.proxyModel} (proxy)`;
|
||||
if (ctx.resolvedModel && ctx.payload.model && ctx.payload.model !== ctx.resolvedModel) {
|
||||
return `${ctx.resolvedModel} (resolved from ${ctx.payload.model})`;
|
||||
}
|
||||
if (ctx.resolvedModel) return ctx.resolvedModel;
|
||||
if (ctx.payload.model) return `${ctx.payload.model} (unresolved)`;
|
||||
return "auto";
|
||||
}
|
||||
|
||||
function resolveAgentForLog(ctx: { agentName: string; resolvedModel: string | undefined }): string {
|
||||
const envAgent = process.env.PULLFROG_AGENT?.trim();
|
||||
if (envAgent && envAgent === ctx.agentName) {
|
||||
return `${ctx.agentName} (override via PULLFROG_AGENT)`;
|
||||
}
|
||||
if (ctx.agentName === "claude" && ctx.resolvedModel) {
|
||||
return `${ctx.agentName} (auto-selected for ${ctx.resolvedModel})`;
|
||||
}
|
||||
return ctx.agentName;
|
||||
}
|
||||
|
||||
import type { ResolvedPayload } from "./utils/payload.ts";
|
||||
|
||||
interface OidcCredentials {
|
||||
requestUrl: string;
|
||||
requestToken: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Billing-layer error surfaced from `/api/proxy-token` as a 402. User-actionable
|
||||
* — distinct from TransientError (503 / transient sync issue) so the job
|
||||
* summary + PR comment can use affirmative "you need to do X" copy rather than
|
||||
* the ambiguous "billing error" label that makes transient outages look like
|
||||
* the user's fault.
|
||||
*
|
||||
* `code` is a server-side discriminator: `router_requires_card` (no card + no
|
||||
* wallet balance on Router), or null for unclassified. `declineCode` is
|
||||
* Stripe's more specific sub-reason on `card_declined` (e.g.
|
||||
* `insufficient_funds`, `lost_card`). `needsReauthentication` is the 3DS case
|
||||
* broken out for convenience.
|
||||
*/
|
||||
class BillingError extends Error {
|
||||
code: string | null;
|
||||
declineCode: string | null;
|
||||
needsReauthentication: boolean;
|
||||
|
||||
constructor(
|
||||
message: string,
|
||||
opts: {
|
||||
code?: string | null;
|
||||
declineCode?: string | null;
|
||||
needsReauthentication?: boolean;
|
||||
} = {}
|
||||
) {
|
||||
super(message);
|
||||
this.name = "BillingError";
|
||||
this.code = opts.code ?? null;
|
||||
this.declineCode = opts.declineCode ?? null;
|
||||
this.needsReauthentication = opts.needsReauthentication ?? false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Transient service failures from `/api/proxy-token` (503: partial OpenRouter
|
||||
* usage sync, DB flake, in-flight payment intent). Not the user's fault — the
|
||||
* summary uses "temporarily unavailable" framing, and the non-zero exit lets
|
||||
* GH Actions apply whatever retry policy the workflow has configured.
|
||||
*/
|
||||
class TransientError extends Error {
|
||||
constructor(message: string) {
|
||||
super(message);
|
||||
this.name = "TransientError";
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Deep link into the right console section for the failing account. Anchors
|
||||
* are defined in `app/console/[owner]/page.tsx` (`#billing`, `#model-access`).
|
||||
* `owner` is the GitHub login of the repo's account — i.e. the org or user
|
||||
* that pays for this repo's runs, which is the right scope for billing.
|
||||
*/
|
||||
function billingConsoleUrl(owner: string, anchor: "billing" | "model-access"): string {
|
||||
return `https://pullfrog.com/console/${encodeURIComponent(owner)}#${anchor}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Render a BillingError as user-facing markdown (shared between GH job summary
|
||||
* and the PR progress comment). Goals:
|
||||
*
|
||||
* - quiet, not alarmist — bold first line instead of an `### ❌` H3, since
|
||||
* the comment already has Pullfrog branding in the footer
|
||||
* - actionable — every branch ends in a single CTA deep-linked to the
|
||||
* correct section of the owner's console
|
||||
* - honest — say what actually went wrong (card declined vs. balance
|
||||
* empty vs. 3DS required), don't lump them under "billing error"
|
||||
*
|
||||
* Branches:
|
||||
* - `router_requires_card`: user is on Router mode with no card AND no
|
||||
* wallet balance. Lead with the carrot ($20 free credit), link to
|
||||
* `#model-access` where the Add Card flow lives.
|
||||
* - `router_balance_exhausted`: user has a card on file but auto-reload is
|
||||
* disabled and they've spent past their $5 overdraft buffer. Frame as
|
||||
* "balance ran out" and surface both remediation paths (top up, or flip
|
||||
* on auto-reload).
|
||||
* - `router_keylimit_exhausted`: OpenRouter rejected mid-run because the
|
||||
* per-run key budget was exhausted while the agent was working. The
|
||||
* wallet is now negative; same remediation as `router_balance_exhausted`
|
||||
* but framed for the after-the-fact case ("this run was cut short").
|
||||
* - `needsReauthentication`: issuer requires 3DS on every off-session
|
||||
* charge. Re-adding the card won't help — the only escape is a manual
|
||||
* top-up where 3DS runs interactively in Stripe Checkout.
|
||||
* - `declineCode` set: Stripe declined a real charge. Show the sub-code
|
||||
* so support can act on it; tell the user we'll retry on next dispatch.
|
||||
* - default: balance hit zero with no in-flight charge (auto-reload off
|
||||
* or amount below threshold). Direct them to top up or enable auto-reload.
|
||||
*/
|
||||
function formatBillingErrorSummary(error: BillingError, owner: string): string {
|
||||
if (error.code === "router_requires_card") {
|
||||
return [
|
||||
"**Add a card to start using Pullfrog Router.**",
|
||||
"",
|
||||
"Router proxies OpenRouter at raw cost — no platform markup, and your first $20 of usage is on us.",
|
||||
"",
|
||||
`[Add a card →](${billingConsoleUrl(owner, "model-access")})`,
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
if (error.code === "router_balance_exhausted") {
|
||||
return [
|
||||
"**Your Pullfrog Router balance is exhausted.**",
|
||||
"",
|
||||
"You have a card on file but auto-reload is disabled, so runs paused once your balance went past the overdraft buffer.",
|
||||
"",
|
||||
`[Top up balance →](${billingConsoleUrl(owner, "billing")}) · [Enable auto-reload →](${billingConsoleUrl(owner, "model-access")})`,
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
if (error.code === "router_keylimit_exhausted") {
|
||||
return [
|
||||
"**This run was cut short — your Pullfrog Router balance ran out mid-run.**",
|
||||
"",
|
||||
"OpenRouter stopped the agent because the per-run budget was exhausted. Your wallet is now negative; top up or enable auto-reload to keep runs flowing.",
|
||||
"",
|
||||
`[Top up balance →](${billingConsoleUrl(owner, "billing")}) · [Enable auto-reload →](${billingConsoleUrl(owner, "model-access")})`,
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
if (error.needsReauthentication) {
|
||||
const code = error.declineCode ?? "authentication_required";
|
||||
return [
|
||||
`**Your card issuer requires 3D Secure on every charge** (\`${code}\`).`,
|
||||
"",
|
||||
"Pullfrog can't complete a 3DS challenge from inside a workflow. Top up your Router balance once in Stripe Checkout — subsequent runs draw from the prepaid balance without re-triggering 3DS.",
|
||||
"",
|
||||
`[Top up balance →](${billingConsoleUrl(owner, "billing")})`,
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
if (error.declineCode) {
|
||||
return [
|
||||
`**Your card was declined** (\`${error.declineCode}\`).`,
|
||||
"",
|
||||
"Update your payment method and Pullfrog will retry on the next run.",
|
||||
"",
|
||||
`[Update payment method →](${billingConsoleUrl(owner, "billing")})`,
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
return [
|
||||
"**Your Pullfrog balance is empty.**",
|
||||
"",
|
||||
"Top up your balance or enable auto-reload to keep runs flowing.",
|
||||
"",
|
||||
`[Manage billing →](${billingConsoleUrl(owner, "billing")})`,
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
/**
|
||||
* Render a TransientError as user-facing markdown. Distinct framing from
|
||||
* BillingError so the user doesn't read an alarm and assume their card
|
||||
* failed — this branch is "our fault, retry shortly", not theirs.
|
||||
*/
|
||||
function formatTransientErrorSummary(error: TransientError, owner: string): string {
|
||||
return [
|
||||
"**Pullfrog billing is temporarily unavailable.**",
|
||||
"",
|
||||
error.message,
|
||||
"",
|
||||
`Usually transient — the next dispatch should succeed. If it persists, check [status.pullfrog.com](https://status.pullfrog.com) or [your console](${billingConsoleUrl(owner, "billing")}).`,
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
async function mintProxyKey(ctx: { oidcCredentials: OidcCredentials }): Promise<string | null> {
|
||||
try {
|
||||
process.env.ACTIONS_ID_TOKEN_REQUEST_URL = ctx.oidcCredentials.requestUrl;
|
||||
process.env.ACTIONS_ID_TOKEN_REQUEST_TOKEN = ctx.oidcCredentials.requestToken;
|
||||
const oidcToken = await core.getIDToken("pullfrog-api");
|
||||
delete process.env.ACTIONS_ID_TOKEN_REQUEST_URL;
|
||||
delete process.env.ACTIONS_ID_TOKEN_REQUEST_TOKEN;
|
||||
|
||||
const response = await apiFetch({
|
||||
path: "/api/proxy-token",
|
||||
method: "POST",
|
||||
headers: { Authorization: `Bearer ${oidcToken}` },
|
||||
});
|
||||
|
||||
if (response.status === 402) {
|
||||
const body = (await response.json().catch(() => null)) as {
|
||||
error?: string;
|
||||
code?: string;
|
||||
declineCode?: string;
|
||||
needsReauthentication?: boolean;
|
||||
} | null;
|
||||
throw new BillingError(body?.error ?? "insufficient balance", {
|
||||
code: body?.code ?? null,
|
||||
declineCode: body?.declineCode ?? null,
|
||||
needsReauthentication: body?.needsReauthentication ?? false,
|
||||
});
|
||||
}
|
||||
|
||||
// 503 = transient sync issue (partial OpenRouter failure, DB flake,
|
||||
// in-flight top-up). Not the user's fault — TransientError renders a
|
||||
// "temporarily unavailable" summary instead of the "billing error"
|
||||
// label that BillingError uses.
|
||||
if (response.status === 503) {
|
||||
const body = (await response.json().catch(() => null)) as { error?: string } | null;
|
||||
throw new TransientError(
|
||||
body?.error ?? "billing service temporarily unavailable — retry shortly"
|
||||
);
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
log.warning(`proxy key mint failed (${response.status})`);
|
||||
return null;
|
||||
}
|
||||
|
||||
const data = (await response.json()) as { key: string };
|
||||
return data.key;
|
||||
} catch (error) {
|
||||
if (error instanceof BillingError) throw error;
|
||||
if (error instanceof TransientError) throw error;
|
||||
log.warning(`proxy key mint error: ${error instanceof Error ? error.message : String(error)}`);
|
||||
return null;
|
||||
} finally {
|
||||
delete process.env.ACTIONS_ID_TOKEN_REQUEST_URL;
|
||||
delete process.env.ACTIONS_ID_TOKEN_REQUEST_TOKEN;
|
||||
}
|
||||
}
|
||||
|
||||
async function resolveProxyModel(ctx: {
|
||||
payload: ResolvedPayload;
|
||||
oss: boolean;
|
||||
plan: AccountPlan;
|
||||
proxyModel?: string | undefined;
|
||||
oidcCredentials: OidcCredentials | null;
|
||||
}): Promise<void> {
|
||||
// env override = BYOK escape hatch, don't proxy
|
||||
if (process.env.PULLFROG_MODEL?.trim()) return;
|
||||
|
||||
const needsProxy = isInfraCovered({ isOss: ctx.oss, plan: ctx.plan }) && ctx.proxyModel;
|
||||
if (!needsProxy) return;
|
||||
|
||||
if (!ctx.oidcCredentials) {
|
||||
log.warning("» proxy requested but no OIDC credentials available — skipping");
|
||||
return;
|
||||
}
|
||||
|
||||
const key = await mintProxyKey({ oidcCredentials: ctx.oidcCredentials });
|
||||
if (!key) return;
|
||||
|
||||
process.env.OPENROUTER_API_KEY = key;
|
||||
core.setSecret(key);
|
||||
ctx.payload.proxyModel = ctx.proxyModel;
|
||||
const label = ctx.oss ? "oss" : "router";
|
||||
log.info(`» proxy: ${label} → ${ctx.proxyModel}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch the most recent persisted PR summary snapshot for this PR.
|
||||
* Returns null on first-time PRs, when summary is disabled, or on any error.
|
||||
* Best-effort: a transient API failure should not block the run.
|
||||
*/
|
||||
async function fetchPreviousSnapshot(ctx: ToolContext, prNumber: number): Promise<string | null> {
|
||||
if (!ctx.githubInstallationToken) return null;
|
||||
try {
|
||||
const response = await apiFetch({
|
||||
path: `/api/repo/${ctx.repo.owner}/${ctx.repo.name}/pr/${prNumber}/summary-comment`,
|
||||
method: "GET",
|
||||
headers: { authorization: `Bearer ${ctx.githubInstallationToken}` },
|
||||
signal: AbortSignal.timeout(10_000),
|
||||
});
|
||||
if (!response.ok) return null;
|
||||
const data = (await response.json()) as { snapshot?: string | null };
|
||||
return typeof data.snapshot === "string" && data.snapshot.length > 0 ? data.snapshot : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the agent-edited PR summary tmpfile and persist to `WorkflowRun.summarySnapshot`.
|
||||
*
|
||||
* Best-effort: any failure is logged and does not affect the run's success
|
||||
* status. Skips the PATCH when the file is byte-identical to its seed —
|
||||
* persisting the seed verbatim would either re-write what the DB already has
|
||||
* (on incremental runs) or serialize the placeholder scaffold (on first
|
||||
* runs), neither of which is useful.
|
||||
*/
|
||||
async function persistSummary(ctx: ToolContext): Promise<void> {
|
||||
const filePath = ctx.toolState.summaryFilePath;
|
||||
if (!filePath) return;
|
||||
// already-completed guard: the error-path call (success path persisted,
|
||||
// then a late step threw) and the SIGINT/SIGTERM handler all funnel
|
||||
// through here; the first one to arrive wins.
|
||||
if (ctx.toolState.summaryPersistAttempted) return;
|
||||
ctx.toolState.summaryPersistAttempted = true;
|
||||
const snapshot = await readSummaryFile(filePath);
|
||||
if (!snapshot) {
|
||||
log.debug(`pr summary tmpfile missing or invalid at ${filePath} — skipping persist`);
|
||||
return;
|
||||
}
|
||||
// soft gate: agent never touched the seeded file. saving the seed back
|
||||
// is a no-op at best (incremental run — DB already has it) and a bug at
|
||||
// worst (first run — serializes the placeholder italics). log a warning
|
||||
// so the failure mode is visible in CI without flipping the run to
|
||||
// failed.
|
||||
const seed = ctx.toolState.summarySeed?.trim();
|
||||
if (seed !== undefined && snapshot === seed) {
|
||||
log.warning(
|
||||
"» pr summary tmpfile unchanged from seed — skipping persist (agent did not edit it)"
|
||||
);
|
||||
return;
|
||||
}
|
||||
await patchWorkflowRunFields(ctx, { summarySnapshot: snapshot }).catch((err) => {
|
||||
log.debug(`pr summary persist failed: ${err instanceof Error ? err.message : String(err)}`);
|
||||
});
|
||||
}
|
||||
|
||||
async function writeJobSummary(toolState: ToolState): Promise<void> {
|
||||
const usageSummary = formatUsageSummary(toolState.usageEntries);
|
||||
const summaryParts = [toolState.lastProgressBody, usageSummary].filter(Boolean);
|
||||
if (summaryParts.length > 0) {
|
||||
await writeSummary(summaryParts.join("\n\n"));
|
||||
}
|
||||
}
|
||||
|
||||
export async function main(): Promise<MainResult> {
|
||||
// normalize env var names to uppercase (handles case-insensitive workflow files)
|
||||
normalizeEnv();
|
||||
|
||||
// apply caller-supplied env overrides — JSON object forwarded as the
|
||||
// UNSAFE_OVERRIDES env var (NOT a `with:` input). gated by `actions:write`
|
||||
// on the repo and refuses integrity-critical names; see utils/overrides.ts
|
||||
// for the deny-list and wiki/e2e-testing.md for usage + threat model.
|
||||
// the `unsafe` prefix is intentional: GH echoes the env-block value in the
|
||||
// step-header log, so the raw JSON is visible to anyone with `actions:read`.
|
||||
const overridesRaw = process.env.UNSAFE_OVERRIDES ?? "";
|
||||
if (overridesRaw.trim()) {
|
||||
const result = applyOverrides({ raw: overridesRaw, env: process.env });
|
||||
if (result.applied.length > 0) {
|
||||
log.info(`» applied ${result.applied.length} env override(s): ${result.applied.join(", ")}`);
|
||||
}
|
||||
if (result.denied.length > 0) {
|
||||
log.warning(
|
||||
`» refused to override ${result.denied.length} protected env var(s): ${result.denied.join(", ")}`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// write usage summary on SIGINT/SIGTERM so the worker can read it after sandbox.exec
|
||||
const usageSummaryPath = process.env.PULLFROG_USAGE_SUMMARY_PATH;
|
||||
if (usageSummaryPath) {
|
||||
@@ -464,12 +115,15 @@ export async function main(): Promise<MainResult> {
|
||||
const runContext = await resolveRunContextData({ octokit: initialOctokit, token: jobToken });
|
||||
timer.checkpoint("runContextData");
|
||||
|
||||
// inject account-level secrets into process.env (YAML secrets take precedence)
|
||||
// inject account-level secrets into process.env (YAML secrets take precedence).
|
||||
// sanitizeSecret trims + masks so accidental trailing whitespace doesn't leak
|
||||
// through GitHub Actions' line-based log masking. whitespace-only values
|
||||
// return null and skip injection so the user sees a clear missing-key error.
|
||||
if (runContext.dbSecrets) {
|
||||
for (const [key, value] of Object.entries(runContext.dbSecrets)) {
|
||||
if (!process.env[key]) {
|
||||
process.env[key] = value;
|
||||
core.setSecret(value);
|
||||
const sanitized = sanitizeSecret(key, value);
|
||||
if (sanitized !== null) process.env[key] = sanitized;
|
||||
}
|
||||
}
|
||||
const count = Object.keys(runContext.dbSecrets).length;
|
||||
@@ -508,38 +162,18 @@ export async function main(): Promise<MainResult> {
|
||||
}
|
||||
|
||||
// Proxy decision: mint an OpenRouter key for OSS repos or managed billing
|
||||
// accounts. BillingError (402) and TransientError (503) both surface here.
|
||||
// Handle explicitly so the user sees an actionable message (job summary +
|
||||
// PR progress comment when one exists) — otherwise the error unwinds past
|
||||
// the main try/catch (which needs toolState) and lands in runMain with only
|
||||
// a generic core.setFailed.
|
||||
try {
|
||||
await resolveProxyModel({
|
||||
payload,
|
||||
oss: runContext.oss,
|
||||
plan: runContext.plan,
|
||||
proxyModel: runContext.proxyModel,
|
||||
oidcCredentials,
|
||||
});
|
||||
} catch (error) {
|
||||
if (error instanceof BillingError) {
|
||||
const summary = formatBillingErrorSummary(error, runContext.repo.owner);
|
||||
await writeSummary(summary).catch(() => {});
|
||||
// Mirror to the PR progress comment if the trigger created one
|
||||
// (mention / PR event). Without this, auto-reload declines are only
|
||||
// visible in the job summary — users rarely open that, so the agent
|
||||
// just appears to silently stop mid-run.
|
||||
await reportErrorToComment({ toolState, error: summary }).catch(() => {});
|
||||
throw error;
|
||||
}
|
||||
if (error instanceof TransientError) {
|
||||
const summary = formatTransientErrorSummary(error, runContext.repo.owner);
|
||||
await writeSummary(summary).catch(() => {});
|
||||
await reportErrorToComment({ toolState, error: summary }).catch(() => {});
|
||||
throw error;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
// accounts. BillingError (402) and TransientError (503) get rendered inside
|
||||
// `runProxyResolution` before being rethrown — handled here (not in the
|
||||
// outer catch) because the outer catch needs `toolContext` (not yet built)
|
||||
// for its general-purpose error path.
|
||||
await runProxyResolution({
|
||||
payload,
|
||||
oss: runContext.oss,
|
||||
proxyModel: runContext.proxyModel,
|
||||
oidcCredentials,
|
||||
repo: runContext.repo,
|
||||
toolState,
|
||||
});
|
||||
|
||||
// create octokit with MCP token for GitHub API calls
|
||||
const octokit = createOctokit(tokenRef.mcpToken);
|
||||
@@ -548,6 +182,7 @@ export async function main(): Promise<MainResult> {
|
||||
let toolContext: ToolContext | undefined;
|
||||
let progressCallbackDisabled = false;
|
||||
let todoTracker: ReturnType<typeof createTodoTracker> | undefined;
|
||||
let vertexCredentials: VertexCredentials | undefined;
|
||||
|
||||
try {
|
||||
if (payload.cwd && process.cwd() !== payload.cwd) {
|
||||
@@ -575,18 +210,48 @@ export async function main(): Promise<MainResult> {
|
||||
await using gitAuthServer = await startGitAuthServer(tmpdir);
|
||||
setGitAuthServer(gitAuthServer);
|
||||
|
||||
const resolvedModel = payload.proxyModel ? undefined : resolveModel({ slug: payload.model });
|
||||
const initialResolvedModel = payload.proxyModel
|
||||
? undefined
|
||||
: resolveModel({ slug: payload.model });
|
||||
|
||||
// BYOK fallback: if the configured model needs a key the runner doesn't
|
||||
// have, swap to a free OpenCode model so the run can still produce
|
||||
// value. Without this, the agent launches with no key, the LLM provider
|
||||
// 401s, and the run dies in seconds with a synthetic "Invalid API key"
|
||||
// — exactly the silent-churn pattern that took out 15 accounts before
|
||||
// this landed. Router/proxy runs are skipped (Pullfrog mints the key);
|
||||
// see `selectFallbackModelIfNeeded` for the full skip set.
|
||||
const fallback = selectFallbackModelIfNeeded({
|
||||
resolvedModel: initialResolvedModel,
|
||||
proxyModel: payload.proxyModel,
|
||||
});
|
||||
// when fallback engages we bypass `resolveModel` for the new slug —
|
||||
// `PULLFROG_MODEL` has higher priority than the slug arg inside that
|
||||
// helper and would otherwise re-override back to the unkeyed model.
|
||||
// the free fallback slug is already a CLI-ready specifier, so using
|
||||
// it verbatim is correct and avoids the override.
|
||||
const effectiveSlug = fallback.fallback ? fallback.to : payload.model;
|
||||
const resolvedModel = fallback.fallback ? fallback.to : initialResolvedModel;
|
||||
if (fallback.fallback) {
|
||||
log.warning(
|
||||
`» fell back from ${fallback.from} to ${fallback.to} — no BYOK key present in runner env. add a provider key in repo secrets to use ${fallback.from} instead.`
|
||||
);
|
||||
toolState.modelFallback = { from: fallback.from };
|
||||
}
|
||||
|
||||
vertexCredentials = materializeVertexCredentials({ model: resolvedModel });
|
||||
|
||||
const agent = resolveAgent({ model: resolvedModel });
|
||||
|
||||
// surface the effective model in comment/review footers. payload.model is
|
||||
// just the stored slug (often undefined for router/oss runs that derive
|
||||
// the target from proxyModel). matching priority with resolveModelForLog
|
||||
// so the "Using `…`" badge reflects what actually ran.
|
||||
toolState.model = payload.proxyModel ?? resolvedModel ?? payload.model;
|
||||
toolState.model = payload.proxyModel ?? resolvedModel ?? effectiveSlug;
|
||||
|
||||
validateAgentApiKey({
|
||||
agent,
|
||||
model: payload.proxyModel ?? resolvedModel ?? payload.model,
|
||||
model: payload.proxyModel ?? resolvedModel ?? effectiveSlug,
|
||||
owner: runContext.repo.owner,
|
||||
name: runContext.repo.name,
|
||||
});
|
||||
@@ -647,6 +312,43 @@ export async function main(): Promise<MainResult> {
|
||||
log.info(`» MCP server started at ${mcpHttpServer.url}`);
|
||||
timer.checkpoint("mcpServer");
|
||||
|
||||
// seed the rolling repo-level learnings tmpfile for every run. the
|
||||
// agent reads the file at startup (path is surfaced in the LEARNINGS
|
||||
// section of the prompt) and may edit it during the post-run
|
||||
// reflection turn. persistLearnings reads it back at end-of-run and
|
||||
// PATCHes any changes to Repo.learnings, byte-trim equality against
|
||||
// the seed gates the API call. always-seed (vs gated): learnings are
|
||||
// universal — any run can produce them, and gating just hides the
|
||||
// affordance.
|
||||
//
|
||||
// wrapped in best-effort try/catch: this block runs unconditionally,
|
||||
// and an unwrapped filesystem failure (ENOSPC, EACCES, hostile sandbox)
|
||||
// would unwind into the outer main() catch and flip an otherwise-
|
||||
// successful run to "❌ Pullfrog failed" before the agent even starts.
|
||||
// on failure toolState.learningsFilePath stays unset, and downstream
|
||||
// consumers (`persistLearnings`, agent harnesses, `resolveInstructions`)
|
||||
// all treat undefined as "no learnings affordance this run".
|
||||
try {
|
||||
const learningsPath = await seedLearningsFile({
|
||||
tmpdir,
|
||||
current: runContext.repoSettings.learnings,
|
||||
});
|
||||
toolState.learningsFilePath = learningsPath;
|
||||
// file on disk is the verbatim DB body, so the seed used for
|
||||
// change-detection is just `current ?? ""` (trimmed). persistLearnings
|
||||
// byte-compares against the trimmed read-back to skip no-op PATCHes.
|
||||
toolState.learningsSeed = (runContext.repoSettings.learnings ?? "").trim();
|
||||
log.info(
|
||||
`» learnings seeded at ${learningsPath} (existing=${runContext.repoSettings.learnings ? "yes" : "no"})`
|
||||
);
|
||||
const ctxForExit = toolContext;
|
||||
onExitSignal(() => persistLearnings(ctxForExit));
|
||||
} catch (err) {
|
||||
log.warning(
|
||||
`» learnings seed failed: ${err instanceof Error ? err.message : String(err)} — continuing without learnings file`
|
||||
);
|
||||
}
|
||||
|
||||
// seed the rolling PR summary tmpfile when the dispatcher requested it.
|
||||
// gated on event being a PR — issue/workflow_dispatch runs have no
|
||||
// summarySnapshot to maintain. file path is exposed to the agent via
|
||||
@@ -659,9 +361,6 @@ export async function main(): Promise<MainResult> {
|
||||
// the post-run retry loop to detect the agent forgetting to edit
|
||||
// the file (byte-identical to seed → nudge once via resume turn)
|
||||
// and by persistSummary to skip the DB write when nothing changed.
|
||||
// we just wrote the file, so the read shouldn't fail; the catch
|
||||
// leaves summarySeed unset (its default), in which case the unchanged
|
||||
// checks downstream are simply skipped.
|
||||
try {
|
||||
toolState.summarySeed = await readFile(filePath, "utf8");
|
||||
} catch {
|
||||
@@ -683,14 +382,7 @@ export async function main(): Promise<MainResult> {
|
||||
|
||||
startInstallation(toolContext);
|
||||
|
||||
const modelForLog = resolveModelForLog({ payload, resolvedModel });
|
||||
const agentForLog = resolveAgentForLog({ agentName: agent.name, resolvedModel });
|
||||
const timeoutForLog = resolveTimeoutForLog(payload.timeout);
|
||||
log.info(`» model: ${modelForLog}`);
|
||||
log.info(`» agent: ${agentForLog}`);
|
||||
log.info(`» push: ${payload.push}`);
|
||||
log.info(`» shell: ${payload.shell}`);
|
||||
log.info(`» timeout: ${timeoutForLog}`);
|
||||
logRunStartup({ payload, resolvedModel, agentName: agent.name });
|
||||
|
||||
const instructions = resolveInstructions({
|
||||
payload,
|
||||
@@ -698,7 +390,8 @@ export async function main(): Promise<MainResult> {
|
||||
modes,
|
||||
agentId,
|
||||
outputSchema,
|
||||
learnings: runContext.repoSettings.learnings,
|
||||
learningsFilePath: toolState.learningsFilePath ?? null,
|
||||
learningsHeadings: runContext.repoSettings.learningsHeadings,
|
||||
});
|
||||
const logParts = [
|
||||
instructions.eventInstructions
|
||||
@@ -790,11 +483,12 @@ export async function main(): Promise<MainResult> {
|
||||
resolvedModel,
|
||||
mcpServerUrl: mcpHttpServer.url,
|
||||
tmpdir,
|
||||
secretDenyPaths: vertexCredentials ? [vertexCredentials.secretDir] : [],
|
||||
instructions,
|
||||
todoTracker,
|
||||
stopScript: runContext.repoSettings.stopScript,
|
||||
summaryFilePath: toolState.summaryFilePath,
|
||||
summarySeed: toolState.summarySeed,
|
||||
toolState,
|
||||
apiToken: runContext.apiToken,
|
||||
onActivityTimeout: onInnerActivityTimeout,
|
||||
onToolUse: (event) => {
|
||||
const wasTracked = recordDiffReadFromToolUse({
|
||||
@@ -862,53 +556,13 @@ export async function main(): Promise<MainResult> {
|
||||
);
|
||||
}
|
||||
|
||||
// post-agent review cleanup: reportReviewNodeId → follow-up re-review dispatch.
|
||||
// runs after the agent exits so ordering is architecturally guaranteed (no LLM involvement).
|
||||
// best-effort: cleanup failures must not turn a successful agent run into a failure.
|
||||
//
|
||||
// note: progress-comment deletion on review submission is owned by
|
||||
// create_pull_request_review (action/mcp/review.ts) and runs atomically
|
||||
// with the submission, so it survives any path out of main (success,
|
||||
// timeout, crash) without relying on cleanup ordering here.
|
||||
if (toolContext) {
|
||||
await postReviewCleanup(toolContext).catch((error) => {
|
||||
log.debug(`post-review cleanup failed: ${error}`);
|
||||
});
|
||||
}
|
||||
|
||||
// read the agent-edited summary tmpfile and persist to the DB. happens
|
||||
// after the agent exits so the file is in its final state.
|
||||
if (toolContext) {
|
||||
await persistSummary(toolContext);
|
||||
}
|
||||
|
||||
// clean up stranded progress comments. the comment is stale unless
|
||||
// report_progress wrote a final summary to it — three sub-cases all reduce
|
||||
// to !finalSummaryWritten:
|
||||
// 1. nothing wrote to the comment ("Leaping into action" orphan)
|
||||
// 2. tracker published a checklist but the agent never finalized it
|
||||
// 3. the agent produced a substantive artifact via another MCP write tool
|
||||
// (create_issue_comment, update_pull_request_body, reply_to_review_comment)
|
||||
// and skipped report_progress — wasUpdated is true, but the progress
|
||||
// comment itself was never touched.
|
||||
// create_pull_request_review owns its own deletion (see action/mcp/review.ts),
|
||||
// so progressComment is already null by the time we get here for that path.
|
||||
// uses finalSummaryWritten (not todoTracker.enabled or wasUpdated) so cleanup
|
||||
// survives API failures in report_progress where cancel() ran but the write
|
||||
// didn't succeed, and isn't fooled by writes to *other* artifacts.
|
||||
if (toolContext && toolState.progressComment && !toolState.finalSummaryWritten) {
|
||||
await deleteProgressComment(toolContext).catch((error) => {
|
||||
log.debug(`stranded progress comment cleanup failed: ${error}`);
|
||||
});
|
||||
}
|
||||
|
||||
await writeJobSummary(toolState);
|
||||
|
||||
// emit structured output marker for test validation
|
||||
if (toolState.output) {
|
||||
log.info(`::pullfrog-output::${Buffer.from(toolState.output).toString("base64")}`);
|
||||
core.setOutput("result", toolState.output);
|
||||
}
|
||||
// success-path cleanup: postReview → persistSummary → persistLearnings →
|
||||
// failure-error-report → stranded-comment cleanup → job summary → output
|
||||
// marker. each step is best-effort; see `finalizeSuccessRun` for ordering
|
||||
// rationale (notably: progress-comment deletion lives in
|
||||
// create_pull_request_review for review-mode runs, so deletion here
|
||||
// covers the non-review success paths).
|
||||
await finalizeSuccessRun({ toolContext, toolState, result, repo: runContext.repo });
|
||||
|
||||
return await handleAgentResult({
|
||||
result,
|
||||
@@ -922,48 +576,19 @@ export async function main(): Promise<MainResult> {
|
||||
killTrackedChildren();
|
||||
log.error(errorMessage);
|
||||
|
||||
// Reclassify OpenRouter "key budget exhausted" mid-run errors as
|
||||
// BillingError. The agent runtime surfaces this as a generic APIError,
|
||||
// but it's a Pullfrog billing concern — the user's Router wallet ran
|
||||
// out partway through the run. Route through the same formatBillingErrorSummary
|
||||
// path as proxy-token 402s so the user gets actionable copy + a top-up
|
||||
// CTA on both the job summary and the PR progress comment, instead of
|
||||
// a generic "❌ Pullfrog failed" stack-trace dump.
|
||||
const billingError = isRouterKeylimitExhaustedError(errorMessage)
|
||||
? new BillingError(errorMessage, { code: "router_keylimit_exhausted" })
|
||||
: null;
|
||||
// classify (BillingError reclassification + hang detection + API-key auth
|
||||
// detection) and render to {summary, comment} markdown bodies.
|
||||
const rendered = renderRunError({
|
||||
errorMessage,
|
||||
repo: runContext.repo,
|
||||
agentDiagnostic: toolState.agentDiagnostic,
|
||||
});
|
||||
await writeRunErrorOutputs({ rendered, toolState });
|
||||
|
||||
// best-effort summary — write the error so it's visible in the Actions summary tab
|
||||
try {
|
||||
const errorSummary = billingError
|
||||
? formatBillingErrorSummary(billingError, runContext.repo.owner)
|
||||
: `### ❌ Pullfrog failed\n\n\`\`\`\n${errorMessage}\n\`\`\``;
|
||||
const usageSummary = formatUsageSummary(toolState.usageEntries);
|
||||
const parts = [errorSummary, toolState.lastProgressBody, usageSummary].filter(Boolean);
|
||||
await writeSummary(parts.join("\n\n"));
|
||||
} catch {}
|
||||
|
||||
try {
|
||||
const commentBody = billingError
|
||||
? formatBillingErrorSummary(billingError, runContext.repo.owner)
|
||||
: errorMessage;
|
||||
await reportErrorToComment({ toolState, error: commentBody });
|
||||
} catch {
|
||||
// error reporting failed, but don't let it mask the original error
|
||||
}
|
||||
|
||||
// best-effort review cleanup (e.g., agent timed out after submitting a review)
|
||||
// best-effort cleanup: review dispatch, summary persist, learnings persist.
|
||||
// a partial edit before the crash is still worth keeping.
|
||||
if (toolContext) {
|
||||
await postReviewCleanup(toolContext).catch((error) => {
|
||||
log.debug(`post-review cleanup failed: ${error}`);
|
||||
});
|
||||
}
|
||||
|
||||
// best-effort summary persist on the error path: if the agent successfully
|
||||
// edited the summary file before timing out / crashing, those edits are
|
||||
// worth keeping for the next incremental run.
|
||||
if (toolContext) {
|
||||
await persistSummary(toolContext);
|
||||
await persistRunArtifacts(toolContext);
|
||||
}
|
||||
|
||||
return {
|
||||
@@ -1003,5 +628,6 @@ export async function main(): Promise<MainResult> {
|
||||
await patchWorkflowRunFields(toolContext, patch);
|
||||
}
|
||||
}
|
||||
cleanupVertexCredentials(vertexCredentials);
|
||||
}
|
||||
}
|
||||
|
||||
+321
-175
@@ -5,9 +5,10 @@ import type { Octokit, RestEndpointMethodTypes } from "@octokit/rest";
|
||||
import { type } from "arktype";
|
||||
import { log } from "../utils/cli.ts";
|
||||
import { countLines, createDiffCoverageState } from "../utils/diffCoverage.ts";
|
||||
import { $git } from "../utils/gitAuth.ts";
|
||||
import { $git, $gitFetchWithDeepen } from "../utils/gitAuth.ts";
|
||||
import { executeLifecycleHook } from "../utils/lifecycle.ts";
|
||||
import { computeIncrementalDiff } from "../utils/rangeDiff.ts";
|
||||
import { retry } from "../utils/retry.ts";
|
||||
import { $ } from "../utils/shell.ts";
|
||||
import { rejectIfLeadingDash } from "./git.ts";
|
||||
import { commentableLinesForFile } from "./review.ts";
|
||||
@@ -258,10 +259,10 @@ async function ensureBeforeShaReachable(params: EnsureBeforeShaParams): Promise<
|
||||
sha: params.sha,
|
||||
ref: tempBranch,
|
||||
});
|
||||
await $git(
|
||||
"fetch",
|
||||
await $gitFetchWithDeepen(
|
||||
["--no-tags", ...(params.isShallow ? ["--depth=1"] : []), "origin", tempBranch],
|
||||
{ token: params.gitToken }
|
||||
{ token: params.gitToken },
|
||||
`before_sha temp branch ${tempBranch}`
|
||||
);
|
||||
log.debug(`» fetched before_sha via temp branch ${tempBranch}`);
|
||||
return true;
|
||||
@@ -281,6 +282,12 @@ type CheckoutPrBranchParams = GitContext & {
|
||||
// legitimate git op that's holding the lock.
|
||||
const STALE_LOCK_AGE_MS = 30_000;
|
||||
|
||||
// PR head refs (refs/pull/N/head) sometimes lag the pull_request.opened
|
||||
// webhook by a few seconds. retry the missing-ref case with backoff
|
||||
// before giving up — see issue #591.
|
||||
const PULL_REF_RETRY_DELAYS_MS = [2_000, 5_000, 10_000];
|
||||
const PULL_REF_MISSING_PATTERN = /couldn't find remote ref pull\/\d+\/head/i;
|
||||
|
||||
const GIT_LOCK_PATHS = [
|
||||
".git/shallow.lock",
|
||||
".git/index.lock",
|
||||
@@ -308,6 +315,62 @@ function cleanupStaleGitLocks(): void {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns false when a PR's current state diverges from what we dispatched
|
||||
* on (closed/merged, or head SHA differs from pr.headSha). Used to short-
|
||||
* circuit the pull/N/head retry loop when the ref is missing because the
|
||||
* PR has moved on, not because of a webhook race.
|
||||
*
|
||||
* Network failures here are treated as "still valid" — we'd rather burn the
|
||||
* retry budget than wrongly abort on a transient API blip.
|
||||
*
|
||||
* Note: this answers "should we keep trying?", NOT "will the next fetch
|
||||
* succeed?". `pulls.get` (REST API) and `pull/N/head` (git ref) are served
|
||||
* by independent GitHub replicas with their own propagation lag, so
|
||||
* `pulls.get` reporting an open PR with a matching head SHA does not
|
||||
* guarantee the git ref is yet visible — and vice versa (see issue #591
|
||||
* for the original webhook-vs-ref replication-lag context).
|
||||
*/
|
||||
async function isPullRequestStillDispatchable(args: {
|
||||
octokit: Octokit;
|
||||
owner: string;
|
||||
repo: string;
|
||||
pr: PrData;
|
||||
}): Promise<boolean> {
|
||||
try {
|
||||
const { data } = await args.octokit.rest.pulls.get({
|
||||
owner: args.owner,
|
||||
repo: args.repo,
|
||||
pull_number: args.pr.number,
|
||||
});
|
||||
if (data.state !== "open") return false;
|
||||
if (data.head.sha !== args.pr.headSha) return false;
|
||||
return true;
|
||||
} catch {
|
||||
// lenient — don't abort on API hiccups
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Throws the friendly clean-abort error when the PR has moved on since
|
||||
* dispatch. Wraps `isPullRequestStillDispatchable` so the abort message
|
||||
* lives in one place and is invoked from the inner `catch` around the
|
||||
* `pull/N/head` fetch on every missing-ref failure.
|
||||
*/
|
||||
async function abortIfPullRequestMoved(args: {
|
||||
octokit: Octokit;
|
||||
owner: string;
|
||||
repo: string;
|
||||
pr: PrData;
|
||||
}): Promise<void> {
|
||||
const stillValid = await isPullRequestStillDispatchable(args);
|
||||
if (stillValid) return;
|
||||
throw new Error(
|
||||
`PR #${args.pr.number} is no longer in the state it was at dispatch (likely closed, merged, or force-pushed between webhook fire and run start). aborting checkout — re-trigger the run if this PR is still active.`
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Shared helper to checkout a PR branch and configure fork remotes.
|
||||
* Assumes origin remote is already configured with authentication.
|
||||
@@ -347,9 +410,17 @@ export async function checkoutPrBranch(
|
||||
toolState.checkoutSha = $("git", ["rev-parse", "HEAD"], { log: false }).trim();
|
||||
const alreadyOnBranch = toolState.checkoutSha === pr.headSha;
|
||||
|
||||
// fetch base branch so origin/<base> exists for diff operations
|
||||
// fetch base branch so origin/<base> exists for diff operations.
|
||||
// wrap with deepen-retry: on shallow clones (the actions/checkout default
|
||||
// is depth=1), repos with deep PR ancestry can't reach the baseRef tip in
|
||||
// a single round trip, surfacing as `Could not read <sha>` / `remote did
|
||||
// not send all necessary objects` (issue #656).
|
||||
log.debug(`» fetching base branch (${pr.baseRef})...`);
|
||||
await $git("fetch", ["--no-tags", "origin", pr.baseRef], { token: gitToken });
|
||||
await $gitFetchWithDeepen(
|
||||
["--no-tags", "origin", pr.baseRef],
|
||||
{ token: gitToken },
|
||||
`base branch ${pr.baseRef}`
|
||||
);
|
||||
|
||||
// alreadyOnBranch only matches for repeated checkout_pr calls for the same PR in one session
|
||||
// (without the tip moving), or if an external setup already checked out the PR head.
|
||||
@@ -363,11 +434,40 @@ export async function checkoutPrBranch(
|
||||
// -B creates or resets the branch to match origin/baseBranch
|
||||
$("git", ["checkout", "-B", pr.baseRef, `origin/${pr.baseRef}`], { log: false });
|
||||
|
||||
// fetch PR branch using pull/{n}/head refspec (works for both fork and same-repo PRs)
|
||||
// fetch PR branch using pull/{n}/head refspec (works for both fork and same-repo PRs).
|
||||
// two transient classes wrap this fetch:
|
||||
// - shallow-unreachable (`Could not read <sha>` etc.) — handled by the
|
||||
// inner `$gitFetchWithDeepen` deepen-retry (one shot, see issue #656)
|
||||
// - pull/N/head webhook race (`couldn't find remote ref pull/N/head`) —
|
||||
// handled by the outer retry below (see issue #591)
|
||||
log.debug(`» fetching PR #${pr.number} (${localBranch})...`);
|
||||
await $git("fetch", ["--no-tags", "origin", `+pull/${pr.number}/head:${localBranch}`], {
|
||||
token: gitToken,
|
||||
});
|
||||
await retry(
|
||||
async () => {
|
||||
try {
|
||||
await $gitFetchWithDeepen(
|
||||
["--no-tags", "origin", `+pull/${pr.number}/head:${localBranch}`],
|
||||
{ token: gitToken },
|
||||
`PR #${pr.number}`
|
||||
);
|
||||
} catch (e) {
|
||||
// on the webhook race, check whether the PR still matches what we
|
||||
// dispatched on. if it's been closed/merged or the head SHA moved,
|
||||
// no amount of retrying will populate the expected ref — surface a
|
||||
// clean abort error instead of burning the full retry budget.
|
||||
const msg = e instanceof Error ? e.message : String(e);
|
||||
if (PULL_REF_MISSING_PATTERN.test(msg)) {
|
||||
await abortIfPullRequestMoved({ octokit, owner, repo: name, pr });
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
},
|
||||
{
|
||||
delaysMs: PULL_REF_RETRY_DELAYS_MS,
|
||||
label: `pull/${pr.number}/head fetch`,
|
||||
shouldRetry: (e) =>
|
||||
PULL_REF_MISSING_PATTERN.test(e instanceof Error ? e.message : String(e)),
|
||||
}
|
||||
);
|
||||
|
||||
// checkout the branch
|
||||
$("git", ["checkout", localBranch], { log: false });
|
||||
@@ -503,186 +603,232 @@ export async function checkoutPrBranch(
|
||||
return { hookWarning: postCheckoutHook.warning };
|
||||
}
|
||||
|
||||
/**
|
||||
* dedupes concurrent `checkout_pr` calls for the same PR. agents (notably
|
||||
* Sonnet/Claude) occasionally emit duplicate parallel tool_use blocks for the
|
||||
* same args in one turn; without this, both invocations race
|
||||
* `checkoutPrBranch` against the same `.git/shallow.lock` and one fails with
|
||||
* `File exists` (issue #642). cleared in `finally` so subsequent same-PR
|
||||
* calls re-do the work normally.
|
||||
*/
|
||||
const inFlightCheckouts = new Map<number, Promise<CheckoutPrResult>>();
|
||||
|
||||
export function CheckoutPrTool(ctx: ToolContext) {
|
||||
const runCheckout = async (pull_number: number): Promise<CheckoutPrResult> => {
|
||||
const prResponse = await ctx.octokit.rest.pulls.get({
|
||||
owner: ctx.repo.owner,
|
||||
repo: ctx.repo.name,
|
||||
pull_number,
|
||||
});
|
||||
|
||||
const headRepo = prResponse.data.head.repo;
|
||||
if (!headRepo) {
|
||||
throw new Error(`PR #${pull_number} source repository was deleted`);
|
||||
}
|
||||
|
||||
const pr: PrData = {
|
||||
number: pull_number,
|
||||
headSha: prResponse.data.head.sha,
|
||||
headRef: prResponse.data.head.ref,
|
||||
headRepoFullName: headRepo.full_name,
|
||||
baseRef: prResponse.data.base.ref,
|
||||
baseRepoFullName: prResponse.data.base.repo.full_name,
|
||||
maintainerCanModify: prResponse.data.maintainer_can_modify,
|
||||
};
|
||||
|
||||
const checkoutResult = await checkoutPrBranch(pr, {
|
||||
octokit: ctx.octokit,
|
||||
owner: ctx.repo.owner,
|
||||
name: ctx.repo.name,
|
||||
gitToken: ctx.gitToken,
|
||||
toolState: ctx.toolState,
|
||||
shell: ctx.payload.shell,
|
||||
postCheckoutScript: ctx.postCheckoutScript,
|
||||
beforeSha: ctx.toolState.beforeSha,
|
||||
});
|
||||
|
||||
const tempDir = process.env.PULLFROG_TEMP_DIR;
|
||||
if (!tempDir) {
|
||||
throw new Error(
|
||||
"PULLFROG_TEMP_DIR not set - checkout_pr must run in pullfrog action context"
|
||||
);
|
||||
}
|
||||
|
||||
const headShort = ctx.toolState.checkoutSha!.slice(0, 7);
|
||||
|
||||
// compute incremental diff if we have a beforeSha to compare against
|
||||
let incrementalDiffPath: string | undefined;
|
||||
if (ctx.toolState.beforeSha && ctx.toolState.checkoutSha) {
|
||||
const beforeShort = ctx.toolState.beforeSha.slice(0, 7);
|
||||
const incremental = computeIncrementalDiff({
|
||||
baseBranch: pr.baseRef,
|
||||
beforeSha: ctx.toolState.beforeSha,
|
||||
headSha: ctx.toolState.checkoutSha,
|
||||
});
|
||||
if (incremental) {
|
||||
incrementalDiffPath = join(
|
||||
tempDir,
|
||||
`pr-${pull_number}-${beforeShort}-${headShort}-incremental.diff`
|
||||
);
|
||||
writeFileSync(incrementalDiffPath, incremental);
|
||||
log.info(
|
||||
`» incremental diff computed (${incremental.length} bytes) → ${incrementalDiffPath}`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// fetch PR files and format with line numbers
|
||||
const formatResult = await fetchAndFormatPrDiff(ctx, pull_number);
|
||||
const diffPreview = formatResult.content.split("\n").slice(0, 100).join("\n");
|
||||
log.debug(`formatted diff preview (first 100 lines):\n${diffPreview}`);
|
||||
const diffPath = join(tempDir, `pr-${pull_number}-${headShort}.diff`);
|
||||
writeFileSync(diffPath, formatResult.content);
|
||||
log.debug(`wrote diff to ${diffPath} (${formatResult.content.length} bytes)`);
|
||||
ctx.toolState.diffCoverage = createDiffCoverageState({
|
||||
diffPath,
|
||||
totalLines: countLines({ content: formatResult.content }),
|
||||
toc: formatResult.toc,
|
||||
previous: ctx.toolState.diffCoverage,
|
||||
});
|
||||
log.debug(
|
||||
`» diff coverage initialized: diffPath=${diffPath}, totalLines=${ctx.toolState.diffCoverage.totalLines}, tocEntries=${ctx.toolState.diffCoverage.tocEntries.length}`
|
||||
);
|
||||
|
||||
// cache commentable-lines snapshot so review-time validation matches what
|
||||
// GitHub will anchor to (commit_id=checkoutSha), even if the PR is updated
|
||||
// between checkout and review.
|
||||
const cached = new Map<string, ReturnType<typeof commentableLinesForFile>>();
|
||||
for (const file of formatResult.files) {
|
||||
cached.set(file.filename, commentableLinesForFile(file.patch));
|
||||
}
|
||||
ctx.toolState.commentableLinesByFile = cached;
|
||||
ctx.toolState.commentableLinesPullNumber = pull_number;
|
||||
ctx.toolState.commentableLinesCheckoutSha = ctx.toolState.checkoutSha;
|
||||
|
||||
const incrementalInstructions = incrementalDiffPath
|
||||
? ` IMPORTANT: incrementalDiffPath contains ONLY the changes since the last reviewed version ` +
|
||||
`(computed via range-diff). you MUST read incrementalDiffPath FIRST to understand what changed, ` +
|
||||
`then use diffPath for full PR context. do NOT skip the incremental diff.`
|
||||
: "";
|
||||
|
||||
// commit metadata relative to the PR base (e.g. main). use origin/<base>
|
||||
// because the local base ref may not exist after a shallow fetch. cap
|
||||
// the log so a PR with thousands of commits doesn't blow up the tool
|
||||
// response. if the base ref can't be resolved (e.g. shallow fetch that
|
||||
// didn't pull down origin/<base>), degrade gracefully rather than
|
||||
// failing the whole checkout_pr call over metadata.
|
||||
const COMMIT_LOG_MAX = 200;
|
||||
const baseRange = `origin/${pr.baseRef}..HEAD`;
|
||||
let commitCount = 0;
|
||||
let commitLog = "";
|
||||
let commitLogUnavailable = false;
|
||||
try {
|
||||
commitCount = parseInt(
|
||||
$("git", ["rev-list", "--count", baseRange], { log: false }).trim() || "0",
|
||||
10
|
||||
);
|
||||
commitLog = $("git", ["log", "--oneline", `--max-count=${COMMIT_LOG_MAX}`, baseRange], {
|
||||
log: false,
|
||||
});
|
||||
} catch (err) {
|
||||
commitLogUnavailable = true;
|
||||
log.debug(
|
||||
`» unable to compute commit metadata for ${baseRange}: ${err instanceof Error ? err.message : String(err)}`
|
||||
);
|
||||
}
|
||||
const commitLogTruncated = commitCount > COMMIT_LOG_MAX;
|
||||
|
||||
const hookWarningInstructions = checkoutResult.hookWarning
|
||||
? ` HOOK WARNING: the post-checkout lifecycle hook reported a non-fatal failure (see hookWarning). ` +
|
||||
`decide whether to retry based on the guidance in that field before proceeding.`
|
||||
: "";
|
||||
|
||||
const commitLogInstructions = commitLogUnavailable
|
||||
? ` NOTE: commit metadata is partial (base ref unreachable, likely a shallow fetch). ` +
|
||||
`commitCount/commitLog may be 0/empty or incomplete; treat them as "unknown" rather than "no commits", ` +
|
||||
`and use \`git log\` directly if you need the full history.`
|
||||
: commitLogTruncated
|
||||
? ` NOTE: commitLog was capped at ${COMMIT_LOG_MAX} entries out of ${commitCount} commits; ` +
|
||||
`use \`git log\` directly if you need the full history.`
|
||||
: "";
|
||||
|
||||
return {
|
||||
success: true,
|
||||
number: prResponse.data.number,
|
||||
title: prResponse.data.title,
|
||||
body: prResponse.data.body,
|
||||
base: pr.baseRef,
|
||||
localBranch: `pr-${pull_number}`,
|
||||
remoteBranch: `refs/heads/${pr.headRef}`,
|
||||
isFork: pr.headRepoFullName !== pr.baseRepoFullName,
|
||||
maintainerCanModify: pr.maintainerCanModify,
|
||||
url: prResponse.data.html_url,
|
||||
headRepo: pr.headRepoFullName,
|
||||
diffPath,
|
||||
incrementalDiffPath,
|
||||
toc: formatResult.toc,
|
||||
commitCount,
|
||||
commitLog,
|
||||
commitLogTruncated,
|
||||
commitLogUnavailable,
|
||||
hookWarning: checkoutResult.hookWarning,
|
||||
instructions:
|
||||
`the diff file at diffPath contains a table of contents (TOC) at the top listing every changed file with its line range. ` +
|
||||
`use the TOC line ranges as your checklist and read specific files from the diff instead of reading the entire file. ` +
|
||||
`for example, if the TOC says "src/foo.ts → lines 5-42", read lines 5-42 from diffPath to see that file's changes. ` +
|
||||
`review files selectively based on relevance rather than reading everything sequentially. ` +
|
||||
`to inspect the PR's changed files, use diffPath — do NOT run \`git diff <base>..<head>\` to re-derive what's already in diffPath. the formatted diff with line numbers is authoritative. ` +
|
||||
`\`git log\` and \`git diff --stat\` are fine for commit-range overview, and \`git diff\` / \`git diff --cached\` are fine for inspecting *your own* uncommitted changes — but PR review content MUST come from diffPath. ` +
|
||||
`before your review is submitted, a one-time coverage pre-flight may error listing unread TOC regions. ` +
|
||||
`retry the same create_pull_request_review call to proceed — optionally after reading the listed ranges. the pre-flight will not block again this session. ` +
|
||||
`the local branch is 'localBranch' (pr-{number}), not the remote branch name. ` +
|
||||
`when pushing, omit branchName to use the current branch. do not use remoteBranch as a local branch name.` +
|
||||
incrementalInstructions +
|
||||
hookWarningInstructions +
|
||||
commitLogInstructions,
|
||||
} satisfies CheckoutPrResult;
|
||||
};
|
||||
|
||||
return tool({
|
||||
name: "checkout_pr",
|
||||
description:
|
||||
"Checkout a pull request branch locally. This fetches the PR branch and sets up push configuration for fork PRs. " +
|
||||
"Returns diffPath pointing to the formatted diff file.",
|
||||
"Returns diffPath pointing to the formatted diff file. " +
|
||||
"Example: `checkout_pr({ pull_number: 1234 })`. " +
|
||||
"Transient fetch timeouts are common — retry the same call up to a few times before treating the failure as terminal. " +
|
||||
"If the error mentions `.git/shallow.lock: File exists` or `.git/index.lock: File exists`, that's a stale lock from a prior timed-out fetch — remove it via the shell tool (`rm -f .git/shallow.lock .git/index.lock`) and retry.",
|
||||
parameters: CheckoutPr,
|
||||
execute: execute(async ({ pull_number }) => {
|
||||
const prResponse = await ctx.octokit.rest.pulls.get({
|
||||
owner: ctx.repo.owner,
|
||||
repo: ctx.repo.name,
|
||||
pull_number,
|
||||
});
|
||||
|
||||
const headRepo = prResponse.data.head.repo;
|
||||
if (!headRepo) {
|
||||
throw new Error(`PR #${pull_number} source repository was deleted`);
|
||||
const inFlight = inFlightCheckouts.get(pull_number);
|
||||
if (inFlight) {
|
||||
log.info(`» checkout_pr({pull_number:${pull_number}}) already in flight — sharing result`);
|
||||
return inFlight;
|
||||
}
|
||||
|
||||
const pr: PrData = {
|
||||
number: pull_number,
|
||||
headSha: prResponse.data.head.sha,
|
||||
headRef: prResponse.data.head.ref,
|
||||
headRepoFullName: headRepo.full_name,
|
||||
baseRef: prResponse.data.base.ref,
|
||||
baseRepoFullName: prResponse.data.base.repo.full_name,
|
||||
maintainerCanModify: prResponse.data.maintainer_can_modify,
|
||||
};
|
||||
|
||||
const checkoutResult = await checkoutPrBranch(pr, {
|
||||
octokit: ctx.octokit,
|
||||
owner: ctx.repo.owner,
|
||||
name: ctx.repo.name,
|
||||
gitToken: ctx.gitToken,
|
||||
toolState: ctx.toolState,
|
||||
shell: ctx.payload.shell,
|
||||
postCheckoutScript: ctx.postCheckoutScript,
|
||||
beforeSha: ctx.toolState.beforeSha,
|
||||
});
|
||||
|
||||
const tempDir = process.env.PULLFROG_TEMP_DIR;
|
||||
if (!tempDir) {
|
||||
throw new Error(
|
||||
"PULLFROG_TEMP_DIR not set - checkout_pr must run in pullfrog action context"
|
||||
);
|
||||
}
|
||||
|
||||
const headShort = ctx.toolState.checkoutSha!.slice(0, 7);
|
||||
|
||||
// compute incremental diff if we have a beforeSha to compare against
|
||||
let incrementalDiffPath: string | undefined;
|
||||
if (ctx.toolState.beforeSha && ctx.toolState.checkoutSha) {
|
||||
const beforeShort = ctx.toolState.beforeSha.slice(0, 7);
|
||||
const incremental = computeIncrementalDiff({
|
||||
baseBranch: pr.baseRef,
|
||||
beforeSha: ctx.toolState.beforeSha,
|
||||
headSha: ctx.toolState.checkoutSha,
|
||||
});
|
||||
if (incremental) {
|
||||
incrementalDiffPath = join(
|
||||
tempDir,
|
||||
`pr-${pull_number}-${beforeShort}-${headShort}-incremental.diff`
|
||||
);
|
||||
writeFileSync(incrementalDiffPath, incremental);
|
||||
log.info(
|
||||
`» incremental diff computed (${incremental.length} bytes) → ${incrementalDiffPath}`
|
||||
// refuse to clobber an uncommitted tree whenever this call would move
|
||||
// HEAD away from the target pr-N branch. keyed off the live current
|
||||
// branch (not toolState.issueNumber, which is also written by
|
||||
// get_issue / get_issue_comments / get_issue_events and so doesn't
|
||||
// mean "currently checked out"). catches the subagent-sharing-cwd
|
||||
// case from zed-industries/cloud (2026-05-18).
|
||||
const currentBranch = $("git", ["rev-parse", "--abbrev-ref", "HEAD"], { log: false }).trim();
|
||||
if (currentBranch !== `pr-${pull_number}`) {
|
||||
const dirty = $("git", ["status", "--porcelain"], { log: false }).trim();
|
||||
if (dirty) {
|
||||
throw new Error(
|
||||
`cannot checkout PR #${pull_number} while the working tree has uncommitted changes. ` +
|
||||
`commit, push, or discard them before switching. dirty paths:\n${dirty}`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// fetch PR files and format with line numbers
|
||||
const formatResult = await fetchAndFormatPrDiff(ctx, pull_number);
|
||||
const diffPreview = formatResult.content.split("\n").slice(0, 100).join("\n");
|
||||
log.debug(`formatted diff preview (first 100 lines):\n${diffPreview}`);
|
||||
const diffPath = join(tempDir, `pr-${pull_number}-${headShort}.diff`);
|
||||
writeFileSync(diffPath, formatResult.content);
|
||||
log.debug(`wrote diff to ${diffPath} (${formatResult.content.length} bytes)`);
|
||||
ctx.toolState.diffCoverage = createDiffCoverageState({
|
||||
diffPath,
|
||||
totalLines: countLines({ content: formatResult.content }),
|
||||
toc: formatResult.toc,
|
||||
previous: ctx.toolState.diffCoverage,
|
||||
});
|
||||
log.debug(
|
||||
`» diff coverage initialized: diffPath=${diffPath}, totalLines=${ctx.toolState.diffCoverage.totalLines}, tocEntries=${ctx.toolState.diffCoverage.tocEntries.length}`
|
||||
);
|
||||
|
||||
// cache commentable-lines snapshot so review-time validation matches what
|
||||
// GitHub will anchor to (commit_id=checkoutSha), even if the PR is updated
|
||||
// between checkout and review.
|
||||
const cached = new Map<string, ReturnType<typeof commentableLinesForFile>>();
|
||||
for (const file of formatResult.files) {
|
||||
cached.set(file.filename, commentableLinesForFile(file.patch));
|
||||
}
|
||||
ctx.toolState.commentableLinesByFile = cached;
|
||||
ctx.toolState.commentableLinesPullNumber = pull_number;
|
||||
ctx.toolState.commentableLinesCheckoutSha = ctx.toolState.checkoutSha;
|
||||
|
||||
const incrementalInstructions = incrementalDiffPath
|
||||
? ` IMPORTANT: incrementalDiffPath contains ONLY the changes since the last reviewed version ` +
|
||||
`(computed via range-diff). you MUST read incrementalDiffPath FIRST to understand what changed, ` +
|
||||
`then use diffPath for full PR context. do NOT skip the incremental diff.`
|
||||
: "";
|
||||
|
||||
// commit metadata relative to the PR base (e.g. main). use origin/<base>
|
||||
// because the local base ref may not exist after a shallow fetch. cap
|
||||
// the log so a PR with thousands of commits doesn't blow up the tool
|
||||
// response. if the base ref can't be resolved (e.g. shallow fetch that
|
||||
// didn't pull down origin/<base>), degrade gracefully rather than
|
||||
// failing the whole checkout_pr call over metadata.
|
||||
const COMMIT_LOG_MAX = 200;
|
||||
const baseRange = `origin/${pr.baseRef}..HEAD`;
|
||||
let commitCount = 0;
|
||||
let commitLog = "";
|
||||
let commitLogUnavailable = false;
|
||||
const promise = runCheckout(pull_number);
|
||||
inFlightCheckouts.set(pull_number, promise);
|
||||
try {
|
||||
commitCount = parseInt(
|
||||
$("git", ["rev-list", "--count", baseRange], { log: false }).trim() || "0",
|
||||
10
|
||||
);
|
||||
commitLog = $("git", ["log", "--oneline", `--max-count=${COMMIT_LOG_MAX}`, baseRange], {
|
||||
log: false,
|
||||
});
|
||||
} catch (err) {
|
||||
commitLogUnavailable = true;
|
||||
log.debug(
|
||||
`» unable to compute commit metadata for ${baseRange}: ${err instanceof Error ? err.message : String(err)}`
|
||||
);
|
||||
return await promise;
|
||||
} finally {
|
||||
inFlightCheckouts.delete(pull_number);
|
||||
}
|
||||
const commitLogTruncated = commitCount > COMMIT_LOG_MAX;
|
||||
|
||||
const hookWarningInstructions = checkoutResult.hookWarning
|
||||
? ` HOOK WARNING: the post-checkout lifecycle hook reported a non-fatal failure (see hookWarning). ` +
|
||||
`decide whether to retry based on the guidance in that field before proceeding.`
|
||||
: "";
|
||||
|
||||
const commitLogInstructions = commitLogUnavailable
|
||||
? ` NOTE: commit metadata is partial (base ref unreachable, likely a shallow fetch). ` +
|
||||
`commitCount/commitLog may be 0/empty or incomplete; treat them as "unknown" rather than "no commits", ` +
|
||||
`and use \`git log\` directly if you need the full history.`
|
||||
: commitLogTruncated
|
||||
? ` NOTE: commitLog was capped at ${COMMIT_LOG_MAX} entries out of ${commitCount} commits; ` +
|
||||
`use \`git log\` directly if you need the full history.`
|
||||
: "";
|
||||
|
||||
return {
|
||||
success: true,
|
||||
number: prResponse.data.number,
|
||||
title: prResponse.data.title,
|
||||
body: prResponse.data.body,
|
||||
base: pr.baseRef,
|
||||
localBranch: `pr-${pull_number}`,
|
||||
remoteBranch: `refs/heads/${pr.headRef}`,
|
||||
isFork: pr.headRepoFullName !== pr.baseRepoFullName,
|
||||
maintainerCanModify: pr.maintainerCanModify,
|
||||
url: prResponse.data.html_url,
|
||||
headRepo: pr.headRepoFullName,
|
||||
diffPath,
|
||||
incrementalDiffPath,
|
||||
toc: formatResult.toc,
|
||||
commitCount,
|
||||
commitLog,
|
||||
commitLogTruncated,
|
||||
commitLogUnavailable,
|
||||
hookWarning: checkoutResult.hookWarning,
|
||||
instructions:
|
||||
`the diff file at diffPath contains a table of contents (TOC) at the top listing every changed file with its line range. ` +
|
||||
`use the TOC line ranges as your checklist and read specific files from the diff instead of reading the entire file. ` +
|
||||
`for example, if the TOC says "src/foo.ts → lines 5-42", read lines 5-42 from diffPath to see that file's changes. ` +
|
||||
`review files selectively based on relevance rather than reading everything sequentially. ` +
|
||||
`to inspect the PR's changed files, use diffPath — do NOT run \`git diff <base>..<head>\` to re-derive what's already in diffPath. the formatted diff with line numbers is authoritative. ` +
|
||||
`\`git log\` and \`git diff --stat\` are fine for commit-range overview, and \`git diff\` / \`git diff --cached\` are fine for inspecting *your own* uncommitted changes — but PR review content MUST come from diffPath. ` +
|
||||
`before your review is submitted, a one-time coverage pre-flight may error listing unread TOC regions. ` +
|
||||
`retry the same create_pull_request_review call to proceed — optionally after reading the listed ranges. the pre-flight will not block again this session. ` +
|
||||
`the local branch is 'localBranch' (pr-{number}), not the remote branch name. ` +
|
||||
`when pushing, omit branchName to use the current branch. do not use remoteBranch as a local branch name.` +
|
||||
incrementalInstructions +
|
||||
hookWarningInstructions +
|
||||
commitLogInstructions,
|
||||
} satisfies CheckoutPrResult;
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
+87
-4
@@ -33,6 +33,7 @@ function buildCommentFooter(ctx: ToolContext, customParts?: string[]): string {
|
||||
: undefined,
|
||||
customParts,
|
||||
model: ctx.toolState.model,
|
||||
fallbackFrom: ctx.toolState.modelFallback?.from,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -65,7 +66,9 @@ export function CreateCommentTool(ctx: ToolContext) {
|
||||
return tool({
|
||||
name: "create_issue_comment",
|
||||
description:
|
||||
"Create a comment on a GitHub issue or PR. For progress/plan updates on the current run use report_progress instead. Use type: 'Plan' for plan comments.",
|
||||
"Create a comment on a GitHub issue or PR. " +
|
||||
'Example: `create_issue_comment({ issueNumber: 1234, body: "Thanks for the report." })`. ' +
|
||||
"For progress/plan updates on the current run use report_progress instead — plan output (initial post AND revisions) is always posted via report_progress, never via this tool.",
|
||||
parameters: Comment,
|
||||
execute: execute(async ({ issueNumber, body, type: commentType }) => {
|
||||
const bodyWithFooter = addFooter(ctx, body);
|
||||
@@ -78,6 +81,7 @@ export function CreateCommentTool(ctx: ToolContext) {
|
||||
});
|
||||
|
||||
ctx.toolState.wasUpdated = true;
|
||||
log.info(`» created comment ${result.data.id}`);
|
||||
|
||||
if (commentType === "Plan") {
|
||||
if (result.data.node_id) {
|
||||
@@ -94,6 +98,7 @@ export function CreateCommentTool(ctx: ToolContext) {
|
||||
comment_id: result.data.id,
|
||||
body: bodyWithPlanLink,
|
||||
});
|
||||
log.info(`» updated comment ${updateResult.data.id}`);
|
||||
|
||||
return {
|
||||
success: true,
|
||||
@@ -132,6 +137,7 @@ export function EditCommentTool(ctx: ToolContext) {
|
||||
comment_id: commentId,
|
||||
body: bodyWithFooter,
|
||||
});
|
||||
log.info(`» updated comment ${result.data.id}`);
|
||||
|
||||
return {
|
||||
success: true,
|
||||
@@ -147,7 +153,7 @@ export function EditCommentTool(ctx: ToolContext) {
|
||||
export const ReportProgress = type({
|
||||
body: type.string.describe("the progress update content to share"),
|
||||
"target_plan_comment?": type("boolean").describe(
|
||||
"when true, update the existing plan comment (from select_mode lookup) instead of the progress comment; use when editing an existing plan"
|
||||
"for revising an existing plan comment ONLY. set to true only when the PlanEdit checklist from select_mode tells you to (i.e. a prior plan comment was found for this issue). NEVER set on the initial plan post — the initial plan reuses the run's progress comment and is posted by calling report_progress without this flag."
|
||||
),
|
||||
});
|
||||
|
||||
@@ -307,7 +313,9 @@ export function ReportProgressTool(ctx: ToolContext) {
|
||||
return tool({
|
||||
name: "report_progress",
|
||||
description:
|
||||
"Share progress on the associated GitHub issue/PR. The first call creates a comment; subsequent calls update it in place. Call this at the end of every run with a brief final summary (1-3 sentences) unless the mode guidance instructs otherwise. The current task list is automatically appended in a collapsible section — do not restate individual steps.",
|
||||
"Share progress on the associated GitHub issue/PR. The first call creates a comment; subsequent calls update it in place. " +
|
||||
'Example: `report_progress({ body: "Implemented the auth check and added tests." })`. ' +
|
||||
"Call this at the end of every run with a brief final summary (1-3 sentences) unless the mode guidance instructs otherwise. The current task list is automatically appended in a collapsible section — do not restate individual steps.",
|
||||
parameters: ReportProgress,
|
||||
execute: execute(async (params) => {
|
||||
let body = params.body;
|
||||
@@ -339,6 +347,10 @@ export function ReportProgressTool(ctx: ToolContext) {
|
||||
};
|
||||
}
|
||||
|
||||
if (result.commentId !== undefined) {
|
||||
log.info(`» ${result.action} comment ${result.commentId}`);
|
||||
}
|
||||
|
||||
if (!params.target_plan_comment) {
|
||||
ctx.toolState.finalSummaryWritten = true;
|
||||
}
|
||||
@@ -391,15 +403,77 @@ export const ReplyToReviewComment = type({
|
||||
),
|
||||
});
|
||||
|
||||
/**
|
||||
* decision returned by `duplicateReplyDecision` when a session has already
|
||||
* posted an identical reply to the same parent review comment.
|
||||
*/
|
||||
export interface DuplicateReplyDecision {
|
||||
kind: "already-replied";
|
||||
commentId: number;
|
||||
url: string | undefined;
|
||||
reason: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* decide whether a second reply_to_review_comment call in the same session
|
||||
* is a duplicate of an earlier reply to the same parent comment.
|
||||
*
|
||||
* the agent is instructed to call reply_to_review_comment exactly once per
|
||||
* parent comment per AddressReviews session, but in practice it sometimes
|
||||
* emits the same call twice. PR #610 reproduced this with Kimi K2:
|
||||
* identical body posted 3 seconds apart, only one tool_use event in the
|
||||
* agent log. the second post is always redundant and clutters the PR thread.
|
||||
*
|
||||
* we key on (comment_id, bodyWithFooter) so a legitimate follow-up reply
|
||||
* with different content still goes through. within a single run the
|
||||
* footer is constant (workflow run + model + jobId), so byte-equal bodies
|
||||
* catch the stutter without blocking real follow-ups.
|
||||
*
|
||||
* mirrors the shape of `duplicateReviewDecision` in mcp/review.ts.
|
||||
*/
|
||||
export function duplicateReplyDecision(params: {
|
||||
existing: { commentId: number; url: string | undefined; bodyWithFooter: string } | undefined;
|
||||
bodyWithFooter: string;
|
||||
}): DuplicateReplyDecision | null {
|
||||
const existing = params.existing;
|
||||
if (!existing) return null;
|
||||
if (existing.bodyWithFooter !== params.bodyWithFooter) return null;
|
||||
return {
|
||||
kind: "already-replied",
|
||||
commentId: existing.commentId,
|
||||
url: existing.url,
|
||||
reason: `reply ${existing.commentId} with identical body was already posted in this session; ignoring duplicate call`,
|
||||
};
|
||||
}
|
||||
|
||||
export function ReplyToReviewCommentTool(ctx: ToolContext) {
|
||||
return tool({
|
||||
name: "reply_to_review_comment",
|
||||
description:
|
||||
"Reply to a PR review comment thread (NOT issue comments — this only works for inline review comments on PR diffs). Call this for EACH comment you address in AddressReviews mode. Keep replies extremely brief (1 sentence max).",
|
||||
"Reply to a PR review comment thread (NOT issue comments — this only works for inline review comments on PR diffs). " +
|
||||
'Example: `reply_to_review_comment({ pull_number: 1234, comment_id: 567890, body: "Fixed by adding a null check." })`. ' +
|
||||
"Call exactly ONCE per parent comment you address in AddressReviews mode — duplicate calls with the same body are a no-op. Keep replies extremely brief (1 sentence max).",
|
||||
parameters: ReplyToReviewComment,
|
||||
execute: execute(async ({ pull_number, comment_id, body }) => {
|
||||
const bodyWithFooter = addFooter(ctx, body);
|
||||
|
||||
// guard against duplicate reply submissions in the same session.
|
||||
// see duplicateReplyDecision for the rationale.
|
||||
const dup = duplicateReplyDecision({
|
||||
existing: ctx.toolState.reviewReplies?.get(comment_id),
|
||||
bodyWithFooter,
|
||||
});
|
||||
if (dup) {
|
||||
log.info(`skipping duplicate review reply: ${dup.reason}`);
|
||||
return {
|
||||
success: true,
|
||||
skipped: true,
|
||||
reason: dup.reason,
|
||||
commentId: dup.commentId,
|
||||
url: dup.url,
|
||||
};
|
||||
}
|
||||
|
||||
const result = await ctx.octokit.rest.pulls.createReplyForReviewComment({
|
||||
owner: ctx.repo.owner,
|
||||
repo: ctx.repo.name,
|
||||
@@ -407,11 +481,20 @@ export function ReplyToReviewCommentTool(ctx: ToolContext) {
|
||||
comment_id,
|
||||
body: bodyWithFooter,
|
||||
});
|
||||
log.info(`» created review comment ${result.data.id} (in reply to ${comment_id})`);
|
||||
|
||||
// mark progress as updated so error reporting + run-result handling know
|
||||
// a substantive write happened (used by reportErrorToComment / handleAgentResult)
|
||||
ctx.toolState.wasUpdated = true;
|
||||
|
||||
// record this reply for in-session dedupe of subsequent identical calls.
|
||||
ctx.toolState.reviewReplies ??= new Map();
|
||||
ctx.toolState.reviewReplies.set(comment_id, {
|
||||
commentId: result.data.id,
|
||||
url: result.data.html_url,
|
||||
bodyWithFooter,
|
||||
});
|
||||
|
||||
return {
|
||||
success: true,
|
||||
commentId: result.data.id,
|
||||
|
||||
+2
-1
@@ -15,7 +15,8 @@ export function CommitInfoTool(ctx: ToolContext) {
|
||||
name: "get_commit_info",
|
||||
description:
|
||||
"Retrieve commit metadata and diff via GitHub API. Use this instead of git show for reviewing commits - " +
|
||||
"it works with shallow clones and shows the actual changes in the commit. Returns diffPath pointing to formatted diff file.",
|
||||
"it works with shallow clones and shows the actual changes in the commit. Returns diffPath pointing to formatted diff file. " +
|
||||
'Example: `get_commit_info({ sha: "2a6ab5d" })`.',
|
||||
parameters: CommitInfo,
|
||||
execute: execute(async ({ sha }) => {
|
||||
const response = await ctx.octokit.rest.repos.getCommit({
|
||||
|
||||
+25
-6
@@ -176,13 +176,32 @@ export function sanitizeToolForGemini<T extends Tool<any, any>>(tool: T): T {
|
||||
}
|
||||
|
||||
/**
|
||||
* true when the effective upstream model is served by google's generative
|
||||
* language API — directly (`google/*`), via opencode (`opencode/gemini-*`),
|
||||
* or via openrouter (`openrouter/google/gemini-*`). slug-substring match
|
||||
* works because every gemini route's model id contains "gemini".
|
||||
* true when the effective upstream model is — or might become — google
|
||||
* generative language API traffic. matches:
|
||||
* - direct `google/*`, opencode `opencode/gemini-*`, openrouter
|
||||
* `openrouter/google/gemini-*` (slug substring "gemini" wins).
|
||||
* - any unresolved specifier: `undefined`, `"auto"`, or a slug that
|
||||
* didn't map through the alias registry (no `provider/` prefix).
|
||||
* these flow through the agent's own auto-select, which may land
|
||||
* on gemini *after* the MCP server has already registered tools —
|
||||
* at which point sanitization is too late to apply. erring on the
|
||||
* side of sanitizing is safe: cases 1 + 2 are universally
|
||||
* compatible JSON-Schema normalizations (enum-only → typed string,
|
||||
* collapsible const-unions → string enum); case 3 is gemini-
|
||||
* specific but only fires on non-collapsible unions, which arktype
|
||||
* does not emit for our current tool schemas. see issue #676 for
|
||||
* the prod failure that motivated this widening.
|
||||
*/
|
||||
export function isGeminiRouted(ctx: ToolContext): boolean {
|
||||
const effective = ctx.payload.proxyModel ?? ctx.resolvedModel ?? ctx.payload.model;
|
||||
if (!effective) return false;
|
||||
return effective.toLowerCase().includes("gemini");
|
||||
if (!effective) return true;
|
||||
const normalized = effective.toLowerCase();
|
||||
if (normalized.includes("gemini")) return true;
|
||||
// every concrete model resolved through the registry carries a
|
||||
// `provider/` prefix (e.g. "anthropic/claude-opus-4-7"). anything
|
||||
// without a slash is either the literal `"auto"` alias or an
|
||||
// unrecognized slug that resolveModel logged a warning for — both
|
||||
// route through the agent's late auto-select, which may pick gemini.
|
||||
if (!normalized.includes("/")) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
+130
-60
@@ -1,10 +1,11 @@
|
||||
import { regex } from "arkregex";
|
||||
import { type } from "arktype";
|
||||
import type { StoredPushDest } from "../toolState.ts";
|
||||
import { log } from "../utils/cli.ts";
|
||||
import { $git } from "../utils/gitAuth.ts";
|
||||
import { executeLifecycleHook } from "../utils/lifecycle.ts";
|
||||
import { $git, $gitFetchWithDeepen } from "../utils/gitAuth.ts";
|
||||
import { executeLifecycleHook, type LifecycleHookFailure } from "../utils/lifecycle.ts";
|
||||
import { $ } from "../utils/shell.ts";
|
||||
import type { StoredPushDest, ToolContext } from "./server.ts";
|
||||
import type { ToolContext } from "./server.ts";
|
||||
import { execute, tool } from "./shared.ts";
|
||||
|
||||
type PushDestination = {
|
||||
@@ -217,10 +218,12 @@ export function PushBranchTool(ctx: ToolContext) {
|
||||
name: "push_branch",
|
||||
description:
|
||||
"Push the current branch to the remote repository. Omit branchName to push the current branch (recommended). " +
|
||||
'Example: `push_branch({})` to push the current branch. Example: `push_branch({ branchName: "pr-1" })` to push a specific local branch. ' +
|
||||
"If specifying branchName, use the LOCAL branch name (e.g., 'pr-1'), not the remote branch name. " +
|
||||
"The correct remote and remote branch are determined automatically from branch config set by checkout_pr. " +
|
||||
"Requires a clean working tree. Runs the repository prepush hook (if configured) before the network push — hook failure means tests/lint or similar in that script failed, not necessarily a Pullfrog timeout. " +
|
||||
"Never force push unless explicitly requested. Pushes to the default branch are blocked in restricted mode.",
|
||||
"Requires a clean working tree. Runs the repository prepush hook (if configured) — best-effort. If the hook fails, the tool returns the failure output and every subsequent call this run skips the hook. " +
|
||||
"Never force push unless explicitly requested. Pushes to the default branch are blocked in restricted mode. " +
|
||||
"If the response reports a timeout, the underlying push may have actually succeeded — verify with `git log origin/<branch>` (or this tool with command 'log') before retrying, otherwise you'll push a duplicate.",
|
||||
parameters: PushBranch,
|
||||
execute: execute(async ({ branchName, force }) => {
|
||||
// permission check
|
||||
@@ -240,13 +243,38 @@ export function PushBranchTool(ctx: ToolContext) {
|
||||
if (status) {
|
||||
throw new Error(
|
||||
`push blocked: working tree is not clean (tracked changes and/or untracked files). commit, discard, or remove stray artifacts before pushing.\n\n` +
|
||||
`git status:\n${status}`
|
||||
`git status:\n${status}` +
|
||||
(ctx.toolState.prepushFailureCount > 0
|
||||
? "\n\nnote: the prepush hook failed earlier this run — once the working tree is clean, push_branch will skip the hook."
|
||||
: "")
|
||||
);
|
||||
}
|
||||
|
||||
// validate push destination matches expected URL
|
||||
const pushDest = validatePushDestination(ctx, branch);
|
||||
|
||||
// backstop against subagent-induced cross-PR clobbers: a subagent
|
||||
// shares cwd + toolState with the orchestrator, so its `checkout_pr(N)`
|
||||
// moves HEAD to pr-N and persists pushDest pointing at the foreign
|
||||
// PR's remote branch. refuse pr-N → origin/<other> pushes unless this
|
||||
// run is itself scoped to PR N (zed-industries/cloud, 2026-05-18).
|
||||
const prBranchMatch = branch.match(/^pr-(\d+)$/);
|
||||
if (prBranchMatch && pushDest.remoteBranch !== branch) {
|
||||
const prNumber = Number(prBranchMatch[1]);
|
||||
const event = ctx.payload.event;
|
||||
const runScoped = event.is_pr === true && event.issue_number === prNumber;
|
||||
if (!runScoped) {
|
||||
throw new Error(
|
||||
`push blocked: local branch '${branch}' would push to '${pushDest.remoteName}/${pushDest.remoteBranch}', ` +
|
||||
`but this run is not scoped to PR #${prNumber}. ` +
|
||||
`the 'pr-${prNumber}' branch was created by a prior checkout_pr call (likely from a subagent — subagents share the working tree and toolState with the orchestrator). ` +
|
||||
`you have probably landed your commit on the wrong branch. ` +
|
||||
`switch to your own feature branch first (e.g. 'git checkout <feature-branch>') and then push. ` +
|
||||
`if the push to PR #${prNumber} is intentional, this run needs to be triggered against that PR.`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// block pushes to default branch in restricted mode
|
||||
if (pushPermission === "restricted" && pushDest.remoteBranch === defaultBranch) {
|
||||
throw new Error(
|
||||
@@ -262,27 +290,31 @@ export function PushBranchTool(ctx: ToolContext) {
|
||||
? ["--force", "-u", pushDest.remoteName, refspec]
|
||||
: ["-u", pushDest.remoteName, refspec];
|
||||
|
||||
// prepush failure should block the push — a passing hook is the gate
|
||||
// that protects main from bad pushes.
|
||||
const prepushHook = await executeLifecycleHook({
|
||||
event: "prepush",
|
||||
script: ctx.prepushScript,
|
||||
});
|
||||
if (prepushHook.warning) {
|
||||
throw new Error(prepushHook.warning);
|
||||
}
|
||||
const prepushSkipped = ctx.toolState.prepushFailureCount > 0;
|
||||
if (prepushSkipped) {
|
||||
log.info(`» skipping prepush hook (failed earlier this run)`);
|
||||
} else if (ctx.prepushScript) {
|
||||
const prepushHook = await executeLifecycleHook({
|
||||
event: "prepush",
|
||||
script: ctx.prepushScript,
|
||||
});
|
||||
if (prepushHook.failure) {
|
||||
ctx.toolState.prepushFailureCount += 1;
|
||||
throw new Error(buildPrepushFailureMessage(prepushHook.failure, ctx.payload.shell));
|
||||
}
|
||||
|
||||
// re-verify clean working tree after prepush. a hook that writes tracked
|
||||
// files (formatter, type generator, build artifacts) would leave those
|
||||
// changes uncommitted — pushing now would silently drop them, and the
|
||||
// agent would report a "successful push" of code the hook had expected
|
||||
// to be included.
|
||||
const postHookStatus = $("git", ["status", "--porcelain"], { log: false });
|
||||
if (postHookStatus) {
|
||||
throw new Error(
|
||||
`push blocked: the prepush hook modified the working tree. those changes are not included in the push. commit or discard them (or change the hook to not mutate tracked files) before retrying.\n\n` +
|
||||
`git status:\n${postHookStatus}`
|
||||
);
|
||||
// re-verify clean working tree after prepush. a hook that writes tracked
|
||||
// files (formatter, type generator, build artifacts) would leave those
|
||||
// changes uncommitted — pushing now would silently drop them, and the
|
||||
// agent would report a "successful push" of code the hook had expected
|
||||
// to be included.
|
||||
const postHookStatus = $("git", ["status", "--porcelain"], { log: false });
|
||||
if (postHookStatus) {
|
||||
throw new Error(
|
||||
`push blocked: the prepush hook modified the working tree. those changes are not included in the push. commit or discard them (or change the hook to not mutate tracked files) before retrying.\n\n` +
|
||||
`git status:\n${postHookStatus}`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
log.debug(`pushing ${branch} to ${pushDest.remoteName}/${pushDest.remoteBranch}`);
|
||||
@@ -351,18 +383,55 @@ export function PushBranchTool(ctx: ToolContext) {
|
||||
throw lastErr instanceof Error ? lastErr : new Error(String(lastErr));
|
||||
}
|
||||
|
||||
const pushedSha = $("git", ["rev-parse", "HEAD"], { log: false }).trim();
|
||||
log.info(
|
||||
`» pushed branch ${branch} to ${pushDest.remoteName}/${pushDest.remoteBranch} (sha ${pushedSha})`
|
||||
);
|
||||
|
||||
const baseMsg = `successfully pushed ${branch} to ${pushDest.remoteName}/${pushDest.remoteBranch}`;
|
||||
const message = prepushSkipped
|
||||
? `${baseMsg} (prepush hook skipped — failed earlier this run).`
|
||||
: baseMsg;
|
||||
|
||||
return {
|
||||
success: true,
|
||||
branch,
|
||||
remoteBranch: pushDest.remoteBranch,
|
||||
remote: pushDest.remoteName,
|
||||
force,
|
||||
message: `successfully pushed ${branch} to ${pushDest.remoteName}/${pushDest.remoteBranch}`,
|
||||
prepushSkipped,
|
||||
message,
|
||||
};
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
/** agent-facing prepush failure message: script output + bypass guidance,
|
||||
* with no generic lifecycle retry advice (which would conflict). */
|
||||
function buildPrepushFailureMessage(
|
||||
failure: LifecycleHookFailure,
|
||||
shell: ToolContext["payload"]["shell"]
|
||||
): string {
|
||||
const header =
|
||||
failure.kind === "exit"
|
||||
? `prepush hook failed with exit code ${failure.exitCode}.\n\nscript output:\n${failure.output || "(empty)"}`
|
||||
: failure.kind === "timeout"
|
||||
? `prepush hook timed out — the script is hung or doing too much work.`
|
||||
: `prepush hook failed to spawn: ${failure.spawnError}.`;
|
||||
|
||||
const ifRealBug =
|
||||
shell === "disabled"
|
||||
? `fix it before pushing again — shell access is disabled in this run, so you can't re-run the hook command yourself.`
|
||||
: `run the hook command yourself via the shell tool to iterate (push_branch will NOT re-run it).`;
|
||||
|
||||
return (
|
||||
`${header}\n\n` +
|
||||
`this repo's prepush hook is best-effort: the next push_branch call will SKIP the hook and proceed. ` +
|
||||
`if the failure is unrelated to your changes (pre-existing breakage, flaky check), just call push_branch again. ` +
|
||||
`if it could be a real bug in your code, ${ifRealBug}`
|
||||
);
|
||||
}
|
||||
|
||||
// commands that require authentication - redirect to dedicated tools.
|
||||
// exported so tests can exercise the same table the runtime uses.
|
||||
//
|
||||
@@ -445,7 +514,10 @@ export function GitTool(ctx: ToolContext) {
|
||||
return tool({
|
||||
name: "git",
|
||||
description:
|
||||
"Run git commands. For push/fetch, use the dedicated MCP tools (push_branch, git_fetch). " +
|
||||
"Run a git subcommand. `command` is a single subcommand; flags and positional args go in `args`. " +
|
||||
'Example: `git({ command: "log", args: ["--oneline", "-n", "20"] })`. ' +
|
||||
'Example: `git({ command: "diff", args: ["origin/main..HEAD"] })`. ' +
|
||||
"For push/fetch, use the dedicated MCP tools (push_branch, git_fetch). " +
|
||||
"git pull is not available — use git_fetch then this tool with command 'merge'.",
|
||||
parameters: Git,
|
||||
execute: execute(async (params) => {
|
||||
@@ -479,6 +551,30 @@ export function GitTool(ctx: ToolContext) {
|
||||
}
|
||||
}
|
||||
|
||||
// `git merge-base --is-ancestor` uses exit codes as data: 0 = ancestor,
|
||||
// 1 = not-an-ancestor, >1 = real error. Surface the binary answer
|
||||
// instead of throwing on exit 1. see #766.
|
||||
if (command === "merge-base" && args.includes("--is-ancestor")) {
|
||||
let isAncestor = true;
|
||||
$("git", [command, ...args], {
|
||||
log: false,
|
||||
onError: (r) => {
|
||||
if (r.status === 1) {
|
||||
isAncestor = false;
|
||||
return;
|
||||
}
|
||||
const detail = [r.stderr, r.stdout]
|
||||
.map((s) => s.trim())
|
||||
.filter(Boolean)
|
||||
.join("\n");
|
||||
throw new Error(
|
||||
`git merge-base --is-ancestor failed (exit ${r.status}): ${detail || "Unknown error"}`
|
||||
);
|
||||
},
|
||||
});
|
||||
return { success: true, isAncestor };
|
||||
}
|
||||
|
||||
const output = $("git", [command, ...args], { log: false });
|
||||
const lineCount = output.split("\n").length;
|
||||
if (lineCount > COLLAPSE_THRESHOLD) {
|
||||
@@ -499,25 +595,12 @@ const GitFetch = type({
|
||||
depth: type.number.describe("Fetch depth (for shallow clones)").optional(),
|
||||
});
|
||||
|
||||
// when an agent-supplied depth is too shallow to reach the merge base, git
|
||||
// surfaces "Could not read <sha>" and "remote did not send all necessary
|
||||
// objects". detect both wordings so a single deepen retry can recover before
|
||||
// the error reaches the agent (issue #564). git emits the full OID via
|
||||
// oid_to_hex, so the bound is 40 (SHA-1) or 64 (SHA-256).
|
||||
const SHALLOW_UNREACHABLE_PATTERNS: RegExp[] = [
|
||||
/Could not read [a-f0-9]{40,64}/,
|
||||
/remote did not send all necessary objects/,
|
||||
];
|
||||
|
||||
// large enough to clear the merge base on most real-world PRs without
|
||||
// downloading the full history; matches the fallback used by checkoutPrBranch
|
||||
// when the compare API is unavailable.
|
||||
const DEEPEN_RETRY_DEPTH = 1000;
|
||||
|
||||
export function GitFetchTool(ctx: ToolContext) {
|
||||
return tool({
|
||||
name: "git_fetch",
|
||||
description: "Fetch refs from remote repository. Use this instead of git fetch directly.",
|
||||
description:
|
||||
"Fetch refs from remote repository. Use this instead of git fetch directly. " +
|
||||
'Example: `git_fetch({ ref: "main" })`. With depth: `git_fetch({ ref: "pull/1234/head", depth: 1 })`.',
|
||||
parameters: GitFetch,
|
||||
execute: execute(async (params) => {
|
||||
rejectIfLeadingDash(params.ref, "ref");
|
||||
@@ -525,22 +608,7 @@ export function GitFetchTool(ctx: ToolContext) {
|
||||
if (params.depth !== undefined) {
|
||||
fetchArgs.push(`--depth=${params.depth}`);
|
||||
}
|
||||
try {
|
||||
await $git("fetch", fetchArgs, { token: ctx.gitToken });
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
const isShallowUnreachable = SHALLOW_UNREACHABLE_PATTERNS.some((p) => p.test(msg));
|
||||
const isShallow =
|
||||
isShallowUnreachable &&
|
||||
$("git", ["rev-parse", "--is-shallow-repository"], { log: false }).trim() === "true";
|
||||
if (!isShallow) throw err;
|
||||
log.info(
|
||||
`» git_fetch hit shallow-unreachable error, retrying with --deepen=${DEEPEN_RETRY_DEPTH}`
|
||||
);
|
||||
await $git("fetch", [`--deepen=${DEEPEN_RETRY_DEPTH}`, "--no-tags", "origin", params.ref], {
|
||||
token: ctx.gitToken,
|
||||
});
|
||||
}
|
||||
await $gitFetchWithDeepen(fetchArgs, { token: ctx.gitToken }, "git_fetch");
|
||||
return { success: true, ref: params.ref };
|
||||
}),
|
||||
});
|
||||
@@ -595,6 +663,7 @@ export function DeleteBranchTool(ctx: ToolContext) {
|
||||
await $git("push", ["origin", "--delete", `refs/heads/${params.branchName}`], {
|
||||
token: ctx.gitToken,
|
||||
});
|
||||
log.info(`» deleted branch ${params.branchName}`);
|
||||
return { success: true, deleted: params.branchName };
|
||||
}),
|
||||
});
|
||||
@@ -625,6 +694,7 @@ export function PushTagsTool(ctx: ToolContext) {
|
||||
await $git("push", pushArgs, {
|
||||
token: ctx.gitToken,
|
||||
});
|
||||
log.info(`» pushed tag ${params.tag}`);
|
||||
return { success: true, tag: params.tag };
|
||||
}),
|
||||
});
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { type } from "arktype";
|
||||
import { log } from "../utils/cli.ts";
|
||||
import { fixDoubleEscapedString } from "../utils/fixDoubleEscapedString.ts";
|
||||
import { patchWorkflowRunFields } from "../utils/patchWorkflowRunFields.ts";
|
||||
import type { ToolContext } from "./server.ts";
|
||||
@@ -32,6 +33,8 @@ export function IssueTool(ctx: ToolContext) {
|
||||
assignees: params.assignees ?? [],
|
||||
});
|
||||
|
||||
log.info(`» created issue #${result.data.number} (id ${result.data.id})`);
|
||||
|
||||
const nodeId = result.data.node_id;
|
||||
if (typeof nodeId === "string" && nodeId.length > 0) {
|
||||
await patchWorkflowRunFields(ctx, {
|
||||
|
||||
@@ -10,7 +10,8 @@ export function GetIssueCommentsTool(ctx: ToolContext) {
|
||||
return tool({
|
||||
name: "get_issue_comments",
|
||||
description:
|
||||
"Get all comments for a GitHub issue. Returns all comments including the issue body and all subsequent discussion comments.",
|
||||
"Get all comments for a GitHub issue. Returns all comments including the issue body and all subsequent discussion comments. " +
|
||||
"Example: `get_issue_comments({ issue_number: 1234 })`.",
|
||||
parameters: GetIssueComments,
|
||||
execute: execute(async ({ issue_number }) => {
|
||||
// set issue context
|
||||
|
||||
+5
-4
@@ -27,10 +27,11 @@ export function GetIssueEventsTool(ctx: ToolContext) {
|
||||
const relevantEventTypes = new Set(["cross_referenced", "referenced"]);
|
||||
|
||||
const parsedEvents = events.flatMap((event) => {
|
||||
// Filter to only events with an 'event' property and relevant types
|
||||
if (!("event" in event) || !relevantEventTypes.has(event.event)) {
|
||||
return [];
|
||||
}
|
||||
// octokit's timeline-event union includes members with `event?:
|
||||
// string`, so `"event" in event` does not narrow it to defined.
|
||||
// require a string before the Set.has() check.
|
||||
if (!("event" in event) || typeof event.event !== "string") return [];
|
||||
if (!relevantEventTypes.has(event.event)) return [];
|
||||
|
||||
const baseEvent: Record<string, any> = {
|
||||
event: event.event,
|
||||
|
||||
+3
-1
@@ -9,7 +9,9 @@ export const IssueInfo = type({
|
||||
export function IssueInfoTool(ctx: ToolContext) {
|
||||
return tool({
|
||||
name: "get_issue",
|
||||
description: "Retrieve GitHub issue information by issue number",
|
||||
description:
|
||||
"Retrieve GitHub issue information by issue number. " +
|
||||
"Example: `get_issue({ issue_number: 1234 })`.",
|
||||
parameters: IssueInfo,
|
||||
execute: execute(async ({ issue_number }) => {
|
||||
const issue = await ctx.octokit.rest.issues.get({
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { type } from "arktype";
|
||||
import { log } from "../utils/cli.ts";
|
||||
import type { ToolContext } from "./server.ts";
|
||||
import { execute, tool } from "./shared.ts";
|
||||
|
||||
@@ -20,6 +21,7 @@ export function AddLabelsTool(ctx: ToolContext) {
|
||||
issue_number,
|
||||
labels,
|
||||
});
|
||||
log.info(`» added labels [${labels.join(", ")}] to issue #${issue_number}`);
|
||||
|
||||
return {
|
||||
success: true,
|
||||
|
||||
@@ -1,41 +0,0 @@
|
||||
import { type } from "arktype";
|
||||
import { apiFetch } from "../utils/apiFetch.ts";
|
||||
import type { ToolContext } from "./server.ts";
|
||||
import { execute, tool } from "./shared.ts";
|
||||
|
||||
const UpdateLearningsParams = type({
|
||||
learnings: type.string.describe(
|
||||
"the FULL merged learnings as a flat bullet list. each line starts with `- `. one discrete, actionable fact per bullet. combine existing bullets from the prompt with your new discoveries. deduplicate — if an existing bullet covers the same fact, update it in place rather than adding a new one. drop bullets that are clearly wrong or no longer relevant to the current codebase. keep the list focused and concise."
|
||||
),
|
||||
});
|
||||
|
||||
export function UpdateLearningsTool(ctx: ToolContext) {
|
||||
return tool({
|
||||
name: "update_learnings",
|
||||
description:
|
||||
"persist operational learnings about this repository (setup steps, test commands, key conventions, patterns). ONLY call this when you have high confidence the information is correct and broadly useful for future runs — not for one-off findings or uncertain observations. format: flat bullet list (`- ` per line, one fact per bullet). pass the FULL merged list — combine existing learnings from the prompt with new discoveries. deduplicate, and drop bullets that are clearly wrong or no longer relevant to the current codebase.",
|
||||
parameters: UpdateLearningsParams,
|
||||
execute: execute(async (params) => {
|
||||
const response = await apiFetch({
|
||||
path: `/api/repo/${ctx.repo.owner}/${ctx.repo.name}/learnings`,
|
||||
method: "PATCH",
|
||||
headers: {
|
||||
authorization: `Bearer ${ctx.apiToken}`,
|
||||
"content-type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
learnings: params.learnings,
|
||||
model: ctx.toolState.model,
|
||||
}),
|
||||
signal: AbortSignal.timeout(10_000),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const error = await response.text();
|
||||
throw new Error(`failed to update learnings: ${error}`);
|
||||
}
|
||||
|
||||
return { success: true };
|
||||
}),
|
||||
});
|
||||
}
|
||||
@@ -23,6 +23,7 @@ function buildPrBodyWithFooter(ctx: ToolContext, body: string): string {
|
||||
? { owner: ctx.repo.owner, repo: ctx.repo.name, runId: ctx.runId, jobId: ctx.jobId }
|
||||
: undefined,
|
||||
model: ctx.toolState.model,
|
||||
fallbackFrom: ctx.toolState.modelFallback?.from,
|
||||
});
|
||||
|
||||
const bodyWithoutFooter = stripExistingFooter(fixDoubleEscapedString(body));
|
||||
@@ -48,6 +49,7 @@ export function UpdatePullRequestBodyTool(ctx: ToolContext) {
|
||||
pull_number: params.pull_number,
|
||||
body: bodyWithFooter,
|
||||
});
|
||||
log.info(`» updated pull request #${result.data.number}`);
|
||||
|
||||
ctx.toolState.wasUpdated = true;
|
||||
|
||||
@@ -80,6 +82,7 @@ export function CreatePullRequestTool(ctx: ToolContext) {
|
||||
base: params.base,
|
||||
draft: params.draft ?? false,
|
||||
});
|
||||
log.info(`» created pull request #${result.data.number} (id ${result.data.id})`);
|
||||
|
||||
// best-effort: request review from the user who triggered the workflow
|
||||
const reviewer = ctx.payload.triggerer;
|
||||
|
||||
+3
-1
@@ -30,7 +30,9 @@ export function PullRequestInfoTool(ctx: ToolContext) {
|
||||
return tool({
|
||||
name: "get_pull_request",
|
||||
description:
|
||||
"Retrieve PR metadata (title, body, state, branches, author, labels, linked issues). To checkout a PR branch locally, use checkout_pr instead.",
|
||||
"Retrieve PR metadata (title, body, state, branches, author, labels, linked issues). " +
|
||||
"Example: `get_pull_request({ pull_number: 1234 })`. " +
|
||||
"To checkout a PR branch locally, use checkout_pr instead.",
|
||||
parameters: PullRequestInfo,
|
||||
execute: execute(async ({ pull_number }) => {
|
||||
// fetch REST and GraphQL in parallel
|
||||
|
||||
+18
-6
@@ -1,6 +1,7 @@
|
||||
import type { RestEndpointMethodTypes } from "@octokit/rest";
|
||||
import { type } from "arktype";
|
||||
import { formatMcpToolRef } from "../external.ts";
|
||||
import type { CommentableLines } from "../toolState.ts";
|
||||
import { getApiUrl } from "../utils/apiUrl.ts";
|
||||
import { buildPullfrogFooter } from "../utils/buildPullfrogFooter.ts";
|
||||
import { log } from "../utils/cli.ts";
|
||||
@@ -16,6 +17,8 @@ import { deleteProgressComment } from "./comment.ts";
|
||||
import type { ToolContext } from "./server.ts";
|
||||
import { execute, tool } from "./shared.ts";
|
||||
|
||||
export type { CommentableLines };
|
||||
|
||||
function getHttpStatus(err: unknown): number | undefined {
|
||||
if (typeof err !== "object" || err === null) return undefined;
|
||||
const status = (err as Record<string, unknown>).status;
|
||||
@@ -46,7 +49,6 @@ export function isTransientReviewError(err: unknown): boolean {
|
||||
export const TRANSIENT_REVIEW_RETRY_DELAYS_MS = [1_000, 3_000];
|
||||
|
||||
type PullFile = RestEndpointMethodTypes["pulls"]["listFiles"]["response"]["data"][number];
|
||||
export type CommentableLines = { RIGHT: Set<number>; LEFT: Set<number> };
|
||||
|
||||
/**
|
||||
* parse a PR file's patch to determine which line numbers on each side are
|
||||
@@ -314,18 +316,20 @@ export const CreatePullRequestReview = type({
|
||||
.optional(),
|
||||
approved: type.boolean
|
||||
.describe(
|
||||
"Set to true to submit as an approval. ONLY when the review contains no actionable feedback — neither inline comments nor actionable content in the body. Defaults to false (comment-only review). Rejections are not supported."
|
||||
"Set to true to submit as an approval. Use for `> ✅ No new issues found.` reviews where the PR is mergeable as-is and nothing in the body warrants code changes — approving also suppresses the Fix-button footer affordance so users don't dispatch a fix run on non-actionable feedback. Reserve approved: false for `> ℹ️ ...` (minor suggestions inline), `> [!IMPORTANT]` (recommended changes), and `> [!CAUTION]` (critical) reviews. Defaults to false (comment-only review). Rejections are not supported."
|
||||
)
|
||||
.optional(),
|
||||
commit_id: type.string
|
||||
.describe("Optional SHA of the commit being reviewed. Defaults to latest.")
|
||||
.describe(
|
||||
"Optional SHA of the commit being reviewed. Defaults to latest. Must be the FULL 40-character SHA — abbreviated SHAs are rejected by GitHub with `422 Unprocessable Entity`. The PR-synchronize event payload's `head_sha` is already full-length."
|
||||
)
|
||||
.optional(),
|
||||
comments: type({
|
||||
path: type.string.describe(
|
||||
"The file path to comment on (relative to repo root). Must be a file that appears in the PR diff."
|
||||
),
|
||||
line: type.number.describe(
|
||||
"Line number to comment on. For multi-line ranges, this is the end line. Use NEW column from diff format."
|
||||
"Line number to comment on. For multi-line ranges, this is the end line. Use NEW column from diff format. Must sit inside a `@@` hunk in the PR diff — anchors on context-only or untouched lines are dropped silently (the rest of the review still posts; dropped entries are reported under `droppedComments` in the response)."
|
||||
),
|
||||
side: type
|
||||
.enumerated("LEFT", "RIGHT")
|
||||
@@ -343,7 +347,7 @@ export const CreatePullRequestReview = type({
|
||||
.optional(),
|
||||
start_line: type.number
|
||||
.describe(
|
||||
"Start line for multi-line comment ranges. Omit for single-line comments. The range [start_line, line] defines which lines a suggestion replaces."
|
||||
"Start line for multi-line comment ranges. Omit for single-line comments. The range [start_line, line] defines which lines a suggestion replaces. Both `start_line` and `line` must sit inside the same `@@` hunk — a `start_line` outside the hunk causes the whole comment to be dropped even when `line` is valid. If you need to comment on context just above/below a hunk, shrink the range to a single line that is provably modified."
|
||||
)
|
||||
.optional(),
|
||||
})
|
||||
@@ -359,6 +363,7 @@ export function CreatePullRequestReviewTool(ctx: ToolContext) {
|
||||
name: "create_pull_request_review",
|
||||
description:
|
||||
"Submit a review for an existing pull request. " +
|
||||
'Example: `create_pull_request_review({ pull_number: 1234, body: "LGTM", approved: true, comments: [{ path: "src/api.ts", line: 42, body: "nit: rename" }] })`. ' +
|
||||
"Each call creates a permanent, visible review on the PR — NEVER submit test or diagnostic reviews. " +
|
||||
"Reviews with no body AND no comments are silently skipped (nothing to post). " +
|
||||
"IMPORTANT: 95%+ of feedback should be in 'comments' array with file paths and line numbers. " +
|
||||
@@ -584,6 +589,7 @@ export function CreatePullRequestReviewTool(ctx: ToolContext) {
|
||||
}
|
||||
const reviewId = result.data.id;
|
||||
const reviewNodeId = result.data.node_id;
|
||||
log.info(`» created review ${reviewId} on pull request #${pull_number}`);
|
||||
|
||||
// reviewedSha = what the agent actually reviewed (checkout SHA), not the
|
||||
// submission anchor (current HEAD). this ensures postReviewCleanup dispatches
|
||||
@@ -706,7 +712,9 @@ function runDiffCoveragePreflight(params: { ctx: ToolContext }): void {
|
||||
.join("\n");
|
||||
throw new Error(
|
||||
`diff coverage pre-flight: some TOC regions were not read before review submission. ` +
|
||||
`this is a one-time nudge — optionally read the ranges below from ${coverageState.diffPath}, then call create_pull_request_review again with the same arguments. ` +
|
||||
`this is a one-time nudge — read the ranges below from ${coverageState.diffPath} on a best-effort basis, then call create_pull_request_review again. ` +
|
||||
`you are NOT obligated to read generated artifacts (lockfiles like pnpm-lock.yaml / package-lock.json / yarn.lock / Cargo.lock; codegen output like *.gen.*, *.pb.go, *.generated.*; snapshot/fixture dirs like __snapshots__/; migration metadata like drizzle/meta/, prisma migration SQL). ` +
|
||||
`if every unread region is generated, retry immediately without reading. ` +
|
||||
`this pre-flight will not block again in this review session.\n\n` +
|
||||
`unread TOC regions:\n${unreadText}\n\n` +
|
||||
`${coverageState.lastBreakdown}`
|
||||
@@ -841,6 +849,9 @@ async function createAndSubmitWithFooter(
|
||||
// API_URL is misconfigured, and future footer-building changes could
|
||||
// introduce new throw paths. keep the whole body wrapped.
|
||||
try {
|
||||
// Fix buttons are suppressed on approving reviews — those are mergeable
|
||||
// by definition (the `> ✅ No new issues found.` tier, with no inline
|
||||
// comments), so dispatching a fix run would be a UX trap.
|
||||
const customParts: string[] = [];
|
||||
if (!opts.approved) {
|
||||
const apiUrl = getApiUrl();
|
||||
@@ -860,6 +871,7 @@ async function createAndSubmitWithFooter(
|
||||
: undefined,
|
||||
customParts,
|
||||
model: ctx.toolState.model,
|
||||
fallbackFrom: ctx.toolState.modelFallback?.from,
|
||||
});
|
||||
|
||||
return await ctx.octokit.rest.pulls.submitReview({
|
||||
|
||||
@@ -602,6 +602,7 @@ export function GetReviewCommentsTool(ctx: ToolContext) {
|
||||
name: "get_review_comments",
|
||||
description:
|
||||
"Get review comments for a pull request review with full thread context. " +
|
||||
"Example: `get_review_comments({ pull_number: 1234, review_id: 567890 })`. " +
|
||||
"Automatically filters to approved comments when applicable. " +
|
||||
"Returns a TOC and commentsPath pointing to a markdown file with full comment details.",
|
||||
parameters: GetReviewComments,
|
||||
@@ -673,7 +674,8 @@ export function ListPullRequestReviewsTool(ctx: ToolContext) {
|
||||
return tool({
|
||||
name: "list_pull_request_reviews",
|
||||
description:
|
||||
"List all reviews for a pull request. Returns all reviews including approvals, request changes, and comments.",
|
||||
"List all reviews for a pull request. Returns all reviews including approvals, request changes, and comments. " +
|
||||
"Example: `list_pull_request_reviews({ pull_number: 1234 })`.",
|
||||
parameters: ListPullRequestReviews,
|
||||
execute: execute(async (params) => {
|
||||
const reviews = await ctx.octokit.paginate(ctx.octokit.rest.pulls.listReviews, {
|
||||
@@ -691,6 +693,8 @@ export function ListPullRequestReviewsTool(ctx: ToolContext) {
|
||||
state: review.state,
|
||||
user: review.user?.login,
|
||||
submitted_at: review.submitted_at,
|
||||
commit_id: review.commit_id,
|
||||
html_url: review.html_url,
|
||||
})),
|
||||
count: reviews.length,
|
||||
};
|
||||
@@ -735,7 +739,7 @@ export function ResolveReviewThreadTool(ctx: ToolContext) {
|
||||
});
|
||||
|
||||
const thread = response.resolveReviewThread.thread;
|
||||
log.debug(`resolved thread ${thread.id}, isResolved=${thread.isResolved}`);
|
||||
log.info(`» resolved review thread ${thread.id}`);
|
||||
|
||||
return {
|
||||
thread_id: thread.id,
|
||||
|
||||
+7
-5
@@ -24,13 +24,14 @@ function buildModeOverrides(t: (name: string) => string): Record<string, string>
|
||||
|
||||
An existing plan comment was found for this issue. Update that comment with the revised plan — do not create a new plan comment.
|
||||
|
||||
1. Use \`previousPlanBody\` from this response as the plan to revise; do not call \`get_issue\` or \`get_issue_comments\`.
|
||||
2. Revise the plan based on the user's request:
|
||||
1. **task list**: create your task list for this run as your first action.
|
||||
2. Use \`previousPlanBody\` from this response as the plan to revise; do not call \`get_issue\` or \`get_issue_comments\`.
|
||||
3. Revise the plan based on the user's request:
|
||||
- incorporate the current plan (\`previousPlanBody\`) and the user's revision request
|
||||
- gather relevant codebase context (file paths, architecture notes from AGENTS.md)
|
||||
- produce a structured plan with clear milestones
|
||||
3. Call \`${t("report_progress")}\` with the full revised plan text and \`{ target_plan_comment: true }\` so it updates the existing plan comment (not the progress comment).
|
||||
4. Then post a short note to the progress comment (e.g. "Plan has been updated in the comment above.") via \`${t("report_progress")}\` so it is not left as "Leaping...".`,
|
||||
4. Call \`${t("report_progress")}\` with the full revised plan text and \`{ target_plan_comment: true }\` so it updates the existing plan comment (not the progress comment).
|
||||
5. Then post a short note to the progress comment (e.g. "Plan has been updated in the comment above.") via \`${t("report_progress")}\` so it is not left as "Leaping...".`,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -120,7 +121,8 @@ export function SelectModeTool(ctx: ToolContext) {
|
||||
return tool({
|
||||
name: "select_mode",
|
||||
description:
|
||||
"Select a mode and receive step-by-step guidance on how to handle the task. Call this to understand the best workflow for the current mode.",
|
||||
"Select a mode and receive step-by-step guidance on how to handle the task. Call this to understand the best workflow for the current mode. " +
|
||||
'Example: `select_mode({ mode: "Review" })` or `select_mode({ mode: "Plan", issue_number: 1234 })`.',
|
||||
parameters: SelectModeParams,
|
||||
execute: execute(async (params) => {
|
||||
if (ctx.toolState.selectedMode) {
|
||||
|
||||
+5
-131
@@ -3,23 +3,14 @@ import "./arkConfig.ts";
|
||||
import { createServer } from "node:net";
|
||||
import { setTimeout as sleep } from "node:timers/promises";
|
||||
import { FastMCP, type Tool } from "fastmcp";
|
||||
import type { AgentUsage } from "../agents/index.ts";
|
||||
import { type AgentId, pullfrogMcpName } from "../external.ts";
|
||||
import type { Mode } from "../modes.ts";
|
||||
import type { PrepResult } from "../prep/index.ts";
|
||||
import type { ToolState } from "../toolState.ts";
|
||||
import { closeBrowserDaemon } from "../utils/browser.ts";
|
||||
import { log } from "../utils/cli.ts";
|
||||
import type { DiffCoverageState } from "../utils/diffCoverage.ts";
|
||||
import type { OctokitWithPlugins } from "../utils/github.ts";
|
||||
import type { ResolvedPayload } from "../utils/payload.ts";
|
||||
import {
|
||||
type ProgressComment,
|
||||
type ProgressCommentType,
|
||||
parseProgressComment,
|
||||
} from "../utils/progressComment.ts";
|
||||
import type { AccountPlan } from "../utils/runContext.ts";
|
||||
import type { RunContextData } from "../utils/runContextData.ts";
|
||||
import type { TodoTracker } from "../utils/todoTracking.ts";
|
||||
import { CheckoutPrTool } from "./checkout.ts";
|
||||
import { GetCheckSuiteLogsTool } from "./checkSuite.ts";
|
||||
import {
|
||||
@@ -39,11 +30,9 @@ import { GetIssueCommentsTool } from "./issueComments.ts";
|
||||
import { GetIssueEventsTool } from "./issueEvents.ts";
|
||||
import { IssueInfoTool } from "./issueInfo.ts";
|
||||
import { AddLabelsTool } from "./labels.ts";
|
||||
import { UpdateLearningsTool } from "./learnings.ts";
|
||||
import { SetOutputTool } from "./output.ts";
|
||||
import { CreatePullRequestTool, UpdatePullRequestBodyTool } from "./pr.ts";
|
||||
import { PullRequestInfoTool } from "./prInfo.ts";
|
||||
import type { CommentableLines } from "./review.ts";
|
||||
import { CreatePullRequestReviewTool } from "./review.ts";
|
||||
import {
|
||||
GetReviewCommentsTool,
|
||||
@@ -55,120 +44,6 @@ import { addTools } from "./shared.ts";
|
||||
import { KillBackgroundTool, ShellTool } from "./shell.ts";
|
||||
import { UploadFileTool } from "./upload.ts";
|
||||
|
||||
export type BackgroundProcess = {
|
||||
pid: number;
|
||||
outputPath: string;
|
||||
pidPath: string;
|
||||
};
|
||||
|
||||
export type BrowserDaemon = { binDir: string; error?: never } | { binDir?: never; error: string };
|
||||
|
||||
export type StoredPushDest = {
|
||||
remoteName: string;
|
||||
remoteBranch: string;
|
||||
localBranch: string;
|
||||
};
|
||||
|
||||
export interface ToolState {
|
||||
// where we're allowed to push - base repo initially, fork URL for fork PRs
|
||||
// set by setupGit, updated by checkout_pr. always set before push validation.
|
||||
pushUrl?: string;
|
||||
// push destination set by checkout_pr - used as primary source in push_branch
|
||||
// because git config reads can fail in certain environments
|
||||
pushDest?: StoredPushDest;
|
||||
// issue or PR number (same number space in GitHub)
|
||||
issueNumber?: number;
|
||||
// PR HEAD sha at checkout time — used to detect new commits pushed during a review
|
||||
checkoutSha?: string;
|
||||
// commentable lines per file at checkoutSha — captured during checkout_pr so
|
||||
// review-time inline-comment validation matches the diff GitHub will anchor
|
||||
// to (commit_id=checkoutSha). without this, a PR update between checkout and
|
||||
// review would make listFiles (latest HEAD) disagree with the anchor,
|
||||
// silently dropping valid comments or letting invalid ones through.
|
||||
//
|
||||
// commentableLinesPullNumber records WHICH PR this snapshot belongs to. if
|
||||
// the agent checks out PR B and then reviews PR A in the same session, the
|
||||
// cached snapshot for B would silently mis-validate A's comments — keying
|
||||
// by PR number forces a re-fetch when the target changes.
|
||||
//
|
||||
// commentableLinesCheckoutSha pins the snapshot to the SHA it was built
|
||||
// against. if a second checkout_pr for the SAME PR bumps checkoutSha but
|
||||
// fails before repopulating the cache (e.g., listFiles rate-limits), the
|
||||
// stale snapshot would silently mis-validate comments against the new SHA.
|
||||
// comparing both fields forces a re-fetch when either moves.
|
||||
commentableLinesByFile?: Map<string, CommentableLines>;
|
||||
commentableLinesPullNumber?: number;
|
||||
commentableLinesCheckoutSha?: string | undefined;
|
||||
// SHA to diff incrementally against — set from event payload on first checkout,
|
||||
// then from checkoutSha when review.ts detects new commits mid-review
|
||||
beforeSha?: string;
|
||||
selectedMode?: string;
|
||||
backgroundProcesses: Map<string, BackgroundProcess>;
|
||||
browserDaemon?: BrowserDaemon | undefined;
|
||||
review?: {
|
||||
id: number;
|
||||
nodeId: string;
|
||||
reviewedSha: string | undefined;
|
||||
};
|
||||
dependencyInstallation?: {
|
||||
status: "not_started" | "in_progress" | "completed" | "failed";
|
||||
promise: Promise<PrepResult[]> | undefined;
|
||||
results: PrepResult[] | undefined;
|
||||
};
|
||||
// undefined = no comment yet, object = active comment, null = deliberately deleted
|
||||
progressComment: ProgressComment | null | undefined;
|
||||
// immutable snapshot: true if a progress comment was pre-created at init time.
|
||||
// survives deleteProgressComment so handleAgentResult can still detect "expected but never reported".
|
||||
hadProgressComment: boolean;
|
||||
lastProgressBody?: string;
|
||||
wasUpdated?: boolean;
|
||||
// set after a non-plan report_progress successfully writes the final summary.
|
||||
// decoupled from todoTracker.enabled so cleanup detection survives API failures.
|
||||
finalSummaryWritten?: boolean;
|
||||
// set by select_mode when Plan + issue_number and plan-comment API returns existing plan (for report_progress target_plan_comment)
|
||||
existingPlanCommentId?: number;
|
||||
previousPlanBody?: string;
|
||||
// absolute path to the PR summary markdown file the agent edits in place.
|
||||
// seeded by main.ts before the agent starts when payload.generateSummary is set;
|
||||
// read back at end-of-run to persist to DB.
|
||||
summaryFilePath?: string;
|
||||
// exact bytes of the seeded snapshot file at run start. compared against
|
||||
// the file content at end-of-run to detect "agent never touched it" — in
|
||||
// that case persistSummary skips the DB write (saving the seed verbatim
|
||||
// would either re-write what the DB already has, on incremental runs, or
|
||||
// serialize the placeholder scaffold, on first runs).
|
||||
summarySeed?: string;
|
||||
// set to true after persistSummary completes once. prevents the error-path
|
||||
// call (which exists so a successful agent edit before a crash still gets
|
||||
// persisted) from redundantly re-running the DB PATCH on the
|
||||
// success-then-late-throw path.
|
||||
summaryPersistAttempted?: boolean;
|
||||
output?: string;
|
||||
usageEntries: AgentUsage[];
|
||||
model?: string | undefined;
|
||||
todoTracker?: TodoTracker | undefined;
|
||||
diffCoverage?: DiffCoverageState | undefined;
|
||||
}
|
||||
|
||||
interface InitToolStateParams {
|
||||
progressComment: { id: string; type: ProgressCommentType } | undefined;
|
||||
}
|
||||
|
||||
export function initToolState(params: InitToolStateParams): ToolState {
|
||||
const resolved = parseProgressComment(params.progressComment);
|
||||
|
||||
if (resolved) {
|
||||
log.info(`» using pre-created progress comment: ${resolved.id} (${resolved.type})`);
|
||||
}
|
||||
|
||||
return {
|
||||
progressComment: resolved,
|
||||
hadProgressComment: !!resolved,
|
||||
backgroundProcesses: new Map(),
|
||||
usageEntries: [],
|
||||
};
|
||||
}
|
||||
|
||||
export interface ToolContext {
|
||||
agentId: AgentId;
|
||||
repo: RunContextData["repo"];
|
||||
@@ -188,10 +63,10 @@ export interface ToolContext {
|
||||
mcpServerUrl: string;
|
||||
tmpdir: string;
|
||||
// repo-level OSS flag + account-level billing plan. together they decide
|
||||
// whether pullfrog is paying for marginal infra — see isInfraCovered in
|
||||
// utils/runContext.ts. plan gating for things like update_learnings is
|
||||
// enforced server-side via 402, so we pass plan along mostly for future
|
||||
// use / observability. see wiki/pricing.md.
|
||||
// whether pullfrog is paying for marginal infra — see `isInfraCovered` in
|
||||
// the server's `utils/billing.ts`. plan gating for endpoints like the
|
||||
// learnings PATCH is enforced server-side via 402, so we pass plan along
|
||||
// mostly for future use / observability. see wiki/pricing.md.
|
||||
oss: boolean;
|
||||
plan: AccountPlan;
|
||||
// resolved upstream model specifier (e.g. "google/gemini-3.1-pro-preview").
|
||||
@@ -288,7 +163,6 @@ function buildOrchestratorTools(ctx: ToolContext, outputSchema?: JsonSchema): To
|
||||
DeleteBranchTool(ctx),
|
||||
CreatePullRequestTool(ctx),
|
||||
UpdatePullRequestBodyTool(ctx),
|
||||
UpdateLearningsTool(ctx),
|
||||
];
|
||||
}
|
||||
|
||||
|
||||
+57
-5
@@ -15,7 +15,9 @@ import { execute, tool } from "./shared.ts";
|
||||
export const ShellParams = type({
|
||||
command: "string",
|
||||
description: "string",
|
||||
"timeout?": "number",
|
||||
"timeout?": type.number.describe(
|
||||
"Timeout in MILLISECONDS (not seconds). Default 30000 (30s), max 120000 (2m). e.g. timeout: 180000 for 3 minutes; timeout: 180 means 180ms and will kill the process almost immediately."
|
||||
),
|
||||
"working_directory?": "string",
|
||||
"background?": "boolean",
|
||||
});
|
||||
@@ -94,6 +96,27 @@ function detectSandboxMethod(): SandboxMethod {
|
||||
const PROC_CLEANUP =
|
||||
"umount /proc 2>/dev/null; umount /proc 2>/dev/null; mount -t proc proc /proc 2>/dev/null;";
|
||||
|
||||
// block container-runtime sockets that would otherwise grant a PID-namespace
|
||||
// escape: `docker run --pid=host --privileged busybox cat /proc/<pid>/environ`
|
||||
// reads the parent action process's env (which contains user secrets) even
|
||||
// though the sandbox itself is unsharing PIDs. GHA `ubuntu-latest` puts the
|
||||
// `runner` user in the `docker` group by default, so the socket is reachable
|
||||
// without sudo. bind-mounting /dev/null on top inside the sandbox's mount
|
||||
// namespace makes the socket unreachable from sandboxed shells without
|
||||
// touching the host runner (so it doesn't break user workflow steps that
|
||||
// run before/after pullfrog and legitimately need docker). same trick for
|
||||
// podman/containerd/cri-o sockets — all silent-fail if the path is missing.
|
||||
const SOCKET_CLEANUP = [
|
||||
"/var/run/docker.sock",
|
||||
"/run/docker.sock",
|
||||
"/var/run/podman/podman.sock",
|
||||
"/run/podman/podman.sock",
|
||||
"/run/containerd/containerd.sock",
|
||||
"/var/run/crio/crio.sock",
|
||||
]
|
||||
.map((path) => `mount --bind /dev/null ${path} 2>/dev/null;`)
|
||||
.join(" ");
|
||||
|
||||
function spawnShell(params: SpawnParams): ChildProcess {
|
||||
const spawnOpts = { env: params.env, cwd: params.cwd, stdio: params.stdio, detached: true };
|
||||
const sandboxMethod = detectSandboxMethod();
|
||||
@@ -108,7 +131,14 @@ function spawnShell(params: SpawnParams): ChildProcess {
|
||||
if (sandboxMethod === "unshare") {
|
||||
return spawn(
|
||||
"unshare",
|
||||
["--pid", "--fork", "--mount-proc", "bash", "-c", `${PROC_CLEANUP} ${params.command}`],
|
||||
[
|
||||
"--pid",
|
||||
"--fork",
|
||||
"--mount-proc",
|
||||
"bash",
|
||||
"-c",
|
||||
`${PROC_CLEANUP} ${SOCKET_CLEANUP} ${params.command}`,
|
||||
],
|
||||
spawnOpts
|
||||
);
|
||||
}
|
||||
@@ -141,7 +171,7 @@ function spawnShell(params: SpawnParams): ChildProcess {
|
||||
"--mount-proc",
|
||||
"bash",
|
||||
"-c",
|
||||
`${PROC_CLEANUP} exec su -p -s /bin/bash ${username} -c '${escaped}'`,
|
||||
`${PROC_CLEANUP} ${SOCKET_CLEANUP} exec su -p -s /bin/bash ${username} -c '${escaped}'`,
|
||||
],
|
||||
{ ...spawnOpts, env: {} }
|
||||
);
|
||||
@@ -174,6 +204,23 @@ function getTempDir(): string {
|
||||
return tempDir;
|
||||
}
|
||||
|
||||
/** chars of shell output kept inline in the agent reply. anything past this
|
||||
* blows the agent's context budget on commands that dump big logs (test
|
||||
* runners, build tools, grep on large trees), so the overflow is spilled
|
||||
* to a tempfile the agent can re-read selectively (cat/tail/grep). */
|
||||
export const MAX_OUTPUT_CHARS = 5000;
|
||||
|
||||
/** if `output` exceeds `MAX_OUTPUT_CHARS`, persist the full body to a
|
||||
* tempfile and return the last `MAX_OUTPUT_CHARS` prefixed with a sentinel
|
||||
* pointing at the saved path. otherwise return as-is. */
|
||||
function capOutput(output: string): string {
|
||||
if (output.length <= MAX_OUTPUT_CHARS) return output;
|
||||
const fullPath = join(getTempDir(), `shell-${randomUUID().slice(0, 8)}.log`);
|
||||
writeFileSync(fullPath, output);
|
||||
const elided = output.length - MAX_OUTPUT_CHARS;
|
||||
return `... [${elided} chars truncated; full output saved to ${fullPath}] ...\n${output.slice(-MAX_OUTPUT_CHARS)}`;
|
||||
}
|
||||
|
||||
/** detect git as a command invocation (not as part of another word like .gitignore) */
|
||||
function isGitCommand(command: string): boolean {
|
||||
const trimmed = command.trim();
|
||||
@@ -187,11 +234,15 @@ export function ShellTool(ctx: ToolContext) {
|
||||
name: "shell",
|
||||
description: `Execute shell commands securely. Environment is filtered to remove API keys and secrets.
|
||||
|
||||
Example: \`shell({ command: "pnpm test", description: "run the test suite" })\`.
|
||||
|
||||
Use this tool to:
|
||||
- Run shell commands (ls, cat, grep, find, etc.)
|
||||
- Execute build tools (npm, pnpm, cargo, make, etc.)
|
||||
- Run tests and linters
|
||||
|
||||
Output is capped at ${MAX_OUTPUT_CHARS} chars: if exceeded, only the tail is returned and the full body is saved to a tempfile (path included in the response). Re-read the tempfile with cat/tail/grep when you need more.
|
||||
|
||||
Do NOT use this tool for git commands — use the dedicated git tools instead.`,
|
||||
parameters: ShellParams,
|
||||
execute: execute(async (params) => {
|
||||
@@ -297,13 +348,14 @@ Do NOT use this tool for git commands — use the dedicated git tools instead.`,
|
||||
: `[timed out after ${timeout}ms]`;
|
||||
|
||||
const finalExitCode = exitCode ?? (timedOut ? 124 : -1);
|
||||
const trimmed = output.trim();
|
||||
if (finalExitCode !== 0) {
|
||||
log.info(`shell command failed with exit code ${finalExitCode}: ${params.command}`);
|
||||
if (output) log.info(`output: ${output.trim()}`);
|
||||
if (trimmed) log.info(`output: ${trimmed}`);
|
||||
}
|
||||
|
||||
return {
|
||||
output: output.trim(),
|
||||
output: capOutput(trimmed),
|
||||
exit_code: finalExitCode,
|
||||
timed_out: timedOut,
|
||||
};
|
||||
|
||||
@@ -3,6 +3,7 @@ import * as path from "node:path";
|
||||
import { type } from "arktype";
|
||||
import { fileTypeFromBuffer } from "file-type";
|
||||
import { apiFetch } from "../utils/apiFetch.ts";
|
||||
import { log } from "../utils/cli.ts";
|
||||
import type { ToolContext } from "./server.ts";
|
||||
import { execute, tool } from "./shared.ts";
|
||||
|
||||
@@ -65,6 +66,8 @@ export function UploadFileTool(ctx: ToolContext) {
|
||||
throw new Error(`failed to upload file: ${uploadResponse.statusText}`);
|
||||
}
|
||||
|
||||
log.info(`» uploaded file ${publicUrl}`);
|
||||
|
||||
return { success: true, publicUrl, filename, contentLength, contentType };
|
||||
}),
|
||||
});
|
||||
|
||||
+75
-2
@@ -2,6 +2,8 @@ import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
getModelEnvVars,
|
||||
getModelProvider,
|
||||
isBedrockAnthropicId,
|
||||
isVertexAnthropicId,
|
||||
modelAliases,
|
||||
parseModel,
|
||||
providers,
|
||||
@@ -55,13 +57,13 @@ describe("getModelEnvVars", () => {
|
||||
|
||||
it("returns empty env vars for free opencode models", () => {
|
||||
expect(getModelEnvVars("opencode/big-pickle")).toEqual([]);
|
||||
expect(getModelEnvVars("opencode/gpt-5-nano")).toEqual([]);
|
||||
expect(getModelEnvVars("opencode/mimo-v2-pro-free")).toEqual([]);
|
||||
expect(getModelEnvVars("opencode/minimax-m2.5-free")).toEqual([]);
|
||||
});
|
||||
|
||||
it("still requires OPENCODE_API_KEY for non-free opencode models", () => {
|
||||
expect(getModelEnvVars("opencode/claude-opus")).toEqual(["OPENCODE_API_KEY"]);
|
||||
expect(getModelEnvVars("opencode/gpt-5-nano")).toEqual(["OPENCODE_API_KEY"]);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -175,7 +177,12 @@ describe("modelAliases registry", () => {
|
||||
|
||||
it("has exactly one preferred model per provider", () => {
|
||||
for (const providerKey of Object.keys(providers)) {
|
||||
const preferred = modelAliases.filter((a) => a.provider === providerKey && a.preferred);
|
||||
// routing-only providers (bedrock) deliberately have no preferred
|
||||
// model — the user picks the actual model via a per-run env var, so
|
||||
// there's no "preferred default" to surface to auto-select.
|
||||
const aliases = modelAliases.filter((a) => a.provider === providerKey);
|
||||
if (aliases.every((a) => a.routing)) continue;
|
||||
const preferred = aliases.filter((a) => a.preferred);
|
||||
expect(preferred.length, `${providerKey} should have exactly 1 preferred model`).toBe(1);
|
||||
}
|
||||
});
|
||||
@@ -190,6 +197,10 @@ describe("modelAliases registry", () => {
|
||||
|
||||
it("all resolve values follow provider/model format", () => {
|
||||
for (const alias of modelAliases) {
|
||||
// routing slugs use a sentinel `resolve` (e.g. "bedrock") that's never
|
||||
// passed to a CLI directly — the harness reads a separate env var to
|
||||
// get the real model ID. format check doesn't apply.
|
||||
if (alias.routing) continue;
|
||||
expect(alias.resolve).toContain("/");
|
||||
}
|
||||
});
|
||||
@@ -200,6 +211,68 @@ describe("modelAliases registry", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("isBedrockAnthropicId", () => {
|
||||
it("matches geo-prefixed Anthropic foundation IDs", () => {
|
||||
expect(isBedrockAnthropicId("us.anthropic.claude-opus-4-7")).toBe(true);
|
||||
expect(isBedrockAnthropicId("eu.anthropic.claude-sonnet-4-6")).toBe(true);
|
||||
expect(isBedrockAnthropicId("global.anthropic.claude-haiku-4-5-20251001-v1:0")).toBe(true);
|
||||
});
|
||||
|
||||
it("matches in-region Anthropic foundation IDs", () => {
|
||||
expect(isBedrockAnthropicId("anthropic.claude-opus-4-7")).toBe(true);
|
||||
});
|
||||
|
||||
it("rejects non-Anthropic foundation IDs", () => {
|
||||
expect(isBedrockAnthropicId("amazon.nova-pro-v1:0")).toBe(false);
|
||||
expect(isBedrockAnthropicId("us.meta.llama4-scout-17b-instruct-v1:0")).toBe(false);
|
||||
expect(isBedrockAnthropicId("deepseek.v3.2")).toBe(false);
|
||||
});
|
||||
|
||||
// regression: PR #720 review caught that a substring-only match was
|
||||
// fragile for inference-profile ARNs (which BEDROCK_MODEL_ID accepts per
|
||||
// the AWS docs). ARN names are user-chosen — both directions of the
|
||||
// heuristic could break depending on what name the operator picked.
|
||||
// We anchor on a discrete dot-segment match (case-insensitive) instead.
|
||||
it("ignores 'anthropic' substrings inside non-segment text", () => {
|
||||
// ARN whose user-chosen profile name happens to contain "anthropic" as
|
||||
// part of a longer word — would route to claude-code under naive
|
||||
// includes("anthropic") even though the backing model is unknown.
|
||||
expect(
|
||||
isBedrockAnthropicId(
|
||||
"arn:aws:bedrock:us-east-2:123456789012:application-inference-profile/my-anthropicish-profile"
|
||||
)
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("matches when 'anthropic' appears as its own dot-segment in ARN", () => {
|
||||
// ARN whose profile name embeds the foundation segment correctly —
|
||||
// operator chose to surface the backing model in the name.
|
||||
expect(
|
||||
isBedrockAnthropicId(
|
||||
"arn:aws:bedrock:us-east-2:123456789012:application-inference-profile/anthropic.claude-opus-4-7"
|
||||
)
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("is case-insensitive", () => {
|
||||
expect(isBedrockAnthropicId("US.ANTHROPIC.CLAUDE-OPUS-4-7")).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("isVertexAnthropicId", () => {
|
||||
it("matches Claude Vertex IDs by anchored prefix", () => {
|
||||
expect(isVertexAnthropicId("claude-opus-4-1@20250805")).toBe(true);
|
||||
});
|
||||
|
||||
it("rejects Gemini IDs", () => {
|
||||
expect(isVertexAnthropicId("gemini-2.5-pro")).toBe(false);
|
||||
});
|
||||
|
||||
it("ignores Anthropic substrings outside the prefix", () => {
|
||||
expect(isVertexAnthropicId("publishers/anthropic/models/claude-opus-4-1")).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("providers registry", () => {
|
||||
it("every provider has envVars", () => {
|
||||
for (const [key, config] of Object.entries(providers)) {
|
||||
|
||||
@@ -7,6 +7,24 @@
|
||||
|
||||
// ── types ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* routing discriminant for entries whose `resolve` is dynamic — looked up
|
||||
* from a separate env var at run time rather than fixed in the catalog.
|
||||
*
|
||||
* `"bedrock"` means the actual model ID comes from `BEDROCK_MODEL_ID`
|
||||
* (an AWS-canonical Bedrock model ID like `us.anthropic.claude-opus-4-7`
|
||||
* or `amazon.nova-pro-v1:0`). `"vertex"` means the actual model ID comes
|
||||
* from `VERTEX_MODEL_ID` (a Vertex AI model ID like
|
||||
* `claude-opus-4-1@20250805` or `gemini-2.5-pro`). enterprise hosted-model
|
||||
* customers self-select for version control — silent alias bumps would break
|
||||
* compliance review, model-access enrollment, and provisioned-throughput
|
||||
* contracts. so the single `bedrock/byok` and `vertex/byok` entries are
|
||||
* routing slugs, not model aliases: the harness reads the backend-specific
|
||||
* env var and routes to claude-code for Anthropic IDs or opencode for
|
||||
* everything else.
|
||||
*/
|
||||
export type ModelRouting = "bedrock" | "vertex";
|
||||
|
||||
export interface ModelAlias {
|
||||
/** stable alias stored in DB, e.g. "anthropic/claude-opus" */
|
||||
slug: string;
|
||||
@@ -14,9 +32,9 @@ export interface ModelAlias {
|
||||
provider: string;
|
||||
/** human-readable name shown in dropdowns */
|
||||
displayName: string;
|
||||
/** concrete models.dev specifier, e.g. "anthropic/claude-opus-4-6" */
|
||||
/** concrete models.dev specifier, e.g. "anthropic/claude-opus-4-6". sentinel for routing entries — never passed to a CLI directly. */
|
||||
resolve: string;
|
||||
/** full models.dev specifier for the OpenRouter equivalent (undefined for free models) */
|
||||
/** full models.dev specifier for the OpenRouter equivalent (undefined for free models and routing entries) */
|
||||
openRouterResolve: string | undefined;
|
||||
/** top-tier pick for this provider — preferred during auto-select */
|
||||
preferred: boolean;
|
||||
@@ -24,6 +42,15 @@ export interface ModelAlias {
|
||||
isFree: boolean;
|
||||
/** slug of a replacement model — presence implies this model is deprecated */
|
||||
fallback: string | undefined;
|
||||
/** dynamic-resolution discriminant — see ModelRouting docs */
|
||||
routing: ModelRouting | undefined;
|
||||
/** alias key (within same provider) of the cheaper sibling reviewfrog should
|
||||
* use as its lens-fanout subagent. e.g. claude-opus → "claude-sonnet". */
|
||||
subagentModel: string | undefined;
|
||||
/** hide from selectable lists (UI dropdowns, CLI pickers). does NOT affect
|
||||
* resolution — for that use `fallback`. used for internal-only tier targets
|
||||
* (e.g. gpt-5.4 as a subagent target without exposing it to users). */
|
||||
hidden: boolean;
|
||||
}
|
||||
|
||||
interface ModelDef {
|
||||
@@ -37,11 +64,23 @@ interface ModelDef {
|
||||
isFree?: boolean;
|
||||
/** slug of a replacement model — presence implies this model is deprecated */
|
||||
fallback?: string;
|
||||
/** dynamic-resolution discriminant — see ModelRouting docs */
|
||||
routing?: ModelRouting;
|
||||
/** alias key (within same provider) of the cheaper sibling reviewfrog should
|
||||
* use as its lens-fanout subagent (e.g. claude-opus → "claude-sonnet"). */
|
||||
subagentModel?: string;
|
||||
/** hide from selectable lists. does NOT affect resolution; for that use `fallback`. */
|
||||
hidden?: boolean;
|
||||
}
|
||||
|
||||
export interface ProviderConfig {
|
||||
displayName: string;
|
||||
envVars: readonly string[];
|
||||
/** credentials authored only via `pullfrog auth <provider>` — never
|
||||
* user-facing in `init`, never documented as a manual GHA secret. counted
|
||||
* for hasAnyKey / log-redaction purposes but excluded from any prompt /
|
||||
* paste flow. CLI-managed magic. see wiki/codex-auth.md. */
|
||||
managedCredentials?: readonly string[];
|
||||
models: Record<string, ModelDef>;
|
||||
}
|
||||
|
||||
@@ -61,6 +100,7 @@ export const providers = {
|
||||
resolve: "anthropic/claude-opus-4-7",
|
||||
openRouterResolve: "openrouter/anthropic/claude-opus-4.7",
|
||||
preferred: true,
|
||||
subagentModel: "claude-sonnet",
|
||||
},
|
||||
"claude-sonnet": {
|
||||
displayName: "Claude Sonnet",
|
||||
@@ -77,17 +117,29 @@ export const providers = {
|
||||
openai: provider({
|
||||
displayName: "OpenAI",
|
||||
envVars: ["OPENAI_API_KEY"],
|
||||
managedCredentials: ["CODEX_AUTH_JSON"],
|
||||
models: {
|
||||
gpt: {
|
||||
displayName: "GPT",
|
||||
resolve: "openai/gpt-5.5",
|
||||
openRouterResolve: "openrouter/openai/gpt-5.5",
|
||||
preferred: true,
|
||||
subagentModel: "gpt-5.4",
|
||||
},
|
||||
"gpt-pro": {
|
||||
displayName: "GPT Pro",
|
||||
resolve: "openai/gpt-5.5-pro",
|
||||
openRouterResolve: "openrouter/openai/gpt-5.5-pro",
|
||||
subagentModel: "gpt",
|
||||
},
|
||||
// hidden subagent target — `gpt` lenses run against this. surfacing
|
||||
// it in the picker would just confuse users (it's the prior-flagship,
|
||||
// and they already have `gpt` and `gpt-mini` to choose from).
|
||||
"gpt-5.4": {
|
||||
displayName: "GPT 5.4",
|
||||
resolve: "openai/gpt-5.4",
|
||||
openRouterResolve: "openrouter/openai/gpt-5.4",
|
||||
hidden: true,
|
||||
},
|
||||
"gpt-mini": {
|
||||
displayName: "GPT Mini",
|
||||
@@ -126,10 +178,15 @@ export const providers = {
|
||||
resolve: "google/gemini-3.1-pro-preview",
|
||||
openRouterResolve: "openrouter/google/gemini-3.1-pro-preview",
|
||||
preferred: true,
|
||||
// Inherit (subagents stay on Pro). Google has no in-between tier;
|
||||
// dropping to Flash for review work was a meaningful capability cliff
|
||||
// (Flash missed the catastrophic camelCase/snake_case mismatch in
|
||||
// the v4 e2e test). Pro is cost-effective enough to use for both
|
||||
// orchestrator and lenses.
|
||||
},
|
||||
"gemini-flash": {
|
||||
displayName: "Gemini Flash",
|
||||
resolve: "google/gemini-3-flash-preview",
|
||||
resolve: "google/gemini-3.5-flash",
|
||||
openRouterResolve: "openrouter/google/gemini-3-flash-preview",
|
||||
},
|
||||
},
|
||||
@@ -144,15 +201,22 @@ export const providers = {
|
||||
openRouterResolve: "openrouter/x-ai/grok-4.3",
|
||||
preferred: true,
|
||||
},
|
||||
// legacy aliases — xAI retired the entire fast/code-fast line on
|
||||
// 2026-05-15 (https://docs.x.ai/developers/migration/may-15-deprecation)
|
||||
// and now redirects every deprecated text-model slug to grok-4.3 at
|
||||
// standard pricing. fall back to the live `xai/grok` so the alias
|
||||
// chain resolves to grok-4.3 for both direct-key and OpenRouter users.
|
||||
"grok-fast": {
|
||||
displayName: "Grok Fast",
|
||||
resolve: "xai/grok-4-1-fast",
|
||||
openRouterResolve: "openrouter/x-ai/grok-4.1-fast",
|
||||
openRouterResolve: "openrouter/x-ai/grok-4.3",
|
||||
fallback: "xai/grok",
|
||||
},
|
||||
"grok-code-fast": {
|
||||
displayName: "Grok Code Fast",
|
||||
resolve: "xai/grok-code-fast-1",
|
||||
openRouterResolve: "openrouter/x-ai/grok-code-fast-1",
|
||||
openRouterResolve: "openrouter/x-ai/grok-4.3",
|
||||
fallback: "xai/grok",
|
||||
},
|
||||
},
|
||||
}),
|
||||
@@ -214,6 +278,7 @@ export const providers = {
|
||||
displayName: "Claude Opus",
|
||||
resolve: "opencode/claude-opus-4-7",
|
||||
openRouterResolve: "openrouter/anthropic/claude-opus-4.7",
|
||||
subagentModel: "claude-sonnet",
|
||||
},
|
||||
"claude-sonnet": {
|
||||
displayName: "Claude Sonnet",
|
||||
@@ -229,11 +294,20 @@ export const providers = {
|
||||
displayName: "GPT",
|
||||
resolve: "opencode/gpt-5.5",
|
||||
openRouterResolve: "openrouter/openai/gpt-5.5",
|
||||
subagentModel: "gpt-5.4",
|
||||
},
|
||||
"gpt-pro": {
|
||||
displayName: "GPT Pro",
|
||||
resolve: "opencode/gpt-5.5-pro",
|
||||
openRouterResolve: "openrouter/openai/gpt-5.5-pro",
|
||||
subagentModel: "gpt",
|
||||
},
|
||||
// hidden subagent target — see openai provider above for context.
|
||||
"gpt-5.4": {
|
||||
displayName: "GPT 5.4",
|
||||
resolve: "opencode/gpt-5.4",
|
||||
openRouterResolve: "openrouter/openai/gpt-5.4",
|
||||
hidden: true,
|
||||
},
|
||||
"gpt-mini": {
|
||||
displayName: "GPT Mini",
|
||||
@@ -257,6 +331,7 @@ export const providers = {
|
||||
displayName: "Gemini Pro",
|
||||
resolve: "opencode/gemini-3.1-pro",
|
||||
openRouterResolve: "openrouter/google/gemini-3.1-pro-preview",
|
||||
// Inherit — see google/gemini-pro for rationale.
|
||||
},
|
||||
"gemini-flash": {
|
||||
displayName: "Gemini Flash",
|
||||
@@ -271,8 +346,7 @@ export const providers = {
|
||||
"gpt-5-nano": {
|
||||
displayName: "GPT Nano",
|
||||
resolve: "opencode/gpt-5-nano",
|
||||
envVars: [],
|
||||
isFree: true,
|
||||
openRouterResolve: "openrouter/openai/gpt-5-nano",
|
||||
},
|
||||
"mimo-v2-pro-free": {
|
||||
displayName: "MiMo V2 Pro",
|
||||
@@ -289,6 +363,39 @@ export const providers = {
|
||||
},
|
||||
},
|
||||
}),
|
||||
bedrock: provider({
|
||||
displayName: "Amazon Bedrock",
|
||||
envVars: ["AWS_BEARER_TOKEN_BEDROCK", "AWS_REGION", "BEDROCK_MODEL_ID"],
|
||||
models: {
|
||||
// single routing entry — the actual Bedrock model ID is read from
|
||||
// BEDROCK_MODEL_ID at run time. see ModelRouting docs for why we
|
||||
// don't catalog individual Bedrock models.
|
||||
byok: {
|
||||
displayName: "Amazon Bedrock",
|
||||
resolve: "bedrock",
|
||||
routing: "bedrock",
|
||||
},
|
||||
},
|
||||
}),
|
||||
vertex: provider({
|
||||
displayName: "Google Vertex AI",
|
||||
envVars: [
|
||||
"VERTEX_SERVICE_ACCOUNT_JSON",
|
||||
"GOOGLE_CLOUD_PROJECT",
|
||||
"VERTEX_LOCATION",
|
||||
"VERTEX_MODEL_ID",
|
||||
],
|
||||
models: {
|
||||
// single routing entry — the actual Vertex AI model ID is read from
|
||||
// VERTEX_MODEL_ID at run time. see ModelRouting docs for why we don't
|
||||
// catalog individual Vertex models.
|
||||
byok: {
|
||||
displayName: "Google Vertex AI",
|
||||
resolve: "vertex",
|
||||
routing: "vertex",
|
||||
},
|
||||
},
|
||||
}),
|
||||
openrouter: provider({
|
||||
displayName: "OpenRouter",
|
||||
envVars: ["OPENROUTER_API_KEY"],
|
||||
@@ -298,6 +405,7 @@ export const providers = {
|
||||
resolve: "openrouter/anthropic/claude-opus-4.7",
|
||||
openRouterResolve: "openrouter/anthropic/claude-opus-4.7",
|
||||
preferred: true,
|
||||
subagentModel: "claude-sonnet",
|
||||
},
|
||||
"claude-sonnet": {
|
||||
displayName: "Claude Sonnet",
|
||||
@@ -313,11 +421,20 @@ export const providers = {
|
||||
displayName: "GPT",
|
||||
resolve: "openrouter/openai/gpt-5.5",
|
||||
openRouterResolve: "openrouter/openai/gpt-5.5",
|
||||
subagentModel: "gpt-5.4",
|
||||
},
|
||||
"gpt-pro": {
|
||||
displayName: "GPT Pro",
|
||||
resolve: "openrouter/openai/gpt-5.5-pro",
|
||||
openRouterResolve: "openrouter/openai/gpt-5.5-pro",
|
||||
subagentModel: "gpt",
|
||||
},
|
||||
// hidden subagent target — see openai provider above for context.
|
||||
"gpt-5.4": {
|
||||
displayName: "GPT 5.4",
|
||||
resolve: "openrouter/openai/gpt-5.4",
|
||||
openRouterResolve: "openrouter/openai/gpt-5.4",
|
||||
hidden: true,
|
||||
},
|
||||
"gpt-mini": {
|
||||
displayName: "GPT Mini",
|
||||
@@ -346,6 +463,7 @@ export const providers = {
|
||||
displayName: "Gemini Pro",
|
||||
resolve: "openrouter/google/gemini-3.1-pro-preview",
|
||||
openRouterResolve: "openrouter/google/gemini-3.1-pro-preview",
|
||||
// Inherit — see google/gemini-pro for rationale.
|
||||
},
|
||||
"gemini-flash": {
|
||||
displayName: "Gemini Flash",
|
||||
@@ -420,6 +538,16 @@ export function getModelEnvVars(slug: string): string[] {
|
||||
return providerConfig.envVars.slice();
|
||||
}
|
||||
|
||||
/** managed credentials are authored only via `pullfrog auth <provider>` — they
|
||||
* count as "configured" for hasAnyKey-style UI checks but are never offered as
|
||||
* a manual-paste option in `init` or the AgentSettings env-var button row.
|
||||
* see `provider.managedCredentials` and wiki/codex-auth.md. */
|
||||
export function getModelManagedCredentials(slug: string): string[] {
|
||||
const parsed = parseModel(slug);
|
||||
const providerConfig = (providers as Record<string, ProviderConfig>)[parsed.provider];
|
||||
return providerConfig?.managedCredentials?.slice() ?? [];
|
||||
}
|
||||
|
||||
// ── derived flat list ──────────────────────────────────────────────────────────
|
||||
|
||||
export const modelAliases: ModelAlias[] = Object.entries(providers).flatMap(
|
||||
@@ -433,6 +561,12 @@ export const modelAliases: ModelAlias[] = Object.entries(providers).flatMap(
|
||||
preferred: def.preferred ?? false,
|
||||
isFree: def.isFree ?? false,
|
||||
fallback: def.fallback,
|
||||
routing: def.routing,
|
||||
// subagentModel is stored as an alias key local to the provider; expand
|
||||
// here to a fully-qualified slug so callers can look up the target alias
|
||||
// directly without re-deriving the provider.
|
||||
subagentModel: def.subagentModel ? `${providerKey}/${def.subagentModel}` : undefined,
|
||||
hidden: def.hidden ?? false,
|
||||
}))
|
||||
);
|
||||
|
||||
@@ -452,7 +586,7 @@ const MAX_FALLBACK_DEPTH = 10;
|
||||
* use this in UI display sites (dropdown trigger labels, PR-comment footers,
|
||||
* etc.) so a deprecated stored slug renders as the model the user actually
|
||||
* runs against — not the historical name. selectable lists should still hide
|
||||
* deprecated aliases by filtering on `!a.fallback`.
|
||||
* deprecated and internal-only aliases by filtering on `!a.fallback && !a.hidden`.
|
||||
*/
|
||||
export function resolveDisplayAlias(slug: string): ModelAlias | undefined {
|
||||
let current = slug;
|
||||
@@ -486,3 +620,53 @@ export function resolveCliModel(slug: string): string | undefined {
|
||||
export function resolveOpenRouterModel(slug: string): string | undefined {
|
||||
return resolveDisplayAlias(slug)?.openRouterResolve;
|
||||
}
|
||||
|
||||
// ── bedrock routing ────────────────────────────────────────────────────────────
|
||||
|
||||
/** env var that supplies the Bedrock model ID for the `bedrock/byok` slug. */
|
||||
export const BEDROCK_MODEL_ID_ENV = "BEDROCK_MODEL_ID";
|
||||
|
||||
/** env var that supplies the Vertex AI model ID for the `vertex/byok` slug. */
|
||||
export const VERTEX_MODEL_ID_ENV = "VERTEX_MODEL_ID";
|
||||
|
||||
/**
|
||||
* the Bedrock model ID passed to claude-code or opencode is whatever the
|
||||
* user set in `BEDROCK_MODEL_ID` — Pullfrog never resolves or upgrades it.
|
||||
* we route by checking whether the ID names an Anthropic model: claude-code
|
||||
* handles Anthropic-on-Bedrock natively (with `CLAUDE_CODE_USE_BEDROCK=1`),
|
||||
* everything else goes through opencode's `amazon-bedrock` provider.
|
||||
*
|
||||
* AWS Bedrock IDs come in two shapes:
|
||||
* - dotted foundation IDs: `us.anthropic.claude-opus-4-7`,
|
||||
* `anthropic.claude-haiku-4-5-20251001-v1:0`, `amazon.nova-pro-v1:0`,
|
||||
* `meta.llama4-scout-17b-instruct-v1:0`. AWS-published, lowercase, the
|
||||
* foundation provider always appears as a discrete dot-segment.
|
||||
* - inference-profile ARNs: `arn:aws:bedrock:us-east-2:<acct>:application-inference-profile/<user-name>`.
|
||||
* `<user-name>` is operator-chosen, so a naive substring check is fragile
|
||||
* in both directions (Anthropic profile named without "anthropic" → routes
|
||||
* to opencode and misses CLAUDE_CODE_USE_BEDROCK; non-Anthropic profile
|
||||
* whose name happens to contain "anthropic" → routes to claude-code).
|
||||
*
|
||||
* we anchor on a discrete dot-segment match (case-insensitive). this catches
|
||||
* every published foundation ID and is conservative for ARN-form IDs: ARN
|
||||
* names that don't include "anthropic" as their own dot-segment route to
|
||||
* opencode by default. operators using ARN-form IDs whose backing model is
|
||||
* Anthropic should set `PULLFROG_AGENT=claude` to force the right route, or
|
||||
* include the foundation segment in the profile name.
|
||||
*/
|
||||
export function isBedrockAnthropicId(bedrockModelId: string): boolean {
|
||||
// split on `.`, `/`, and `:` so the check works for both dotted foundation
|
||||
// IDs (anthropic.* / us.anthropic.*) and ARN-form IDs (where the relevant
|
||||
// foundation segment sits between `/` and `.` inside the resource name).
|
||||
return bedrockModelId.toLowerCase().split(/[./:]/).includes("anthropic");
|
||||
}
|
||||
|
||||
/**
|
||||
* Vertex Anthropic model IDs start with the Claude family name, e.g.
|
||||
* `claude-opus-4-1@20250805`. partner-model resource paths can contain the
|
||||
* substring "anthropic" elsewhere, so the Bedrock segment check does not
|
||||
* transfer — anchor on the model ID prefix instead.
|
||||
*/
|
||||
export function isVertexAnthropicId(vertexModelId: string): boolean {
|
||||
return /^claude-/i.test(vertexModelId.trim());
|
||||
}
|
||||
|
||||
@@ -10,64 +10,154 @@ export interface Mode {
|
||||
prompt?: string | undefined;
|
||||
}
|
||||
|
||||
// Default user-facing summary format embedded in Review mode review bodies.
|
||||
// Deliberately scoped to Review (initial PR review). IncrementalReview keeps
|
||||
// its own terser bullet-list "Reviewed changes" shape since re-review bodies
|
||||
// are deltas, not introductions. Distinct from the agent-internal snapshot
|
||||
// (action/utils/prSummary.ts) which has its own stable scaffold and is never
|
||||
// shaped by user instructions — see selectMode.ts for the firewall.
|
||||
// Default user-facing summary format embedded in BOTH Review and
|
||||
// IncrementalReview review bodies. The two modes share the preamble +
|
||||
// cross-cutting + nitpicks shape; the only difference is scope (full PR for
|
||||
// Review vs delta against the prior pullfrog review for IncrementalReview).
|
||||
// Distinct from the agent-internal snapshot (action/utils/prSummary.ts) which
|
||||
// has its own stable scaffold and is never shaped by user instructions — see
|
||||
// selectMode.ts for the firewall.
|
||||
export const PR_SUMMARY_FORMAT = `### Default format
|
||||
|
||||
Follow this structure exactly:
|
||||
The body has at most three parts in this exact order:
|
||||
|
||||
<b>TL;DR</b> — 1-3 sentences on what the PR does and why. Focus on intent, not mechanics.
|
||||
NOTE: use HTML bold <b>TL;DR</b>, NOT markdown bold **TL;DR**.
|
||||
1. **Reviewed changes preamble** — one bolded inline lead-in describing what was reviewed in this run, a bullet list of the substantive changes, and an HTML comment carrying review metadata for downstream agents.
|
||||
2. **Cross-cutting issue sections** (zero or more) — one \`### \` heading per concern, with a human-readable problem write-up and a collapsed \`<details>Technical details</details>\` block underneath.
|
||||
3. **\`### ℹ️ Nitpicks\`** at the very bottom (only if there are nits worth surfacing in the body) — a flat bullet list, no technical-details block.
|
||||
|
||||
### Key changes
|
||||
Inline-vs-body split: concerns that anchor to a specific line go inline (use the \`comments\` parameter). Body \`### \` sections are reserved for concerns that **have no line to anchor to** — typically because the concern is about *absence* (something the diff should have done but didn't), *sequencing* (rollout / deletion / migration order), *design decisions only the human can make*, or *scope questions the diff implicitly raises but doesn't address*. A concern that anchors to a line but has broad implications still goes inline (use the technical-details block there to capture the implications — see Inline technical details below). If you found no non-anchorable concerns, the body has zero \`### \` issue sections — just the preamble + metadata.
|
||||
|
||||
- **Short human-readable title** — 1 sentence per change. Write a short prose phrase (title case or sentence case); when you name a file, type, or function, put that name in backticks (e.g. **Add \`TodoTracker\` for live checklists**). A reviewer should understand the full PR from this list alone.
|
||||
## 1. Reviewed changes preamble
|
||||
|
||||
<sub><b>Summary</b> | {file_count} files | {commit_count} commits | base: \`{base}\` ← \`{head}\`</sub>
|
||||
NOTE: the metadata line goes AFTER the bullet list, not before it.
|
||||
Open with a single bolded inline lead-in followed immediately by the bullet list (no \`### Key changes\` heading, no \`<b>TL;DR</b>\`):
|
||||
|
||||
Then for each key change, a ## section with a short descriptive title that reads like a documentation heading (e.g. ## Live todo checklist tracking).
|
||||
\`\`\`
|
||||
**Reviewed changes** — one sentence on what was reviewed in this run. For Review (initial), this is what the PR does and why. For IncrementalReview, this is what changed since the prior pullfrog review. Focus on intent, not mechanics.
|
||||
|
||||
<br/>
|
||||
- **Short human-readable title** — 1 sentence per substantive change. Write a short prose phrase; when you name a file, type, or function, put that name in backticks (e.g. **Add \\\`TodoTracker\\\` for live checklists**). A reviewer should understand the full reviewed scope from this list alone — this IS the dispassionate "what was reviewed and what changed" overview, so cover the substantive changes, not just the loudest ones.
|
||||
|
||||
## Example readable section title
|
||||
<!--
|
||||
Pullfrog review metadata — for any agent (or human-with-agent) reading this
|
||||
review. Incorporate the fields below into your understanding of the context
|
||||
this review was made in. The findings below were written against
|
||||
{head_sha_short}; if new commits have landed on {head_ref} since this review
|
||||
was submitted, treat any specific bug, file, or line callout as POTENTIALLY
|
||||
STALE — re-diff against {head_sha_short} (or trigger a fresh review) and
|
||||
factor commits past {head_sha_short} into your understanding of the current
|
||||
state before acting on findings.
|
||||
|
||||
> **Before:** [old behavior/state]<br/>**After:** [new behavior/state]
|
||||
IMPORTANT: Before and After MUST be on a SINGLE blockquote line with an inline <br/> between them. Two separate \`>\` lines creates a double line break.
|
||||
- Mode: Review (initial) or IncrementalReview (delta against prior pullfrog review)
|
||||
- Files reviewed: {file_count}
|
||||
- Commits reviewed: {commit_count}
|
||||
- Base: {base_ref} ({base_sha_short})
|
||||
- Head: {head_ref} ({head_sha_short})
|
||||
- Reviewed commits:
|
||||
- {sha_short} — {commit_subject}
|
||||
- ...
|
||||
- Prior pullfrog review: none or {prior_sha_short} ({prior_review_html_url})
|
||||
- Submitted at: {iso_timestamp}
|
||||
-->
|
||||
\`\`\`
|
||||
|
||||
1-2 sentences of explanation. Break up text with tables, blockquotes, or lists — NEVER 3+ plain paragraphs in a row.
|
||||
Pull every metadata field from the \`checkout_pr\` tool's response — file count, commit count, base/head ref + SHA, the commit list. For \`IncrementalReview\` runs, populate \`Prior pullfrog review\` with the prior review's commit_id (short SHA) and \`html_url\` from \`list_pull_request_reviews\`.
|
||||
|
||||
If a change warrants deeper explanation, use a blockquoted details/summary framed as a question:
|
||||
> <details><summary>How does X work?</summary>
|
||||
> Extended explanation here.
|
||||
> </details>
|
||||
## 2. Cross-cutting issue sections (zero or more)
|
||||
|
||||
End each section with a file links trail (3-4 key files max):
|
||||
[\`file.ts\`](https://github.com/{owner}/{repo}/pull/{number}/files#diff-{sha256hex_of_filepath}) · ...
|
||||
For each cross-cutting concern, one \`### \` section. Use this exact shape:
|
||||
|
||||
Single-feature PRs: skip the ## sections. Fold before/after and explanation into the header after key changes.
|
||||
\`\`\`
|
||||
### {emoji} {short, descriptive title — what's wrong, not what to do}
|
||||
|
||||
CRITICAL — GitHub markdown rendering rule:
|
||||
GitHub's markdown parser requires a blank line between ALL block-level elements. This includes transitions between: HTML tags (<br/>, <sub>, <details>, <b>, etc.) and markdown syntax (headings, lists, blockquotes, paragraphs). Without a blank line, GitHub treats the following content as a continuation of the HTML block and renders markdown syntax as literal text. ALWAYS separate block-level elements with a blank line.
|
||||
{Human-readable problem write-up. Describes the PROBLEM only — what's broken, what the symptom is, what the blast radius is. NO asks, NO suggested fixes, NO "the right thing to do is...". Asks and fixes live in the technical-details block below; the visible part is for the human to *understand* the problem, not to implement it.}
|
||||
|
||||
Rules:
|
||||
- \`##\` titles and key-change bullet lead-ins are plain-language summaries; backtick only actual code tokens (files, types, functions) where they appear in the title
|
||||
- ALL variable names, identifiers, and file names in body text must be in backticks
|
||||
- ALL file references MUST link to the PR Files Changed view. Use the \`diff-<hex>\` anchor precomputed next to each filename in the \`checkout_pr\` TOC — do NOT run \`sha256sum\` or any other shell command to compute anchors. NEVER fabricate hex strings. If a file is not in the TOC, omit the \`#diff-\` anchor rather than guessing.
|
||||
- Add <br/> before each ## heading for visual spacing. Do NOT use horizontal rules (---)
|
||||
- Do NOT include raw diff stats like '+123 / -45' or line counts
|
||||
- Do NOT include code blocks or repeat diff contents
|
||||
- Do NOT include a changelog section — the key changes list serves this purpose
|
||||
- Focus on *intent*, not *what* — the diff already shows what changed
|
||||
- Get the file count and commit count from the checkout_pr metadata, not by counting manually`;
|
||||
<details><summary>Technical details</summary>
|
||||
|
||||
function learningsStep(t: (toolName: string) => string, n: number): string {
|
||||
return `${n}. **learnings** (only if high confidence): if you discovered something about repo setup, test commands, conventions, or patterns that you are confident is correct and would reliably help future runs, call \`${t("update_learnings")}\` to persist it. skip this step if you are unsure or the finding is speculative/one-off. format as a flat bullet list (\`- \` per line, one fact per bullet). merge with existing learnings from the prompt — pass the FULL merged list. deduplicate, and drop bullets that are clearly wrong or no longer relevant to the current codebase.`;
|
||||
}
|
||||
\\\`\\\`\\\`\\\`markdown
|
||||
# {title repeated}
|
||||
|
||||
## Affected sites
|
||||
- {file path:line} — {what's wrong there}
|
||||
- ...
|
||||
|
||||
## Required outcome
|
||||
- {what the fix needs to achieve, not how to achieve it}
|
||||
- ...
|
||||
|
||||
## Suggested approach (optional)
|
||||
{When the fix shape is non-obvious, sketch one or more reasonable directions. Skip when the outcome alone makes the fix obvious.}
|
||||
|
||||
## Open questions for the human (optional)
|
||||
- {Any decision an implementing agent shouldn't make unilaterally — pricing thresholds, breaking-change policy, naming, scope of follow-up.}
|
||||
\\\`\\\`\\\`\\\`
|
||||
|
||||
</details>
|
||||
\`\`\`
|
||||
|
||||
Concrete example of the visible part of a non-anchored section (technical-details block unchanged from the template above):
|
||||
|
||||
\`\`\`
|
||||
### ℹ️ Legacy \`opencode.ts\` has no documented deletion plan
|
||||
|
||||
The v2 harness lands alongside the v1 file and imports one helper from it. Worth a follow-up issue or a TODO so the next maintainer doesn't have to re-derive the cleanup plan.
|
||||
\`\`\`
|
||||
|
||||
The example's value is its *shape*: a finding about absence (no deletion plan), not a line-anchored bug. Body sections live or die on whether the concern genuinely doesn't fit on a line.
|
||||
|
||||
**Heading severity emoji** — every \`### \` heading carries one:
|
||||
|
||||
- 🚨 critical — blocks merge (data loss, security, broken core flow)
|
||||
- ⚠️ important — must address before merging (regression, missing validation, incorrect behavior)
|
||||
- ℹ️ informational — surfaced for awareness; mergeable as-is
|
||||
|
||||
**Visible problem write-up rules:**
|
||||
|
||||
- **No asks, no suggested fixes** in the visible part. The visible portion describes the problem; the technical-details block describes the fix shape and any open questions. The exception: a fix so self-evident that NOT stating it would be weird (e.g. "the typo is missing an 'r'") — in that case, fold it into the problem statement and skip the suggested-approach block in technical details too.
|
||||
- **Never two successive plain paragraphs.** Every transition between block-level elements must alternate prose with structure: paragraph → bullet list → paragraph; paragraph → code fence → bullet list; paragraph → table → paragraph. Two consecutive paragraphs in a row create a wall of text that's impossible to digest. If you catch yourself writing one, find a way to split it: pull a list out of it, drop a 2-3 line code fence between them, or merge them into a single tighter paragraph.
|
||||
- **Per-paragraph budget:** ~3 sentences max. Past that, you're explaining where you should be structuring.
|
||||
- **Identifier discipline still applies** in the visible part. Lead with behavior in plain English; name an identifier only when it's the subject of the concern or a public surface a reader would recognize. The technical-details block is where dense identifier references belong.
|
||||
|
||||
**Technical-details block rules:**
|
||||
|
||||
- Wrapped in a 4-backtick markdown fence (\`\\\`\\\`\\\`\\\`markdown ... \\\`\\\`\\\`\\\`\`) so it's visually distinct, one-click copyable, and can contain its own 3-backtick code fences without escape gymnastics. The contents are agent-readable — a fix-agent will pull the body down and use this block as the brief.
|
||||
- File paths and \`file:line\` refs are encouraged (and necessary) — the next agent uses these to navigate. Identifier density is fine here.
|
||||
- Slightly more verbose than the absolute minimum is OK when it materially helps the next agent: a small code snippet showing the symptom, a short table of mismatched key/column pairs, a one-paragraph "why CI doesn't catch it" note. Skip massive regression-test scaffolding or full route rewrites — the implementing agent writes those.
|
||||
- Use the four standard sections (\`Affected sites\`, \`Required outcome\`, optional \`Suggested approach\`, optional \`Open questions for the human\`). Skip the optional sections when they wouldn't add anything.
|
||||
|
||||
## Inline technical details
|
||||
|
||||
Inline comments are short (~2-3 sentences) by default. When an inline finding has broader implications worth recording for a fix-agent — e.g. a localized bug whose proper fix requires touching several files, or where the right fix depends on a design decision the human needs to make — append a collapsed \`<details><summary>Technical details</summary>\` block to the inline comment's body. Same shape as the body-section technical-details block (4-backtick fenced markdown, \`## Affected sites\` / \`## Required outcome\` / optional \`## Suggested approach\` / optional \`## Open questions for the human\`).
|
||||
|
||||
GitHub renders the same markdown parser in inline comments as in the review body, so the collapsed-details affordance works the same way. The visible part of the inline comment stays scannable; the depth is one click away for any agent that needs it.
|
||||
|
||||
## 3. \`### ℹ️ Nitpicks\` (optional, last section)
|
||||
|
||||
Only when there are nits that for some reason can't be inlined. Filepaths in nit text are fine — these are simple enough that a human or agent reads once and acts. No technical-details block.
|
||||
|
||||
\`\`\`
|
||||
### ℹ️ Nitpicks
|
||||
|
||||
- {nit, with file path inline if useful, ≤ ~200 chars}
|
||||
- ...
|
||||
\`\`\`
|
||||
|
||||
## Inline comment shape
|
||||
|
||||
Inline comments use the same severity framing as body \`### \` sections, scaled down for line-anchored use:
|
||||
|
||||
- **Lead with a 1-2 sentence problem statement.** The reader is looking at the line in question, so don't restate what the line says — describe what's wrong with it. Optionally prefix the visible line with a severity emoji (🚨 / ⚠️ / ℹ️) when severity isn't obvious from context.
|
||||
- **Optional \`<details><summary>Technical details</summary>...</details>\` collapsible** for findings whose technical context (longer file:line references, related-code snippets, suggested approach, regression-risk notes) would overwhelm the human-readable lead-in. Same agent-readable purpose, same 4-backtick fence shape, and same 4-section structure as the body's technical-details block — see *Inline technical details* above. Encouraged whenever the depth helps a downstream fix-agent; don't force one when the inline lead-in already says everything.
|
||||
- **Visible portion ≤ 2-3 sentences.** If you find yourself writing more, that's the cue to split the depth into the \`Technical details\` collapsible.
|
||||
|
||||
## Body-wide rules
|
||||
|
||||
- **Inline-vs-body discipline (repeated for emphasis):** anything that anchors to a specific line goes inline (with a \`<details>Technical details</details>\` block when the implications are broad). The body is for non-anchorable concerns only — absence, sequencing, design decisions, scope questions, architectural risk.
|
||||
- **No \`### Issues found\` heading** above the issue sections — each \`### \` heading IS the issue.
|
||||
- **Severity emoji on every \`### \` heading** (🚨 / ⚠️ / ℹ️). No emoji on the preamble lead-in or anywhere else.
|
||||
- **GitHub block-level rendering**: GitHub's markdown parser requires a blank line between ALL block-level elements (HTML tags like \`<br/>\`, \`<sub>\`, \`<details>\`, \`<b>\` and markdown syntax like headings, lists, blockquotes, code fences, paragraphs). Without a blank line, GitHub treats following content as a continuation of the HTML block and renders markdown syntax as literal text. ALWAYS separate block-level elements with a blank line.
|
||||
- **Backtick-wrap** every variable, identifier, or file name when you mention one (in either visible or technical-details portions).
|
||||
- **Don't repeat diff content**, don't include raw \`+123 / -45\` stats, don't include a changelog section, don't use horizontal rules (\`---\`).
|
||||
- **Pull file/commit counts from \`checkout_pr\` metadata** — never count manually.
|
||||
- **Legacy headings REMOVED.** Do not use \`### Key changes\`, \`### Issues found\`, \`<b>TL;DR</b>\`, or \`<sub><b>Summary</b>\`. The new structure subsumes them.`;
|
||||
|
||||
export function computeModes(agentId: AgentId): Mode[] {
|
||||
const t = (toolName: string) => formatMcpToolRef(agentId, toolName);
|
||||
@@ -78,18 +168,20 @@ export function computeModes(agentId: AgentId): Mode[] {
|
||||
"Implement, build, create, or develop code changes; make specific changes to files or features; execute a plan; or handle tasks with specific implementation details",
|
||||
prompt: `### Checklist
|
||||
|
||||
1. **plan** (optional, for complex tasks): analyze requirements, read AGENTS.md and relevant code, produce a step-by-step implementation plan.
|
||||
1. **task list**: create your task list for this run as your first action.
|
||||
|
||||
2. **setup**: checkout or create the branch:
|
||||
2. **plan** (optional, for complex tasks): analyze requirements, read AGENTS.md and relevant code, produce a step-by-step implementation plan.
|
||||
|
||||
3. **setup**: checkout or create the branch:
|
||||
- **PR event, modifying the existing PR**: call \`${t("checkout_pr")}\`
|
||||
- **new branch**: use \`${t("git")}\` to create a branch (\`git checkout -b pullfrog/branch-name\`)
|
||||
|
||||
3. **build**: implement changes using your native file and shell tools:
|
||||
4. **build**: implement changes using your native file and shell tools:
|
||||
- follow the plan (if you ran a plan phase)
|
||||
- plan your approach before writing code: identify which files need to change, key design decisions, and edge cases. for non-trivial changes, consider whether there's a more elegant approach.
|
||||
- run relevant tests/lints before committing
|
||||
|
||||
4. **self-review**: judgment call — does YOUR diff warrant a fresh-eyes pass?
|
||||
5. **self-review**: judgment call — does YOUR diff warrant a fresh-eyes pass?
|
||||
|
||||
Skip self-review (commit directly) when the diff is **genuinely trivial**:
|
||||
- doc typos, comment-only edits, whitespace/format-only, import reordering
|
||||
@@ -109,7 +201,7 @@ export function computeModes(agentId: AgentId): Mode[] {
|
||||
|
||||
Otherwise delegate the \`${REVIEWER_AGENT_NAME}\` subagent to review your diff with fresh eyes against YOUR TASK. The subagent's baked-in system prompt enforces a non-mutative + non-recursive contract: read-only file/search/web tools and read-only MCP queries only; no writes, shell side effects, state-changing MCP calls, or nested subagent dispatch. Enforcement is prose-only — restate the constraint in your dispatch instructions and do not relax it.
|
||||
|
||||
Provide the subagent with YOUR TASK, the output of \`git diff\`, and a tight summary (not raw output) of any lint/typecheck/test failures you fixed during build — what broke, root cause, the fix — so it can check that fixes addressed root causes rather than suppressed symptoms; say "no build-phase failures" if the build path was clean. Instruct it to flag bugs, logic errors, missing edge cases, gaps between request and diff, and unintended changes.
|
||||
Provide the subagent with YOUR TASK, the output of \`git diff origin/<base-branch>\` (single-rev form, no \`HEAD\` — this compares the working tree against the remote base and captures committed + staged + unstaged work; \`main...HEAD\` and \`--cached\` both miss the uncommitted edits Build self-review runs on, since self-review happens BEFORE the commit), and a tight summary (not raw output) of any lint/typecheck/test failures you fixed during build — what broke, root cause, the fix — so it can check that fixes addressed root causes rather than suppressed symptoms; say "no build-phase failures" if the build path was clean. Instruct it to flag bugs, logic errors, missing edge cases, gaps between request and diff, and unintended changes.
|
||||
|
||||
Delegation + research discipline (distilled from \`/anneal\` canonical — these are codified learnings from many review rounds, not theoretical best practices):
|
||||
- Do NOT summarize what you implemented — that biases the subagent toward validating the shape of your solution rather than questioning it.
|
||||
@@ -118,15 +210,13 @@ export function computeModes(agentId: AgentId): Mode[] {
|
||||
- Do NOT defect-hunt the diff yourself in parallel with the subagent. Your role is dispatch + evaluation; doing the review yourself reintroduces the implementation bias the subagent is meant to mitigate.
|
||||
- For diffs that rely on third-party API contracts, SDK semantics, framework directives, or DB engine specifics, instruct the subagent to verify load-bearing claims via web search and quote source URLs rather than trust training data — this is the single most common review-quality failure mode.
|
||||
|
||||
Review the findings, address valid points, and discard nitpicks or false positives. The reviewer is fallible — it biases toward *recommending additions* (defensive checks for impossible cases, extra logging, new abstractions used once, comments restating code, tests asserting tautologies, "just-in-case" guards). For each finding, ask: would applying it leave the code more sound, correct, AND elegant? Two-out-of-three is usually a signal to look harder for a fix that gets all three before settling for one that trades elegance for correctness. Reject bloat-shaped findings without applying them, and after applying the rest re-read your diff and be discerning about what *you just changed*: if any fix turned out to be bloat in context, revert it. The goal is code that is sound and correct *while remaining elegant*; the smallest diff that fixes the real defect almost always wins. Then verify only intended changes are present, no debug artifacts or commented-out code remain, no unrelated files were modified. Commit locally via shell (\`git add . && git commit -m "..."\`).
|
||||
Be **discerning** about what comes back. The reviewer is an AI subagent and is fallible — treat every finding as a hypothesis, not a directive, and **verify each one yourself** against the diff and the code before deciding whether to apply. You are searching for a solution that is **complete, minimal, and elegant** — you may need to think hard to find it. Do not over-engineer, do not be over-defensive, **do not write AI slop**. Reviewers bias toward *recommending additions*, and that bias has a recognizable slop texture: defensive checks for cases that cannot happen, extra logging, new abstractions used once, comments restating code, tests asserting tautologies, "just-in-case" guards, error handlers for cases the type system already rules out. Reject those. For each surviving finding, ask: would applying it leave the code more sound, correct, AND elegant? Two-out-of-three means look harder for a fix that gets all three before settling. After applying the fixes you accept, re-read your diff and be discerning about what *you just changed*: if any fix turned out to be bloat in context, revert it. Then verify only intended changes are present, no debug artifacts or commented-out code remain, no unrelated files were modified. Commit locally via shell (\`git add . && git commit -m "..."\`).
|
||||
|
||||
5. **finalize**:
|
||||
6. **finalize**:
|
||||
- confirm a clean working tree, then push via \`${t("push_branch")}\` (see *SYSTEM* Git rules if this fails — prepush errors are usually the repo's tests/lint, not infra timeouts)
|
||||
- create a PR via \`${t("create_pull_request")}\`
|
||||
- call \`${t("report_progress")}\` with the PR link or the exact error if push/PR failed
|
||||
|
||||
${learningsStep(t, 6)}
|
||||
|
||||
### Notes
|
||||
|
||||
For simple, well-defined tasks, skip the plan phase and go straight to build.`,
|
||||
@@ -137,51 +227,65 @@ For simple, well-defined tasks, skip the plan phase and go straight to build.`,
|
||||
"Address PR review feedback; respond to reviewer comments; make requested changes to an existing PR",
|
||||
prompt: `### Checklist
|
||||
|
||||
1. Checkout the PR branch via \`${t("checkout_pr")}\`.
|
||||
1. **task list**: create your task list for this run as your first action.
|
||||
|
||||
2. Fetch review comments via \`${t("get_review_comments")}\`.
|
||||
2. Checkout the PR branch via \`${t("checkout_pr")}\`.
|
||||
|
||||
3. For each comment:
|
||||
3. Fetch review comments via \`${t("get_review_comments")}\`.
|
||||
|
||||
4. For each comment:
|
||||
- understand the feedback
|
||||
- evaluate whether applying it would leave the code more **sound, correct, AND elegant**. reviewers are fallible and bias toward *recommending additions* (defensive checks for impossible cases, extra abstractions, comments restating obvious code, tests asserting tautologies, "just-in-case" guards). if a request would add bloat — ceremony without commensurate correctness benefit — push back in your reply rather than mechanically applying it. two-out-of-three is usually a signal to look harder for a fix that gets all three before settling.
|
||||
- **verify the finding yourself** against the actual code before deciding whether to apply — every comment (human or agent) is a hypothesis, not a directive. agent reviewers especially are fallible.
|
||||
- you are searching for a solution that is **complete, minimal, and elegant** — you may need to think hard to find it. do not over-engineer, do not be over-defensive, **do not write AI slop**. reviewers bias toward *recommending additions*, and that bias has a recognizable slop texture: defensive checks for impossible cases, extra abstractions used once, comments restating obvious code, tests asserting tautologies, "just-in-case" guards, error handlers for cases the type system already rules out. reject those. evaluate whether applying the finding would leave the code more **sound, correct, AND elegant**; two-out-of-three is a signal to look harder for a fix that gets all three. if a request would add bloat — ceremony without commensurate correctness benefit — push back in your reply rather than mechanically applying it.
|
||||
- if the request stands, make the code change using your native tools; otherwise reply explaining why
|
||||
- record what was done (or why nothing was done)
|
||||
|
||||
4. Quality check:
|
||||
5. Quality check:
|
||||
- test changes, then review the diff before committing — verify only intended changes are present, no debug artifacts remain, no fix turned out to be bloat in context (revert any that did), and the changes are clean enough that a senior engineer would approve without hesitation
|
||||
- commit locally via shell (\`git add . && git commit -m "..."\`)
|
||||
|
||||
5. Finalize:
|
||||
6. Finalize. Reply + resolve are paired write actions: do BOTH or NEITHER for each thread.
|
||||
- confirm a clean working tree, then push via \`${t("push_branch")}\` (same push/prepush guidance as Build mode in *SYSTEM*)
|
||||
- reply to each comment using \`${t("reply_to_review_comment")}\`
|
||||
- resolve addressed threads via \`${t("resolve_review_thread")}\`
|
||||
- call \`${t("report_progress")}\` with a brief summary (or the exact push error if push failed)
|
||||
|
||||
${learningsStep(t, 6)}`,
|
||||
- **if push fails**, call \`${t("report_progress")}\` with the exact error and STOP — do NOT reply or resolve any thread until the fix is live on the remote. Resolving a thread without the fix landing misleads the reviewer.
|
||||
- **on push success**, for each thread you acted on:
|
||||
- reply ONCE via \`${t("reply_to_review_comment")}\`. The \`comment_id\` parameter takes the root comment's numeric \`id=\` (from the first \`comment author=...\` tag in the \`${t("get_review_comments")}\` output) — NOT the \`thread=\` value; that's a separate GraphQL ID used by resolve. The runtime dedupes identical bodies within a session.
|
||||
- **immediately** call \`${t("resolve_review_thread")}\` with that thread's \`thread=\` value as \`thread_id\`. Resolve every thread where you (a) made the requested code change in full — partial fixes leave the thread open — OR (b) replied with a substantive answer the user explicitly asked for. Do NOT resolve threads where you pushed back on the request and the disagreement is unresolved; leave those open for the human to mediate.
|
||||
- call \`${t("report_progress")}\` with a brief summary`,
|
||||
},
|
||||
// Review and IncrementalReview use the multi-lens orchestrator pattern
|
||||
// (canonical source: .claude/commands/anneal.md). The orchestrator does
|
||||
// triage → parallel read-only subagent fan-out → aggregate → draft comments
|
||||
// → submit. For someone else's PR, parallel lenses (correctness, security,
|
||||
// research-validated claims, user-journey, etc.) provide breadth across
|
||||
// angles that a single subagent can't carry coherently. Build mode keeps
|
||||
// a single fresh-eyes subagent (different problem shape — orchestrator
|
||||
// wrote the code and bias-mitigation comes from delegating to one
|
||||
// subagent that doesn't share the implementation context).
|
||||
// Deliberate omission vs canonical /anneal: severity categorization in the
|
||||
// final message (the review body has its own CAUTION/IMPORTANT framing
|
||||
// instead of a severity table).
|
||||
// Review and IncrementalReview use a 0-or-2+ lens pattern. The default is
|
||||
// 0 lenses (orchestrator handles the review solo). Multi-lens (2+
|
||||
// reviewfrog subagents in parallel) only fires for substantive PRs or
|
||||
// high-stakes-subsystem touches — and when it fires, ALL lenses must
|
||||
// dispatch in a single assistant turn or the parallelism win disappears.
|
||||
// We never dispatch exactly one lens: a single lens is just a worse,
|
||||
// slower version of doing the work yourself.
|
||||
//
|
||||
// Build mode self-review is a different problem shape: the orchestrator
|
||||
// wrote the code, so bias-mitigation comes from delegating to one
|
||||
// fresh-eyes subagent that doesn't share the implementation context. A
|
||||
// single subagent there is appropriate; the 0-or-2+ rule applies only to
|
||||
// the Review/IncrementalReview lens fan-out where independence between
|
||||
// perspectives is what's being purchased.
|
||||
//
|
||||
// Severity categorization is split across two surfaces: the opening
|
||||
// callout (CAUTION/IMPORTANT/ℹ️/✅) sets the review's overall tier, and
|
||||
// per-bullet emoji prefixes (🚨/⚠️/ℹ️ in PR_SUMMARY_FORMAT) tag
|
||||
// individual points inside summary sections — scoping severity to the
|
||||
// specific bullet rather than the whole section keeps a section that
|
||||
// mixes a 🚨 and an ℹ️ from being mislabeled by either of them.
|
||||
{
|
||||
name: "Review",
|
||||
description:
|
||||
"Review code, PRs, or implementations; provide feedback or suggestions; identify issues; or check code quality, style, and correctness",
|
||||
prompt: `### Checklist
|
||||
|
||||
1. **checkout**: call \`${t("checkout_pr")}\` — this returns PR metadata and a \`diffPath\`. read the diff TOC end-to-end and treat its file line ranges as your coverage checklist.
|
||||
1. **task list**: create your task list for this run as your first action.
|
||||
|
||||
2. **triage**: orient yourself on the PR — identify *what kind of thing this is* (domain it touches, seams it crosses, external contracts it depends on, user-facing surfaces it changes). orientation only — defer specific defect-hunting to the subagents; pre-reviewing biases the lenses you pick. use \`${t("get_pull_request")}\` and other read-only GitHub tools for additional context if needed.
|
||||
2. **checkout**: call \`${t("checkout_pr")}\` — this returns PR metadata and a \`diffPath\`. read the diff TOC end-to-end and treat its file line ranges as your coverage checklist.
|
||||
|
||||
if the PR is **genuinely trivial**, skip steps 3–4 entirely and submit a \`No new issues found.\` review per step 5. there's no value in dispatching even one lens for a typo.
|
||||
3. **triage**: orient yourself on the PR — identify *what kind of thing this is* (domain it touches, seams it crosses, external contracts it depends on, user-facing surfaces it changes). pull as much context as you need to render a confident, well-grounded review: read related files, grep for callers of changed symbols, check tests that exercise the touched paths, fetch related GitHub state. **you are the synthesizer** — never delegate understanding to subagents.
|
||||
|
||||
if the PR is **genuinely trivial**, skip the fan-out entirely and submit a \`No new issues found.\` review per step 7.
|
||||
|
||||
"Genuinely trivial" (skip):
|
||||
- single-word doc typo, whitespace/format-only, comment-only across any number of files
|
||||
@@ -200,23 +304,25 @@ ${learningsStep(t, 6)}`,
|
||||
- any "typo fix" in user-facing copy that changes meaning ("approved" → "denied")
|
||||
- mixed diffs where a semantic 1-liner is buried in whitespace/formatting changes
|
||||
|
||||
When unsure, treat as non-trivial. The cost of one extra subagent is cents; the cost of a missed billing/auth/data bug is much more.
|
||||
4. **lens decision — 0 or 2+, NEVER 1**.
|
||||
|
||||
otherwise pick lenses by where the PR concentrates risk — **there's no fixed count**. lens count is judgment, not a formula. concrete shapes to anchor against:
|
||||
The default is **0 lenses**: handle the review yourself end-to-end. Most PRs land here.
|
||||
|
||||
- **1 lens** — pure refactor / mechanical rename across many files (impact); new test file with no source change (test-integrity); small isolated bug fix (correctness); doc-only PR with non-trivial technical content (research-validated or holistic)
|
||||
- **2–3 lenses (most PRs land here)** — new CRUD endpoint (correctness + security + test-integrity); new UI flow (user-journey + correctness); a single bug fix in a non-critical subsystem (correctness + test-integrity); design doc covering one domain (research-validated + correctness or holistic)
|
||||
- **4–5 lenses (high-stakes subsystem touches)** — any billing/payments change (billing-subsystem + correctness + security + operational-readiness); new auth flow (auth-subsystem + correctness + security + test-integrity); schema migration (schema-migration-subsystem + correctness + operational-readiness + impact); cross-subsystem PR that touches billing AND auth AND schema (one subsystem lens per domain + correctness)
|
||||
- **6+ lenses** — almost always a smell; you're either covering overlapping ground or this PR should have been split. push back via the review body rather than expanding lens count.
|
||||
Dispatch **2+ \`${REVIEWER_AGENT_NAME}\` lenses in parallel** ONLY when ALL of the following are true:
|
||||
- the PR is substantive (>5 files changed AND >200 net lines), OR touches a high-stakes subsystem (auth, billing, payments, schema migration, webhooks, secrets, RBAC, multi-tenant isolation, cron/scheduling)
|
||||
- you can name 2+ distinct concrete failure modes that warrant independent lenses (one lens per failure mode; orthogonal, not overlapping)
|
||||
- parallel-orchestrated independent perspectives meaningfully outperform what you'd find solo
|
||||
|
||||
lenses come in two flavors, and you can mix them:
|
||||
**NEVER dispatch exactly one lens.** A single lens is just a more expensive version of doing the work yourself with a worse model — it adds wall time and a context-handoff for no orthogonality benefit. Either you have at least two genuinely independent failure-mode hypotheses (dispatch all in one turn), or you don't (do the review yourself).
|
||||
|
||||
When you do go multi-lens, lens framings come in two flavors:
|
||||
- **themed lenses** — a perspective applied across the whole diff (correctness, security, user-journey, performance, etc.).
|
||||
- **subsystem lenses** — a domain-scoped frame for high-stakes subsystems the PR touches (e.g. "the auth lens", "the billing lens", "the schema-migration lens"). a subsystem lens is "review the PR specifically for what could go wrong in this subsystem" and naturally combines theme + scope. **for high-stakes domains, lead with the subsystem lens rather than the generic themed equivalent** — "billing-subsystem" outperforms "correctness on billing code" because the framing primes the subagent to remember domain-specific failure modes (double-charges, refund races, currency rounding, dispute flows) the generic lens misses.
|
||||
- **subsystem lenses** — a domain-scoped frame for high-stakes subsystems the PR touches (e.g. "the auth lens", "the billing lens", "the schema-migration lens"). **for high-stakes domains, lead with the subsystem lens rather than the generic themed equivalent** — "billing-subsystem" outperforms "correctness on billing code" because the framing primes the subagent to remember domain-specific failure modes (double-charges, refund races, currency rounding, dispute flows) the generic lens misses.
|
||||
|
||||
starter menu (combine, omit, or invent your own):
|
||||
- **correctness & invariants** — bugs, races, error handling, edge cases, state-machine boundaries
|
||||
- **impact** — when the PR removes features, deletes exports, renames identifiers, or changes architectural patterns: stale references in code, tests, docs (\`docs/\`, \`wiki/\`), comments, configs, UI
|
||||
- **research-validated assumptions** — third-party API contracts, SDK semantics, framework directives, version-gated behavior. the subagent must verify load-bearing claims via web search and quote source URLs.
|
||||
- **impact** — stale references in code/tests/docs/configs/UI after rename/remove
|
||||
- **research-validated assumptions** — third-party API contracts, SDK semantics, framework directives, version-gated behavior. **only pick when the PR's correctness depends on the contract behaving a specific way** — not when the API is merely used. The bar is "if the third-party contract differs from what the diff assumes, the PR is incorrect." When dispatched, the subagent must verify load-bearing claims via web search and quote source URLs.
|
||||
- **security** — new endpoints, authZ, input validation, secrets handling, replay/CSRF/injection, cross-tenant isolation
|
||||
- **user-journey** — UX-touching flows: walk through happy path and failure modes as a user
|
||||
- **operational readiness** — observability, alerting, migrations (forward + rollback), feature flags, on-call burden
|
||||
@@ -226,98 +332,164 @@ ${learningsStep(t, 6)}`,
|
||||
- **holistic** — does the PR make sense as a whole? symmetric flows (delete for every create, rollback for every migration)?
|
||||
- **subsystem lenses** (invent as the PR demands) — auth, billing, payments, schema migration, webhooks, secrets, RBAC, multi-tenant isolation, cron/scheduling, etc.
|
||||
|
||||
3. **fan out**: dispatch one \`${REVIEWER_AGENT_NAME}\` subagent per lens — its baked-in system prompt enforces the non-mutative + non-recursive contract (read-only file/search/web tools and read-only MCP queries; no writes, shell side effects, state-changing MCP calls, or nested subagent dispatch). when picking 2+ lenses, dispatch them in a **single assistant turn with multiple parallel subagent calls**; issuing one and awaiting reply before the next collapses the fan-out into a serial review. if a subagent errors out, times out, or returns nothing usable, retry once with the same lens; if it still fails, proceed with partial coverage and note the missing lens in the review body — do not skip step 3 entirely on a single subagent failure. each subagent gets:
|
||||
The only subagent type is \`${REVIEWER_AGENT_NAME}\` — used for lens judgment work ("is this safe / correct / well-tested?"), runs on a mid-tier model.
|
||||
|
||||
5. **fan out (only if step 4 said 2+ lenses)**: dispatch every \`${REVIEWER_AGENT_NAME}\` subagent for this run **IN A SINGLE ASSISTANT TURN, AS MULTIPLE PARALLEL TASK TOOL_USE BLOCKS IN ONE MESSAGE.**
|
||||
|
||||
⚠️ CRITICAL — PARALLELISM IS THE ONLY REASON LENSES EXIST. ⚠️
|
||||
The default tool-call behavior of Claude Code (and most agent runtimes) is **serial dispatch**: emit one Task call, await result, emit next, await, etc. This collapses your fan-out into a sequential review where each lens adds N × (orchestrator-think-time + lens-execution-time) to wall time. **YOU MUST OVERRIDE THIS DEFAULT.** Emit ALL of your Task tool_use blocks in the SAME assistant message, BEFORE you read ANY result from ANY of them. If you find yourself emitting one Task call, then thinking about the result, then emitting another — STOP and re-issue them all together. The whole point of going multi-lens is the wall-clock speedup from parallel execution; serial dispatch defeats it entirely.
|
||||
|
||||
✅ Right pattern: one assistant turn with N Task tool_use blocks → wait → N results arrive together → aggregate.
|
||||
❌ Wrong pattern: turn 1 = Task(lens A) → turn 2 (after A's result) = Task(lens B) → turn 3 (after B's result) = Task(lens C). This is the failure mode. Do not do this.
|
||||
|
||||
You can also include your own \`read\` / \`grep\` / \`webfetch\` calls in the SAME turn as the parallel \`${REVIEWER_AGENT_NAME}\` dispatches — concurrent context-pulling on the orchestrator side runs in parallel with the lens fan-out and costs zero extra wall time.
|
||||
|
||||
if a subagent errors out, times out, or returns nothing usable, retry once with the same lens; if it still fails, proceed with partial coverage and note the missing lens in the review body — do not skip the fan-out entirely on a single subagent failure. each subagent gets:
|
||||
- the diff path / target — reading the diff and the codebase is its job
|
||||
- **only one lens** — never a multi-section "review for X, Y, and Z" prompt
|
||||
- **a Task \`description\` set to the lens name** (e.g. \`"security"\`, \`"correctness"\`, \`"billing-subsystem"\`) — the harness reads this field to label the subagent's log lines so parallel runs can be told apart in CI output. without it, every subagent shows up as \`subagent#N\`.
|
||||
- the read-only contract restated in your dispatch instructions so the rule is present twice (the subagent's system prompt also enforces it). The test: would this call still be a no-op if reverted? If not (PR comments, branch pushes, issue updates, set_output, label changes, dependency installs, etc.), don't make it.
|
||||
- if the lens touches external contracts, instruct the subagent to verify load-bearing claims via web search rather than trust training data, and to quote source URLs in its reasoning. action runs are non-interactive — there's no human in the loop to catch "I'm pretty sure Stripe does X."
|
||||
- ask the subagent to report findings with file paths and NEW line numbers from the diff so you can anchor inline comments without re-reading the entire diff.
|
||||
|
||||
delegation discipline:
|
||||
- do NOT lens-review the diff yourself in parallel with the subagents (your job is dispatch + comment-drafting; doing the lens work yourself reintroduces the bias the fan-out avoids)
|
||||
- do NOT summarize the PR for them (biases toward a validation frame)
|
||||
- do NOT hand them a curated reading list (let them discover scope)
|
||||
- do NOT pre-shape their output with a finding schema
|
||||
- do NOT mention the other lenses (independence is the point — overlapping findings are a strong signal)
|
||||
|
||||
4. **aggregate & draft**: merge findings; de-dup overlaps (two lenses catching the same issue = higher-confidence signal); trace each finding yourself before accepting it. drop praise, style preferences, speculative/unverified claims, findings about pre-existing code unrelated to the PR (heuristic: if the finding's root cause lives in lines this PR added or modified, it's in scope; otherwise drop unless the PR plausibly introduced or amplified the regression), and anything not actionable. also drop **bloat-shaped findings** — proposed fixes that would add defensive checks for cases that can't happen, abstractions used once, comments restating obvious code, tests asserting tautologies, or "just-in-case" guards. subagents are fallible and bias toward recommending changes; the bar for an actionable inline comment is sound + correct + elegant. recommending a change that improves only one of the three (or worse, degrades elegance to nominally improve correctness) makes the codebase worse, not better.
|
||||
6. **aggregate & draft**: when the fan-out lands, merge findings; de-dup overlaps (two lenses catching the same issue = higher-confidence signal); trace each finding yourself before accepting it. drop praise, style preferences, speculative/unverified claims, findings about pre-existing code unrelated to the PR (heuristic: if the finding's root cause lives in lines this PR added or modified, it's in scope; otherwise drop unless the PR plausibly introduced or amplified the regression), and anything not actionable. also drop **bloat-shaped findings** — proposed fixes that would add defensive checks for cases that can't happen, abstractions used once, comments restating obvious code, tests asserting tautologies, or "just-in-case" guards. subagents are fallible and bias toward recommending changes; the bar for an actionable inline comment is sound + correct + elegant. recommending a change that improves only one of the three (or worse, degrades elegance to nominally improve correctness) makes the codebase worse, not better.
|
||||
|
||||
for surviving findings, draft inline comments with NEW line numbers from the diff. every comment must be actionable, 2-3 sentences max. use GitHub permalink format for code references. for impact-analysis findings (stale references after rename/remove), report them in the review body ordered by severity (runtime breakage > incorrect docs > stale comments) rather than as inline comments unless they're anchored to a specific line.
|
||||
**Hunt for non-anchored concerns before drafting.** After collecting your anchored findings, deliberately scan for concerns that have no specific line to point at — typically: deletion / cleanup plans for code the diff replaces or shadows; rollout sequencing (what happens to in-flight state during deploy / revert?); coverage gaps the diff implies but doesn't add; scope questions that only the human can answer (e.g. is the legacy path going away or is this a long-term dual track?); architectural risks the diff opens up that aren't a single-line bug. On substantial PRs (migrations, refactors, multi-file rewrites, version bumps that change runtime semantics), at least one such concern almost always exists; if you can't think of any, your bar is probably too high.
|
||||
|
||||
5. **submit**: ALWAYS submit exactly one review via \`${t("create_pull_request_review")}\`. Do NOT call \`report_progress\` — the review is the final record and the progress comment will be cleaned up automatically.
|
||||
for surviving findings, draft inline comments with NEW line numbers from the diff — attach a \`<details>Technical details</details>\` block to any inline comment whose fix is non-trivial or has cross-file implications (see Inline technical details in the format below). every comment must be actionable, 2-3 sentences max in the visible part. use GitHub permalink format for code references. for impact-analysis findings (stale references after rename/remove), report them in the review body ordered by severity (runtime breakage > incorrect docs > stale comments) rather than as inline comments unless they're anchored to a specific line.
|
||||
|
||||
7. **submit**: ALWAYS submit exactly one review via \`${t("create_pull_request_review")}\`. Do NOT call \`report_progress\` — the review is the final record and the progress comment will be cleaned up automatically.
|
||||
|
||||
note: the first create_pull_request_review submission may error with a one-time diff-coverage nudge listing unread TOC regions. retry the same call to proceed — optionally after reading the listed ranges. the pre-flight will not block again this session.
|
||||
|
||||
The review body is structured as: \`[optional alert blockquote]\` → \`[PR summary using the default format below]\`. Inline comments are passed via the \`comments\` parameter, not in the body.
|
||||
|
||||
- **critical issues** (blocks merge — bugs, security, data loss):
|
||||
The opening callout is what the author sees first — pick the one that matches what you want them to do. Five tiers, from loudest to friendliest:
|
||||
|
||||
- \`[!CAUTION]\` — large red banner. Reads as "this will break something."
|
||||
- \`[!IMPORTANT]\` — large purple banner. Reads as "you need to look at this before merging."
|
||||
- \`> ℹ️ ...\` — informational blockquote. Reads as "minor suggestions, nothing blocking."
|
||||
- \`> ✅ ...\` — green friendly blockquote. Reads as "no concerns, mergeable."
|
||||
|
||||
Two reinforcing levers: callout intensity (above) and \`approved\` (which gates the footer Fix-button affordance — Fix renders on every non-approving review, so \`approved: true\` suppresses it). Wrapping mergeable feedback in \`[!IMPORTANT]\` trains users to click Fix on reviews that don't need fixing. Pick the tier the author's actual next action justifies.
|
||||
|
||||
- **critical issues** (blocks merge — bugs, security, data loss, broken core flows):
|
||||
\`approved: false\`. Body opens with \`> [!CAUTION]\\n> This PR introduces ...\`, followed by the PR summary. Include all inline comments via \`comments\`.
|
||||
- **recommended changes** (non-critical):
|
||||
\`approved: false\`. Body opens with \`> [!IMPORTANT]\\n> Consider ...\`, followed by the PR summary. Include all inline comments via \`comments\`.
|
||||
- **must-address non-critical findings** (real consequences if shipped — incorrect behavior in non-critical paths, missing validation on user input, regressions the author should fix before merge):
|
||||
\`approved: false\`. Body opens with \`> [!IMPORTANT]\\n> ...\`, followed by the PR summary. Reserve this tier for findings with concrete fallout — do NOT use \`[!IMPORTANT]\` for nits, style preferences, or "consider also" suggestions. Include all inline comments via \`comments\`.
|
||||
- **minor suggestions only** (single-line nits, doc/comment polish, defer-able observations, "rough edges"):
|
||||
\`approved: false\`. Body opens with \`> ℹ️ No critical issues — minor suggestions inline.\\n\\n\` followed by the PR summary. Include all inline comments via \`comments\`. Vary the wording after the emoji to fit the review (e.g. "Minor suggestions only.", "Two rough edges worth a look."), but always keep the ℹ️ prefix and keep it short.
|
||||
- **informational observations** (mergeable as-is, nothing actionable — e.g. prior feedback addressed cleanly, surfacing a minor stale doc reference, calling out something noteworthy without recommending a change):
|
||||
\`approved: true\`. Body opens with \`> ✅ No new issues found.\\n\\n\` followed by the PR summary. Do NOT include inline \`comments\` — the ✅ signals "no action needed", which contradicts an actionable anchor; if a point is concrete enough to anchor to a line, downgrade the whole review to "minor suggestions only" (\`approved: false\`) instead.
|
||||
- **no actionable issues**:
|
||||
\`approved: true\`. Body opens with \`No new issues found.\` followed by the PR summary.
|
||||
\`approved: true\`. Body opens with \`> ✅ No new issues found.\\n\\n\` followed by the PR summary.
|
||||
|
||||
${PR_SUMMARY_FORMAT}`,
|
||||
},
|
||||
// IncrementalReview shares Review's multi-lens orchestrator pattern but
|
||||
// scopes the target to the incremental diff. The "issues must be NEW
|
||||
// since the last Pullfrog review" filter lives at aggregation time
|
||||
// (step 5), NOT in the subagent prompt — pushing the filter into
|
||||
// subagents matches the canonical anneal anti-pattern of "list known
|
||||
// pre-existing failures — don't flag these" and suppresses signal on
|
||||
// regressions the new commits amplified. The review body is just
|
||||
// "Reviewed changes" — a separate "Prior review feedback" checklist
|
||||
// would duplicate the rolling PR summary snapshot's record of what
|
||||
// earlier runs already addressed and add noise to the user-facing
|
||||
// body. Same severity-table omission as Review.
|
||||
// IncrementalReview shares Review's 0-or-2+ lens pattern AND its body
|
||||
// format (PR_SUMMARY_FORMAT), scoped to the incremental delta against the
|
||||
// prior pullfrog review. The "issues must be NEW since the last Pullfrog
|
||||
// review" filter lives at aggregation time (step 8), NOT in the subagent
|
||||
// prompt — pushing the filter into subagents matches the canonical anneal
|
||||
// anti-pattern of "list known pre-existing failures — don't flag these"
|
||||
// and suppresses signal on regressions the new commits amplified. A
|
||||
// separate "Prior review feedback" checklist would duplicate the rolling
|
||||
// PR summary snapshot's record of what earlier runs already addressed and
|
||||
// add noise to the user-facing body. Same opening-callout + per-bullet
|
||||
// emoji severity split as Review.
|
||||
{
|
||||
name: "IncrementalReview",
|
||||
description:
|
||||
"Re-review a PR after new commits are pushed; focus on new changes since the last review",
|
||||
prompt: `### Checklist
|
||||
|
||||
1. **checkout**: call \`${t("checkout_pr")}\` — this returns PR metadata, \`diffPath\` (full diff), and \`incrementalDiffPath\` (changes since last reviewed version, if available). read the diff TOC first and use its line ranges as your coverage checklist.
|
||||
1. **task list**: create your task list for this run as your first action.
|
||||
|
||||
2. **incremental scope**: if \`incrementalDiffPath\` is present, read it to see what changed since the last review. this is a range-diff that isolates the net changes, filtering out base branch noise. if not present, fall back to reviewing the full PR diff and determine what changed since Pullfrog's most recent review.
|
||||
2. **checkout**: call \`${t("checkout_pr")}\` — this returns PR metadata, \`diffPath\` (full diff), and \`incrementalDiffPath\` (changes since last reviewed version, if available). read the diff TOC first and use its line ranges as your coverage checklist.
|
||||
|
||||
3. **prior feedback**: fetch previous reviews via \`${t("list_pull_request_reviews")}\`. for the most recent Pullfrog review, call \`${t("get_review_comments")}\` with the review ID to retrieve specific prior line-level feedback. you'll use this to filter your aggregation in step 5 — anything already flagged in a prior review and not changed by the new commits should not be re-raised. you do NOT need to render this in the review body; the rolling PR summary snapshot is the durable record of what's been addressed.
|
||||
3. **incremental scope**: if \`incrementalDiffPath\` is present, read it to see what changed since the last review. this is a range-diff that isolates the net changes, filtering out base branch noise. if not present, fall back to reviewing the full PR diff and determine what changed since Pullfrog's most recent review.
|
||||
|
||||
4. **triage & fan out**: orient on the *incremental* changes — domain, seams, external contracts, user-facing surfaces.
|
||||
4. **prior feedback — read AND retire it**: fetch previous reviews via \`${t("list_pull_request_reviews")}\`, then call \`${t("get_review_comments")}\` on each prior Pullfrog review. Each thread renders as a section whose first line is a fenced tag \`comment author=<login> id=<fullDatabaseId> review=<reviewId> thread=<graphqlId>\`; section headers carry \`[RESOLVED]\` / \`[OUTDATED]\` when relevant. For every **open, Pullfrog-originated** thread, decide and act:
|
||||
|
||||
if the incremental changes are **genuinely trivial**, skip the fan-out entirely and jump to step 7's non-substantive path (do NOT submit a review).
|
||||
- **Pullfrog-originated** means the FIRST \`comment author=...\` tag in the section is \`author=pullfrog[bot]\`. The \`*\` marker on individual comments is unrelated — it flags whether a comment belongs to the queried review, not whether it is the thread root.
|
||||
- **addressed?** read the file at the thread's anchor and judge whether the substantive concern is now resolved by the new commits. Lines being modified isn't enough: reformatting, renaming, or moving the same code elsewhere doesn't address a concern. If the comment raised multiple distinct concerns, ALL must be addressed. The \`[OUTDATED]\` tag means GitHub moved the anchor (line shift, force-push, rename) — it does NOT mean the concern was addressed; re-read the code at its new location before deciding.
|
||||
- **if addressed**: call \`${t("reply_to_review_comment")}\` with the root tag's numeric \`id=\` as \`comment_id\` (NOT the \`thread=\` value — that's a separate GraphQL ID used only by resolve) and a one-line body (e.g. \`Addressed in <short-sha>.\`), then call \`${t("resolve_review_thread")}\` with the root tag's \`thread=\` value as \`thread_id\`. Do this BEFORE drafting the new review so the GitHub thread state aligns with the new review by the time it lands.
|
||||
- **if uncertain or partially addressed**: leave open. False-positive resolutions erode trust faster than false negatives.
|
||||
- **scope**: only retire Pullfrog-originated threads. Threads from human reviewers belong to those humans to resolve, even if the commit happened to address them.
|
||||
|
||||
The remaining open threads feed step 8's dedup filter — anything already flagged and unchanged by the new commits should not be re-raised. The rolling PR summary snapshot is the durable record of retire activity; you don't need to surface it in the review body.
|
||||
|
||||
5. **triage**: orient on the *incremental* changes — domain, seams, external contracts, user-facing surfaces. pull as much context as you need to render a confident review: read related files, grep for callers of changed symbols, check tests that exercise the touched paths. **you are the synthesizer.**
|
||||
|
||||
if the incremental changes are **genuinely trivial**, skip the fan-out entirely and jump to step 10's non-substantive path (do NOT submit a review).
|
||||
|
||||
"Genuinely trivial" (skip): formatting/comment tweaks, import reordering, lockfile regen, mechanical rename of import paths, whitespace-only.
|
||||
"Looks trivial but isn't" (do NOT skip — same anti-patterns as Review mode): 1-line changes to SQL/regex/auth/billing/permissions/signature-verification code; flipping feature-flag defaults or retry/timeout constants; money/tax/HTTP-method/redirect changes; tightening or loosening a comparison operator; mixed diffs with a semantic line buried in formatting.
|
||||
When unsure, treat as non-trivial.
|
||||
|
||||
otherwise pick lenses by where the new commits concentrate risk — **there's no fixed count**, same calibration as Review mode (1 lens for pure refactor / isolated fix; 2–3 for typical features; 4–5 for high-stakes subsystem touches; 6+ is a smell). lens framing follows Review mode: themed lenses (correctness & invariants, impact when new commits remove/rename/deprecate things, research-validated assumptions, security, user-journey, operational readiness, integration & cross-cutting, test integrity, performance, holistic) and subsystem lenses (auth, billing, schema migration, etc.) — for high-stakes domains lead with the subsystem lens rather than the generic themed equivalent.
|
||||
6. **lens decision — 0 or 2+, NEVER 1**.
|
||||
|
||||
dispatch one \`${REVIEWER_AGENT_NAME}\` subagent per lens — its baked-in system prompt enforces the non-mutative + non-recursive contract (read-only file/search/web tools and read-only MCP queries; no writes, shell side effects, state-changing MCP calls, or nested subagent dispatch). dispatch them in a **single assistant turn with multiple parallel subagent calls** (serial dispatch collapses the fan-out). if a subagent errors out, times out, or returns nothing usable, retry once with the same lens; if it still fails, proceed with partial coverage and note the missing lens in the review body — do not skip step 4 entirely on a single subagent failure. each subagent gets:
|
||||
- the diff scope (incremental diff path if available, full diff otherwise). do NOT tell them to skip pre-existing issues — that suppresses regressions the new commits amplified; the "issues must be NEW" filter lives at aggregation time (step 5), not in the subagent prompt
|
||||
The default is **0 lenses**: handle the re-review yourself end-to-end. Most incremental reviews land here — especially thread-reply re-reviews where the user is asking "did you address X?" rather than "review the diff again."
|
||||
|
||||
Dispatch **2+ \`${REVIEWER_AGENT_NAME}\` lenses in parallel** ONLY when ALL of the following are true:
|
||||
- the incremental changes are substantive (>5 files changed AND >200 net new lines), OR touch a high-stakes subsystem (auth, billing, payments, schema migration, webhooks, secrets, RBAC, multi-tenant isolation, cron/scheduling)
|
||||
- you can name 2+ distinct concrete failure modes the new commits plausibly introduce that warrant independent lenses
|
||||
- parallel-orchestrated independent perspectives meaningfully outperform what you'd find solo
|
||||
|
||||
**NEVER dispatch exactly one lens.** Single-lens dispatch adds wall time and cost for no orthogonality benefit. Either go multi-lens (≥2 in parallel) or do the re-review yourself.
|
||||
|
||||
Lens framing follows Review mode: themed lenses (correctness, security, etc.) and subsystem lenses (auth, billing, schema-migration, etc.) — for high-stakes domains lead with the subsystem lens.
|
||||
|
||||
7. **fan out (only if step 6 said 2+ lenses)**: dispatch every \`${REVIEWER_AGENT_NAME}\` subagent for this run **IN A SINGLE ASSISTANT TURN, AS MULTIPLE PARALLEL TASK TOOL_USE BLOCKS IN ONE MESSAGE.**
|
||||
|
||||
⚠️ CRITICAL — PARALLELISM IS THE ONLY REASON LENSES EXIST. ⚠️
|
||||
Default tool-call behavior is **serial dispatch**: emit one Task call, await result, emit next, await, etc. This collapses your fan-out into a sequential review where each lens adds N × (orchestrator-think-time + lens-execution-time) to wall time. **YOU MUST OVERRIDE THIS DEFAULT.** Emit ALL of your Task tool_use blocks in the SAME assistant message, BEFORE you read ANY result from ANY of them.
|
||||
|
||||
✅ Right pattern: one assistant turn with N Task tool_use blocks → wait → N results arrive together → aggregate.
|
||||
❌ Wrong pattern: turn 1 = Task(lens A) → turn 2 (after A's result) = Task(lens B). This is the failure mode.
|
||||
|
||||
You can also include your own \`read\` / \`grep\` / \`webfetch\` calls in the SAME turn as the parallel \`${REVIEWER_AGENT_NAME}\` dispatches.
|
||||
|
||||
if a subagent errors out, times out, or returns nothing usable, retry once with the same lens; if it still fails, proceed with partial coverage and note the missing lens in the review body. each subagent gets:
|
||||
- the diff scope (incremental diff path if available, full diff otherwise). do NOT tell them to skip pre-existing issues — that suppresses regressions the new commits amplified; the "issues must be NEW" filter lives at aggregation time (step 8), not in the subagent prompt
|
||||
- **only one lens** — never a multi-section "review for X, Y, and Z" prompt
|
||||
- **a Task \`description\` set to the lens name** (e.g. \`"security"\`, \`"correctness"\`, \`"billing-subsystem"\`) — the harness reads this field to label the subagent's log lines so parallel runs can be told apart in CI output. without it, every subagent shows up as \`subagent#N\`.
|
||||
- the read-only contract restated in your dispatch instructions so the rule is present twice (the subagent's system prompt also enforces it). The test: would this call still be a no-op if reverted? If not (PR comments, branch pushes, issue updates, set_output, label changes, dependency installs, etc.), don't make it.
|
||||
- if the lens touches external contracts, instruct the subagent to verify load-bearing claims via web search and quote source URLs. action runs are non-interactive — there's no human to catch "I'm pretty sure Stripe does X."
|
||||
- **a Task \`description\` set to the lens name** — the harness reads this field to label log lines so parallel runs can be told apart.
|
||||
- if the lens touches external contracts, instruct the subagent to verify load-bearing claims via web search and quote source URLs.
|
||||
- ask the subagent to report findings with file paths and NEW line numbers from the full PR diff so you can anchor inline comments.
|
||||
|
||||
delegation discipline:
|
||||
- do NOT lens-review the diff yourself in parallel with the subagents
|
||||
- do NOT summarize the changes for them (biases toward validation frame)
|
||||
- do NOT hand them a curated reading list (let them discover scope)
|
||||
- do NOT pre-shape their output with a finding schema
|
||||
- do NOT mention the other lenses (independence is the point)
|
||||
|
||||
5. **aggregate, draft, self-critique**: merge findings; de-dup overlaps; trace each finding yourself. drop praise, style preferences, speculative/unverified claims, findings about pre-existing code unrelated to the new commits, anything not actionable, and anything that re-states prior review feedback (heuristic: if the finding's root cause lives in lines the *new commits* added or modified, it's in scope; otherwise drop). also drop **bloat-shaped findings** — proposed fixes that would add defensive checks for cases that can't happen, abstractions used once, comments restating obvious code, tests asserting tautologies, or "just-in-case" guards. subagents are fallible and bias toward recommending changes; the bar for an actionable inline comment is sound + correct + elegant. recommending a change that improves only one of the three (or degrades elegance to nominally improve correctness) makes the codebase worse, not better. To compute "lines the new commits added or modified": if \`incrementalDiffPath\` from step 1 is present, use it directly. Otherwise, take the prior Pullfrog review's \`commit_id\` (returned alongside each entry from \`${t("list_pull_request_reviews")}\` in step 3) and run \`git diff <prior-review-sha>..HEAD\` to isolate the lines added since that review. draft inline comments with NEW line numbers from the full PR diff — every comment must be actionable, 2-3 sentences max.
|
||||
8. **aggregate, draft, self-critique**: merge findings (yours + any subagent output if you went multi-lens); de-dup overlaps; trace each finding yourself. drop praise, style preferences, speculative/unverified claims, findings about pre-existing code unrelated to the new commits, anything not actionable, and anything that re-states prior review feedback (heuristic: if the finding's root cause lives in lines the *new commits* added or modified, it's in scope; otherwise drop). also drop **bloat-shaped findings** — proposed fixes that would add defensive checks for cases that can't happen, abstractions used once, comments restating obvious code, tests asserting tautologies, or "just-in-case" guards. subagents are fallible and bias toward recommending changes; the bar for an actionable inline comment is sound + correct + elegant. recommending a change that improves only one of the three (or degrades elegance to nominally improve correctness) makes the codebase worse, not better. To compute "lines the new commits added or modified": if \`incrementalDiffPath\` from step 2 is present, use it directly. Otherwise, take the prior Pullfrog review's \`commit_id\` (returned alongside each entry from \`${t("list_pull_request_reviews")}\` in step 4) and run \`git diff <prior-review-sha>..HEAD\` to isolate the lines added since that review.
|
||||
|
||||
6. **build the review body** — a single "Reviewed changes" section: summarize at the logical-change level, not per-file. each bullet starts with a past-tense verb (e.g. \`- Extracted shared CLI runtime into a single module\`, \`- Renamed package to pullfrog\`). avoid file paths unless they add clarity. if the changes can be described in one sentence, use one sentence — no bullets needed. do NOT include a separate "Prior review feedback" checklist; that's tracked in the rolling PR summary snapshot for the next agent run, and surfacing it in the user-facing body is noise (changes that addressed prior feedback are already covered by the Reviewed-changes bullets). in some cases you may receive a complete diff for the whole pull request instead of an incremental one — when this happens, you will need to determine what changes have happened since Pullfrog's most recent review.
|
||||
**Hunt for non-anchored concerns before drafting.** After collecting your anchored findings, deliberately scan for concerns that have no specific line to point at — typically: deletion / cleanup plans for code the new commits replace or shadow; rollout sequencing (what happens to in-flight state during deploy / revert?); coverage gaps the new commits imply but don't add; scope questions that only the human can answer (e.g. is the legacy path going away or is this a long-term dual track?); architectural risks the new commits open up that aren't a single-line bug. On substantial incremental diffs (migrations, refactors, multi-file rewrites, version bumps that change runtime semantics), at least one such concern almost always exists; if you can't think of any, your bar is probably too high.
|
||||
|
||||
7. Submit — Do NOT call \`report_progress\` or \`create_issue_comment\` — the review is the final record and the progress comment will be cleaned up automatically. Follow these rules:
|
||||
draft inline comments with NEW line numbers from the full PR diff — attach a \`<details>Technical details</details>\` block to any inline comment whose fix is non-trivial or has cross-file implications (see Inline technical details in the format below). every comment must be actionable, 2-3 sentences max in the visible part.
|
||||
|
||||
9. **build the review body**: use the same default format as Review mode (preamble + optional cross-cutting \`### \` sections + optional \`### ℹ️ Nitpicks\`) — scoped to the **incremental delta**, not the full PR. The "Reviewed changes" bullets describe what changed since the prior pullfrog review (each bullet starts with a past-tense verb, e.g. \`- Extracted shared CLI runtime into a single module\`). Do NOT include a separate "Prior review feedback" checklist — that's tracked in the rolling PR summary snapshot for the next agent run, and surfacing it in the user-facing body is noise (changes that addressed prior feedback are already covered by the Reviewed-changes bullets). In some cases you may receive a complete diff for the whole PR instead of an incremental one; when this happens, determine what changed since Pullfrog's most recent review yourself before drafting bullets.
|
||||
|
||||
10. Submit — every run must end with EXACTLY ONE of \`${t("create_pull_request_review")}\` (substantive review) or \`${t("report_progress")}\` (no-review acknowledgement). do NOT call \`create_issue_comment\` for review output.
|
||||
|
||||
Same callout ladder as Review mode — \`[!CAUTION]\` (red, "will break") → \`[!IMPORTANT]\` (purple, "must address before merging") → \`> ℹ️ ...\` (informational, "minor suggestions only") → \`> ✅ ...\` (green friendly, "no concerns"). Same Fix-button lever: the footer renders a Fix button on every non-approving review, so \`approved: true\` suppresses it. Wrapping mergeable feedback in \`[!IMPORTANT]\` trains users to click Fix on reviews that don't need fixing — pick the tier the author's actual next action justifies.
|
||||
|
||||
Follow these rules:
|
||||
- note: the first create_pull_request_review submission may error with a one-time diff-coverage nudge listing unread TOC regions. retry the same call to proceed — optionally after reading the listed ranges. the pre-flight will not block again this session.
|
||||
- IF NO NEW ISSUES, NON-SUBSTANTIVE CHANGES ONLY (trivial formatting, import reordering, comment tweaks): do NOT submit a review. Do NOT call \`report_progress\`. Exit — the progress comment will be cleaned up automatically.
|
||||
- ELSE IF NEW CRITICAL ISSUES (blocks merge): call \`${t("create_pull_request_review")}\` with \`approved: false\`, all comments, and the review body. body opens with a GitHub alert blockquote (e.g. \`> [!CAUTION]\\n> This PR introduces ...\`), then the Reviewed-changes summary.
|
||||
- ELSE IF NEW RECOMMENDED CHANGES (non-critical): call \`${t("create_pull_request_review")}\` with \`approved: false\`, all comments, and the review body. body opens with \`> [!IMPORTANT]\\n> ...\` alert, then the Reviewed-changes summary.
|
||||
- ELSE IF NO NEW ISSUES, SUBSTANTIVE CHANGES (new functionality, behavior changes, or fixes to prior review feedback): call \`${t("create_pull_request_review")}\` to create a PR review. If all previous reviews have been properly addressed and no new issues were discovered, you can set \`approved: true\`. body opens with \`No new issues. Reviewed the following changes:\\n\`, then the Reviewed-changes summary.`,
|
||||
- IF NO NEW ISSUES, NON-SUBSTANTIVE CHANGES ONLY (trivial formatting, import reordering, comment tweaks): do NOT submit a review. Instead call \`${t("report_progress")}\` with a 1-2 sentence note explaining no review was warranted (e.g. "No new issues. Changes since last review are formatting-only."). this leaves a visible signal that the run completed.
|
||||
- ELSE IF NEW CRITICAL ISSUES (blocks merge — bugs, security, data loss, broken core flows): call \`${t("create_pull_request_review")}\` with \`approved: false\`, all comments, and the review body. body opens with \`> [!CAUTION]\\n> This PR introduces ...\`, followed by the PR summary using the default format below.
|
||||
- ELSE IF NEW MUST-ADDRESS NON-CRITICAL FINDINGS (real consequences if shipped — incorrect behavior, missing validation, regressions the author should fix before merge): call \`${t("create_pull_request_review")}\` with \`approved: false\`, all comments, and the review body. body opens with \`> [!IMPORTANT]\\n> ...\`, followed by the PR summary using the default format below. Do NOT use this tier for nits, style preferences, or "consider also" suggestions.
|
||||
- ELSE IF NEW MINOR SUGGESTIONS ONLY (single-line nits, doc/comment polish, defer-able observations, "rough edges"): call \`${t("create_pull_request_review")}\` with \`approved: false\`, all comments, and the review body. body opens with \`> ℹ️ No critical issues — minor suggestions inline.\\n\\n\` (vary the wording after ℹ️ to fit the review), followed by the PR summary using the default format below.
|
||||
- ELSE IF INFORMATIONAL OBSERVATIONS (mergeable as-is, but worth surfacing — e.g. prior feedback addressed cleanly with one minor stale doc reference, or a noteworthy positive observation): call \`${t("create_pull_request_review")}\` with \`approved: true\`, NO inline comments, and the review body. body opens with \`> ✅ No new issues found.\\n\\n\` (or similar friendly green opener), followed by the PR summary using the default format below. If a point is concrete enough to anchor to a line, downgrade the whole review to "minor suggestions only" (\`approved: false\`) instead — the ✅ signals "no action needed", which contradicts an actionable anchor.
|
||||
- ELSE IF NO NEW ISSUES, SUBSTANTIVE CHANGES (new functionality, behavior changes, or fixes to prior review feedback): call \`${t("create_pull_request_review")}\` to create a PR review. If all previous reviews have been properly addressed and no new issues were discovered, set \`approved: true\`. body opens with \`> ✅ No new issues found.\\n\\n\`, followed by the PR summary using the default format below.
|
||||
|
||||
${PR_SUMMARY_FORMAT}`,
|
||||
},
|
||||
{
|
||||
name: "Plan",
|
||||
@@ -325,15 +497,15 @@ ${PR_SUMMARY_FORMAT}`,
|
||||
"Create plans, break down tasks, outline steps, analyze requirements, understand scope of work, or provide task breakdowns",
|
||||
prompt: `### Checklist
|
||||
|
||||
1. Analyze the task and gather context:
|
||||
1. **task list**: create your task list for this run as your first action.
|
||||
|
||||
2. Analyze the task and gather context:
|
||||
- read AGENTS.md and relevant codebase files
|
||||
- understand the architecture and constraints
|
||||
|
||||
2. Produce a structured, actionable plan with clear milestones.
|
||||
3. Produce a structured, actionable plan with clear milestones.
|
||||
|
||||
3. Call \`${t("report_progress")}\` with the plan.
|
||||
|
||||
${learningsStep(t, 4)}`,
|
||||
4. Call \`${t("report_progress")}\` with the plan body. Do NOT set \`target_plan_comment\` — that flag is exclusively for revising an existing plan, and \`${t("select_mode")}\` will route you to a separate PlanEdit checklist when a prior plan comment exists for this issue.`,
|
||||
},
|
||||
{
|
||||
name: "Fix",
|
||||
@@ -341,46 +513,48 @@ ${learningsStep(t, 4)}`,
|
||||
"Fix CI failures; debug failing tests or builds; investigate and resolve check suite failures",
|
||||
prompt: `### Checklist
|
||||
|
||||
1. Checkout the PR branch via \`${t("checkout_pr")}\`.
|
||||
1. **task list**: create your task list for this run as your first action.
|
||||
|
||||
2. Fetch check suite logs via \`${t("get_check_suite_logs")}\`.
|
||||
2. Checkout the PR branch via \`${t("checkout_pr")}\`.
|
||||
|
||||
3. **CRITICAL**: verify the failure was INTRODUCED BY THIS PR before fixing. If unrelated, abort and report.
|
||||
3. Fetch check suite logs via \`${t("get_check_suite_logs")}\`.
|
||||
|
||||
4. Diagnose and fix:
|
||||
4. **CRITICAL**: verify the failure was INTRODUCED BY THIS PR before fixing. If unrelated, abort and report.
|
||||
|
||||
5. Diagnose and fix:
|
||||
- read the workflow file, reproduce locally with the EXACT same commands CI runs
|
||||
- fix the issue using your native file and shell tools
|
||||
- verify the fix by re-running the exact CI command
|
||||
- review the diff before committing — verify only the fix is present, no debug artifacts, no unrelated changes. the fix should be clean enough that a senior engineer would approve without hesitation.
|
||||
- commit locally via shell (\`git add . && git commit -m "..."\`)
|
||||
|
||||
5. Finalize:
|
||||
6. Finalize:
|
||||
- confirm a clean working tree, then push via \`${t("push_branch")}\` (same push/prepush guidance as Build mode in *SYSTEM*)
|
||||
- call \`${t("report_progress")}\` with the diagnosis and fix summary (or the exact push error if push failed)
|
||||
|
||||
${learningsStep(t, 6)}`,
|
||||
- call \`${t("report_progress")}\` with the diagnosis and fix summary (or the exact push error if push failed)`,
|
||||
},
|
||||
{
|
||||
name: "ResolveConflicts",
|
||||
description: "Resolve merge conflicts in a PR branch against the base branch",
|
||||
prompt: `### Checklist
|
||||
|
||||
1. **Setup**:
|
||||
1. **task list**: create your task list for this run as your first action.
|
||||
|
||||
2. **Setup**:
|
||||
- Call \`${t("checkout_pr")}\` to get the PR branch.
|
||||
- Call \`${t("get_pull_request")}\` to identify the base branch (e.g., 'main').
|
||||
- Call \`${t("git_fetch")}\` to fetch the base branch.
|
||||
|
||||
2. **Merge Attempt**:
|
||||
3. **Merge Attempt**:
|
||||
- Run \`git merge origin/<base_branch>\` via shell.
|
||||
- If it succeeds automatically, confirm a clean working tree, push via \`${t("push_branch")}\` (same push/prepush guidance as Build mode in *SYSTEM*), and call \`${t("report_progress")}\` with a brief success note or the exact push error if push failed — **then stop; do not run steps 3–4.**
|
||||
- If it fails (conflicts), resolve them manually (continue to steps 3–4).
|
||||
- If it succeeds automatically, confirm a clean working tree, push via \`${t("push_branch")}\` (same push/prepush guidance as Build mode in *SYSTEM*), and call \`${t("report_progress")}\` with a brief success note or the exact push error if push failed — **then stop; do not run steps 4–5.**
|
||||
- If it fails (conflicts), resolve them manually (continue to steps 4–5).
|
||||
|
||||
3. **Resolve Conflicts**:
|
||||
4. **Resolve Conflicts**:
|
||||
- Run \`git status\` or parse the merge output to find the list of conflicting files.
|
||||
- For each conflicting file: read it, find the conflict markers (\`<<<<<<<\`, \`=======\`, \`>>>>>>>\`), understand the code context, and rewrite the file with the correct resolution. Remove all markers.
|
||||
- Verify the file syntax is correct after resolution.
|
||||
|
||||
4. **Finalize**:
|
||||
5. **Finalize**:
|
||||
- Run a final verification (build/test) to ensure the resolution works.
|
||||
- \`git add . && git commit -m "resolve merge conflicts"\`
|
||||
- confirm a clean working tree, then push via \`${t("push_branch")}\` (same push/prepush guidance as Build mode in *SYSTEM*)
|
||||
@@ -392,23 +566,38 @@ ${learningsStep(t, 6)}`,
|
||||
"General-purpose tasks that don't fit other modes: answering questions, adding comments, labeling, running ad-hoc commands, or any direct request",
|
||||
prompt: `### Checklist
|
||||
|
||||
1. Analyze the task. For simple operations (labeling, commenting, answering questions, running a single command), handle directly.
|
||||
1. **task list**: create your task list for this run as your first action.
|
||||
|
||||
2. For substantial work — code changes across multiple files, multi-step investigations:
|
||||
2. Analyze the task. For simple operations (labeling, commenting, answering questions, running a single command), handle directly.
|
||||
|
||||
3. For substantial work — code changes across multiple files, multi-step investigations:
|
||||
- plan your approach before starting
|
||||
- use native file and shell tools for local operations
|
||||
- use ${pullfrogMcpName} MCP tools for GitHub/git operations
|
||||
- if code changes are needed: review your own diff before committing — verify only intended changes are present, no debug artifacts remain, and the changes are clean enough that a senior engineer would approve without hesitation
|
||||
|
||||
3. Finalize:
|
||||
4. Finalize:
|
||||
- if code changes were made, push to a pull request (new or existing) using \`${t("push_branch")}\` and \`${t("create_pull_request")}\` as needed. \`git status\` must be clean before you finish (see *SYSTEM* Git rules if push fails).
|
||||
- call \`${t("report_progress")}\` once with results — include exact tool errors if push or PR creation failed
|
||||
- if the task involved labeling, commenting, or other GitHub operations, perform those directly
|
||||
|
||||
${learningsStep(t, 4)}`,
|
||||
- if the task involved labeling, commenting, or other GitHub operations, perform those directly`,
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
// static export for UI display — uses opencode format as the readable default
|
||||
export const modes: Mode[] = computeModes("opencode");
|
||||
|
||||
/**
|
||||
* modes that legitimately never modify the working tree. used by the post-run
|
||||
* dirty-tree gate to suppress the "commit and push" nudge — those modes
|
||||
* complete by submitting a review (`Review` / `IncrementalReview`) or by
|
||||
* posting a Plan comment (`Plan`), not by touching files. any leftover in the
|
||||
* tree at end-of-run is incidental tool noise (e.g. a `node_modules/` from a
|
||||
* stray install attempt) on an ephemeral worktree; nudging the agent to
|
||||
* commit it would produce a spurious PR.
|
||||
*/
|
||||
export const NON_COMMITTING_MODES: ReadonlySet<string> = new Set([
|
||||
"Review",
|
||||
"IncrementalReview",
|
||||
"Plan",
|
||||
]);
|
||||
|
||||
+3
-2
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "pullfrog",
|
||||
"version": "0.1.0",
|
||||
"version": "0.1.11",
|
||||
"type": "module",
|
||||
"bin": {
|
||||
"pullfrog": "dist/cli.mjs",
|
||||
@@ -16,6 +16,7 @@
|
||||
"typecheck": "tsc --noEmit",
|
||||
"build": "node esbuild.config.js && tsc -p tsconfig.exports.json",
|
||||
"check:entrypoints": "node scripts/check-entrypoint-imports.ts",
|
||||
"docker": "node docker.ts",
|
||||
"play": "node play.ts",
|
||||
"runtest": "node test/run.ts",
|
||||
"scratch": "node scratch.ts",
|
||||
@@ -49,7 +50,7 @@
|
||||
"fastmcp": "^3.34.0",
|
||||
"file-type": "^21.3.0",
|
||||
"husky": "^9.0.0",
|
||||
"opencode-ai": "1.1.56",
|
||||
"opencode-ai": "1.15.1",
|
||||
"package-manager-detector": "^1.6.0",
|
||||
"picocolors": "^1.1.1",
|
||||
"semver": "^7.7.3",
|
||||
|
||||
@@ -1,22 +1,28 @@
|
||||
import { execSync } from "node:child_process";
|
||||
import { mkdtemp } from "node:fs/promises";
|
||||
import { devNull, tmpdir } from "node:os";
|
||||
// thin CLI for ad-hoc fixture runs against the Pullfrog action.
|
||||
//
|
||||
// invoke from the repo root:
|
||||
// pnpm play [args…] # host, in-process — fast iteration (default)
|
||||
// pnpm play:docker [args…] # local docker container that mocks GHA
|
||||
// pnpm docker play.ts [args…] # explicit container form (equivalent to `pnpm play:docker`)
|
||||
//
|
||||
// see wiki/docker.md for when host vs container matters.
|
||||
import { dirname, join, resolve } from "node:path";
|
||||
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
import arg from "arg";
|
||||
import { config } from "dotenv";
|
||||
import type { AgentResult } from "./agents/shared.ts";
|
||||
import { type Inputs, main } from "./main.ts";
|
||||
import type { Inputs } from "./main.ts";
|
||||
import { defineFixture } from "./test/utils.ts";
|
||||
import { log } from "./utils/cli.ts";
|
||||
import { runInDocker } from "./utils/docker.ts";
|
||||
import { ensureGitHubToken } from "./utils/github.ts";
|
||||
import { isInsideDocker } from "./utils/globals.ts";
|
||||
import { setupTestRepo } from "./utils/setup.ts";
|
||||
import { run } from "./utils/runFixture.ts";
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
config();
|
||||
config({ path: join(__dirname, "..", ".env") });
|
||||
|
||||
/**
|
||||
* default play fixture for ad-hoc testing.
|
||||
* change this freely without affecting any tests.
|
||||
* default fixture for ad-hoc `pnpm play` runs. change this freely without
|
||||
* affecting any tests — it's only consumed by this script's no-arg path.
|
||||
*/
|
||||
export const playFixture = defineFixture(
|
||||
{
|
||||
@@ -25,85 +31,6 @@ export const playFixture = defineFixture(
|
||||
{ localOnly: true }
|
||||
);
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = dirname(__filename);
|
||||
|
||||
// load action's .env file in case it exists for local dev
|
||||
config();
|
||||
// also load .env from repo root (for monorepo structure)
|
||||
config({ path: join(__dirname, "..", ".env") });
|
||||
|
||||
export async function run(inputsOrPrompt: Inputs | string): Promise<AgentResult> {
|
||||
await ensureGitHubToken();
|
||||
|
||||
// play.ts is a CI-emulator — isolate it from the developer's user- and
|
||||
// system-scope gitconfig so checks like `validatePushDestination` see the
|
||||
// raw stored remote URL instead of values mutated by `url.*.insteadOf`
|
||||
// rewrites (a common SSH-auth convenience on dev boxes). CI runners have
|
||||
// empty gitconfigs so this is a no-op there; locally it makes `pnpm play`
|
||||
// and real runs produce identical git state. `os.devNull` canonicalizes
|
||||
// the null device across Unix (`/dev/null`) and Windows (`\\.\nul`).
|
||||
process.env.GIT_CONFIG_GLOBAL = devNull;
|
||||
process.env.GIT_CONFIG_SYSTEM = devNull;
|
||||
|
||||
// create unique temp directory path in OS temp location for parallel execution
|
||||
// use a parent dir from mkdtemp, then clone into a 'repo' subdirectory
|
||||
const tempParent = await mkdtemp(join(tmpdir(), "pullfrog-play-"));
|
||||
const tempDir = join(tempParent, "repo");
|
||||
const originalCwd = process.cwd();
|
||||
|
||||
try {
|
||||
setupTestRepo({ tempDir });
|
||||
process.chdir(tempDir);
|
||||
|
||||
// run repo setup commands if provided (for pre-planting test state like symlinks).
|
||||
// this runs AFTER clone but BEFORE the agent, simulating pre-existing repo content.
|
||||
if (process.env.PULLFROG_TEST_REPO_SETUP) {
|
||||
log.info("» running repo setup commands...");
|
||||
execSync(process.env.PULLFROG_TEST_REPO_SETUP, { cwd: tempDir, stdio: "pipe" });
|
||||
}
|
||||
|
||||
// set GITHUB_WORKSPACE to tempDir so main() doesn't try to chdir to the CI checkout path
|
||||
process.env.GITHUB_WORKSPACE = tempDir;
|
||||
|
||||
// allow passing full Inputs object or just a prompt string
|
||||
const inputs: Inputs =
|
||||
typeof inputsOrPrompt === "string" ? { prompt: inputsOrPrompt } : inputsOrPrompt;
|
||||
|
||||
// set INPUT_* env vars for @actions/core.getInput()
|
||||
for (const [key, value] of Object.entries(inputs)) {
|
||||
if (value !== undefined && value !== null) {
|
||||
process.env[`INPUT_${key.toUpperCase()}`] = String(value);
|
||||
}
|
||||
}
|
||||
|
||||
const result: AgentResult = await main();
|
||||
|
||||
process.chdir(originalCwd);
|
||||
|
||||
if (result.success) {
|
||||
log.success("Action completed successfully");
|
||||
return { success: true, output: result.output || undefined, error: undefined };
|
||||
} else {
|
||||
log.error(`Action failed: ${result.error || "Unknown error"}`);
|
||||
return { success: false, error: result.error || undefined, output: undefined };
|
||||
}
|
||||
} catch (err) {
|
||||
const errorMessage = (err as Error).message;
|
||||
log.error(`Error: ${errorMessage}`);
|
||||
return { success: false, error: errorMessage, output: undefined };
|
||||
} finally {
|
||||
// cleanup temp directory - use sudo rm because sandbox isolation may create
|
||||
// files with different ownership that rmSync can't delete
|
||||
process.chdir(originalCwd);
|
||||
try {
|
||||
execSync(`sudo rm -rf "${tempParent}"`, { stdio: "ignore" });
|
||||
} catch {
|
||||
// ignore - cleanup failure is not critical
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const isDirectExecution = process.argv[1]
|
||||
? import.meta.url === pathToFileURL(resolve(process.argv[1])).href
|
||||
: false;
|
||||
@@ -112,71 +39,40 @@ if (isDirectExecution) {
|
||||
const args = arg({
|
||||
"--help": Boolean,
|
||||
"--raw": String,
|
||||
"--local": Boolean,
|
||||
"-h": "--help",
|
||||
"-l": "--local",
|
||||
});
|
||||
|
||||
if (args["--help"]) {
|
||||
log.info(`
|
||||
Usage: node play.ts [options]
|
||||
Usage: pnpm play [--raw <input>] (host, in-process; this entry)
|
||||
pnpm play:docker [--raw <input>] (local docker container that mocks GHA)
|
||||
|
||||
Test the Pullfrog action with the inline playFixture.
|
||||
Run the Pullfrog action against an inline fixture.
|
||||
|
||||
Options:
|
||||
--raw [input] Use raw string as prompt, or JSON object as full fixture
|
||||
--local, -l Run locally (default: runs in Docker)
|
||||
-h, --help Show this help message
|
||||
|
||||
Environment:
|
||||
PLAY_LOCAL=1 Same as --local
|
||||
--raw <input> raw string used as the prompt, or JSON object as full fixture
|
||||
-h, --help show this message
|
||||
|
||||
Examples:
|
||||
node play.ts # Run inline playFixture
|
||||
node play.ts --raw "Hello world" # Use raw string as prompt
|
||||
node play.ts --raw '{"prompt":"Hello","timeout":"5s"}' # Use JSON fixture
|
||||
pnpm play
|
||||
pnpm play --raw "Hello world"
|
||||
pnpm play --raw '{"prompt":"Hi","timeout":"5s"}'
|
||||
`);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
// default: run in Docker (unless --local, PLAY_LOCAL=1, or already inside Docker)
|
||||
const useLocal = args["--local"] || process.env.PLAY_LOCAL === "1" || isInsideDocker;
|
||||
|
||||
if (!useLocal) {
|
||||
const passArgs = process.argv
|
||||
.slice(2)
|
||||
.map((a) => `'${a.replace(/'/g, "'\\''")}'`)
|
||||
.join(" ");
|
||||
const nodeCmd = `node play.ts ${passArgs}`;
|
||||
|
||||
const volumeName = "pullfrog-action-node-modules";
|
||||
|
||||
const result = runInDocker({
|
||||
actionDir: __dirname,
|
||||
args: process.argv.slice(2),
|
||||
nodeCmd,
|
||||
volumeName,
|
||||
envFilterMode: "passthrough",
|
||||
onStart: () => log.info("» running in Docker container..."),
|
||||
});
|
||||
|
||||
process.exit(result.status ?? 1);
|
||||
}
|
||||
|
||||
if (args["--raw"]) {
|
||||
const raw = args["--raw"];
|
||||
// try to parse as JSON, otherwise treat as prompt string
|
||||
let input: Inputs | string = raw;
|
||||
try {
|
||||
input = JSON.parse(raw) as Inputs;
|
||||
} catch {
|
||||
// not valid JSON, use as prompt string
|
||||
// not valid JSON — treat as a literal prompt string.
|
||||
}
|
||||
const result = await run(input);
|
||||
process.exit(result.success ? 0 : 1);
|
||||
}
|
||||
|
||||
// no args - use inline playFixture
|
||||
const result = await run(playFixture);
|
||||
process.exit(result.success ? 0 : 1);
|
||||
}
|
||||
|
||||
Generated
+60
-49
@@ -84,8 +84,8 @@ importers:
|
||||
specifier: ^9.0.0
|
||||
version: 9.1.7
|
||||
opencode-ai:
|
||||
specifier: 1.1.56
|
||||
version: 1.1.56
|
||||
specifier: 1.15.1
|
||||
version: 1.15.1
|
||||
package-manager-detector:
|
||||
specifier: ^1.6.0
|
||||
version: 1.6.0
|
||||
@@ -1444,62 +1444,69 @@ packages:
|
||||
once@1.4.0:
|
||||
resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==}
|
||||
|
||||
opencode-ai@1.1.56:
|
||||
resolution: {integrity: sha512-OAF0G/1jVXpOrCh++M5gFEZ0bRLiXOnbxSYMFx5TOoD0OhCjHJS1JlARzaLDAx461qBnn+jocI9BBxD0wwFH3Q==}
|
||||
opencode-ai@1.15.1:
|
||||
resolution: {integrity: sha512-xLb1NuYZcMJ1p33hC/kgTMcJAueACVTfX6ps91a54GOFTM/wFp7br0t2cqHopEU9paqItbAnQwZB573qmPKH6w==}
|
||||
cpu: [arm64, x64]
|
||||
os: [darwin, linux, win32]
|
||||
hasBin: true
|
||||
|
||||
opencode-darwin-arm64@1.1.56:
|
||||
resolution: {integrity: sha512-0HqvLm7tcYZr4VJgEzi3Wicia5M9yCX66O7Cv470Qu4+GGbCC2sTTmzQu6pCehSleYgkZsiSBNRcFcW/6F7v0Q==}
|
||||
opencode-darwin-arm64@1.15.1:
|
||||
resolution: {integrity: sha512-eNgIfATsnHcud4Pr58OIR+TJGSsDvWmyNlfSDVVgP92qdnHFdZ5YsHKjcUGmeuuUN+oZwPb/z5nZSrkf+CCB2g==}
|
||||
cpu: [arm64]
|
||||
os: [darwin]
|
||||
|
||||
opencode-darwin-x64-baseline@1.1.56:
|
||||
resolution: {integrity: sha512-Z9QO9cTC9TnlUxTfEtDbvZbRFxc5Je8rs2Ei0cLH46W9gHGhh1fW4oX64tcrlhF5NUDKJzr/qwdoMfmKkeu53A==}
|
||||
opencode-darwin-x64-baseline@1.15.1:
|
||||
resolution: {integrity: sha512-XDx90Hhj+SPUxu0rqewsNR10JTny7+VE4C5pjWB04I6eoiEuBWy2EMvPXPg2FUA5Suz1PXXJ6yThfRtOxXNHuw==}
|
||||
cpu: [x64]
|
||||
os: [darwin]
|
||||
|
||||
opencode-darwin-x64@1.1.56:
|
||||
resolution: {integrity: sha512-8ZhNd4sFbDviT1OJs42C35Bx2/z6mkcb7uNhPHX3KKm27KOm7cYjFQa1UTF5or3ZpjkjPQpR/cpo7TELvnnmRA==}
|
||||
opencode-darwin-x64@1.15.1:
|
||||
resolution: {integrity: sha512-tNbzF6n+TczILEqo0adtup1ZXBgAcqftQd11+eQohGxtNAjmD7Z/gCTVpEzh9GlHUPzUEuREZ4gRAbJmPpafBQ==}
|
||||
cpu: [x64]
|
||||
os: [darwin]
|
||||
|
||||
opencode-linux-arm64-musl@1.1.56:
|
||||
resolution: {integrity: sha512-f30SmYX4xE2fUsnNl66dDX++8iTMI9PLXz7BRHhgXL4XAdMpUUJbOKIsw3ZUB0KWHwjR8cjkEJQOZxR3HPbrZg==}
|
||||
opencode-linux-arm64-musl@1.15.1:
|
||||
resolution: {integrity: sha512-UuoizYN32eTWmQT494bw70Sq4AByS0pGk46Mo/z+KzV+KTQlsDXRQrnKATKYFDqE2T3c1VsPM1KqRV/DqvnXxw==}
|
||||
cpu: [arm64]
|
||||
os: [linux]
|
||||
|
||||
opencode-linux-arm64@1.1.56:
|
||||
resolution: {integrity: sha512-wEmIEXiEKghurQYgKY5yFUFOmnT/QIlAbYnVeH73gSqOwlYFUUlowXLDJnF+3OGd6m000qalhCxjExL7qKINWw==}
|
||||
opencode-linux-arm64@1.15.1:
|
||||
resolution: {integrity: sha512-MG6tuLZqzDjHGeaotejhYuuv2USR0y3v8N+6g5gWPHScX/iJWkJDMFBeT6+KOV/CWawrGRqZfBDfdJSKirX2LQ==}
|
||||
cpu: [arm64]
|
||||
os: [linux]
|
||||
|
||||
opencode-linux-x64-baseline-musl@1.1.56:
|
||||
resolution: {integrity: sha512-EC8EGTJCgTZgGIMZdQjOKGXw02+igo0am1Ry6wrPrB0li7XBlYbY6dz8tL3FlknRVMLamtGtb3tXQ6tqbwjX0g==}
|
||||
opencode-linux-x64-baseline-musl@1.15.1:
|
||||
resolution: {integrity: sha512-Is50zWUqa9fIJ+tiDOpxENcgn2XBk0QKNEocbu/x9aOdpfFsHhtxe33zi/+9CNdSr+O/6y9jRAMGn7AirJyZlg==}
|
||||
cpu: [x64]
|
||||
os: [linux]
|
||||
|
||||
opencode-linux-x64-baseline@1.1.56:
|
||||
resolution: {integrity: sha512-jKOvhkNLcn0h6zmKX+hDdkODijSVDxjAGTZp8BPbJCCJVxnKll9dYfDMTqxi3YN6yXp6sBvyaJT1mcpf8knkeA==}
|
||||
opencode-linux-x64-baseline@1.15.1:
|
||||
resolution: {integrity: sha512-ExKWMk/6ULM9HBda2KKZJNE5Ejzaa51QWpr7+Ljv1AlazxQQZKwJfqcZcSNfk0YsgXDESw2w2dwBmOcMaxQZKA==}
|
||||
cpu: [x64]
|
||||
os: [linux]
|
||||
|
||||
opencode-linux-x64-musl@1.1.56:
|
||||
resolution: {integrity: sha512-9BFYBPgpY2RrUd7/Ul0VNh+6B9l0FmCNRUV4yYmeFGee9ZDgPem5YGicsKAMQqS/5X0jW3ZWje+KBbBQz4RMwQ==}
|
||||
opencode-linux-x64-musl@1.15.1:
|
||||
resolution: {integrity: sha512-feNjVo7XGjqFHf5lejxuyZIkNi9Yi4B2H3w+p2SF9vcyUdPaJnta2/6Os7Pf8kwElRs6EnWRyUO2JVg4hjAjjg==}
|
||||
cpu: [x64]
|
||||
os: [linux]
|
||||
|
||||
opencode-linux-x64@1.1.56:
|
||||
resolution: {integrity: sha512-0FP3BzLjn+a9naTtm9hpHibJ+eV1RkI+tItcDQXKwHtvzoBUvytDRP8v6TotEHNtNF0ZLAb3OGurwDIIdtLarg==}
|
||||
opencode-linux-x64@1.15.1:
|
||||
resolution: {integrity: sha512-mKRg+iHdwEYNDS+DYa9VQnN903zlw8FInCQRGpY155aR/AF1r3hIn+7IopOTDAwqkutL9vJWMXELxmNpPdaTQg==}
|
||||
cpu: [x64]
|
||||
os: [linux]
|
||||
|
||||
opencode-windows-x64-baseline@1.1.56:
|
||||
resolution: {integrity: sha512-zEokdohfoDjWzwULvlSq0Y/tRoVmz2/6GofYO936buPQf09cBcPqpAgLUjZV+pQI0Atyd05YG3BCO4PQxmXzdw==}
|
||||
opencode-windows-arm64@1.15.1:
|
||||
resolution: {integrity: sha512-M3Wz4U+hF8paqrBpOWPqOM16MhDDZsnb0EZc1fFdKMfu1a8g2oR3gtq1heQgUOKd8FHaeDQHvBYOqaaJdoaCmA==}
|
||||
cpu: [arm64]
|
||||
os: [win32]
|
||||
|
||||
opencode-windows-x64-baseline@1.15.1:
|
||||
resolution: {integrity: sha512-sFvI5sY4kijrkIt9qry34aqZASRo9jJKBLm6PH/zZbGdRtvFM32/n+A26Z/NDbowya8fOtj7MX2Ih5DvR9Md1A==}
|
||||
cpu: [x64]
|
||||
os: [win32]
|
||||
|
||||
opencode-windows-x64@1.1.56:
|
||||
resolution: {integrity: sha512-zm/oaWT5uGrW3DumKRHiqv2L2pKwrTvhsT4XUSfIPLQn5EvrYNU6bh3WFO4v71ZBjSIKx5Q7rEvTEQvFiFEZFQ==}
|
||||
opencode-windows-x64@1.15.1:
|
||||
resolution: {integrity: sha512-MdCBncbhpcImw3zjYBuoI+ZqfMR1uI4mc8KCltwIgI2DrxuOZNe66A/3feOhWd9MQQ2c2PSdyyJfW9PE0FA/Ow==}
|
||||
cpu: [x64]
|
||||
os: [win32]
|
||||
|
||||
@@ -3154,51 +3161,55 @@ snapshots:
|
||||
dependencies:
|
||||
wrappy: 1.0.2
|
||||
|
||||
opencode-ai@1.1.56:
|
||||
opencode-ai@1.15.1:
|
||||
optionalDependencies:
|
||||
opencode-darwin-arm64: 1.1.56
|
||||
opencode-darwin-x64: 1.1.56
|
||||
opencode-darwin-x64-baseline: 1.1.56
|
||||
opencode-linux-arm64: 1.1.56
|
||||
opencode-linux-arm64-musl: 1.1.56
|
||||
opencode-linux-x64: 1.1.56
|
||||
opencode-linux-x64-baseline: 1.1.56
|
||||
opencode-linux-x64-baseline-musl: 1.1.56
|
||||
opencode-linux-x64-musl: 1.1.56
|
||||
opencode-windows-x64: 1.1.56
|
||||
opencode-windows-x64-baseline: 1.1.56
|
||||
opencode-darwin-arm64: 1.15.1
|
||||
opencode-darwin-x64: 1.15.1
|
||||
opencode-darwin-x64-baseline: 1.15.1
|
||||
opencode-linux-arm64: 1.15.1
|
||||
opencode-linux-arm64-musl: 1.15.1
|
||||
opencode-linux-x64: 1.15.1
|
||||
opencode-linux-x64-baseline: 1.15.1
|
||||
opencode-linux-x64-baseline-musl: 1.15.1
|
||||
opencode-linux-x64-musl: 1.15.1
|
||||
opencode-windows-arm64: 1.15.1
|
||||
opencode-windows-x64: 1.15.1
|
||||
opencode-windows-x64-baseline: 1.15.1
|
||||
|
||||
opencode-darwin-arm64@1.1.56:
|
||||
opencode-darwin-arm64@1.15.1:
|
||||
optional: true
|
||||
|
||||
opencode-darwin-x64-baseline@1.1.56:
|
||||
opencode-darwin-x64-baseline@1.15.1:
|
||||
optional: true
|
||||
|
||||
opencode-darwin-x64@1.1.56:
|
||||
opencode-darwin-x64@1.15.1:
|
||||
optional: true
|
||||
|
||||
opencode-linux-arm64-musl@1.1.56:
|
||||
opencode-linux-arm64-musl@1.15.1:
|
||||
optional: true
|
||||
|
||||
opencode-linux-arm64@1.1.56:
|
||||
opencode-linux-arm64@1.15.1:
|
||||
optional: true
|
||||
|
||||
opencode-linux-x64-baseline-musl@1.1.56:
|
||||
opencode-linux-x64-baseline-musl@1.15.1:
|
||||
optional: true
|
||||
|
||||
opencode-linux-x64-baseline@1.1.56:
|
||||
opencode-linux-x64-baseline@1.15.1:
|
||||
optional: true
|
||||
|
||||
opencode-linux-x64-musl@1.1.56:
|
||||
opencode-linux-x64-musl@1.15.1:
|
||||
optional: true
|
||||
|
||||
opencode-linux-x64@1.1.56:
|
||||
opencode-linux-x64@1.15.1:
|
||||
optional: true
|
||||
|
||||
opencode-windows-x64-baseline@1.1.56:
|
||||
opencode-windows-arm64@1.15.1:
|
||||
optional: true
|
||||
|
||||
opencode-windows-x64@1.1.56:
|
||||
opencode-windows-x64-baseline@1.15.1:
|
||||
optional: true
|
||||
|
||||
opencode-windows-x64@1.15.1:
|
||||
optional: true
|
||||
|
||||
package-manager-detector@1.6.0: {}
|
||||
|
||||
@@ -135,14 +135,21 @@ export const installNodeDependencies: PrepDefinition = {
|
||||
}
|
||||
}
|
||||
|
||||
// get the frozen install command (or fallback to regular install)
|
||||
const resolved = resolveCommand(agent, "frozen", []) || resolveCommand(agent, "install", []);
|
||||
// frozen-lockfile install only. eager prep is non-mutating by contract:
|
||||
// we run it before the agent starts and any artifact it leaves in the
|
||||
// tree (e.g. a generated `package-lock.json`) trips the dirty-tree
|
||||
// post-run gate and produces a spurious PR. `frozen` commands
|
||||
// (`npm ci`, `pnpm install --frozen-lockfile`, etc.) fail cleanly
|
||||
// without modifying state when there's no lockfile, which is exactly
|
||||
// what we want — repos that need a non-frozen install must opt in via
|
||||
// a `setup` lifecycle hook (`action/utils/lifecycle.ts`).
|
||||
const resolved = resolveCommand(agent, "frozen", []);
|
||||
if (!resolved) {
|
||||
return {
|
||||
language: "node",
|
||||
packageManager,
|
||||
dependenciesInstalled: false,
|
||||
issues: [`no install command found for ${agent}`],
|
||||
issues: [`no frozen-install command available for ${agent}`],
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -104,6 +104,15 @@ function createRuntimeContext(): RuntimeContext {
|
||||
const env: NodeJS.ProcessEnv = { ...process.env };
|
||||
env.npm_config_registry = NPM_REGISTRY;
|
||||
env.COREPACK_NPM_REGISTRY = NPM_REGISTRY;
|
||||
// bypass customer-side release-age gates (npm's `min-release-age`, pnpm's
|
||||
// `minimumReleaseAge`) so our bootstrap can resolve the latest publish.
|
||||
// pullfrog's npm version is server-stamped from a SHA-pinned action ref the
|
||||
// customer already vets at the action layer — not a customer-vetted dep, so
|
||||
// the gate is the wrong affordance here. env beats .npmrc in both tools.
|
||||
// npm uses `npm_config_*`; pnpm v11+ requires `pnpm_config_*` (the v10→v11
|
||||
// migration renamed the prefix). tracked: #713
|
||||
env.npm_config_min_release_age = "0";
|
||||
env.pnpm_config_minimum_release_age = "0";
|
||||
const currentPath = process.env.PATH ?? "";
|
||||
env.PATH = currentPath ? `${nodeBinDir}${delimiter}${currentPath}` : nodeBinDir;
|
||||
|
||||
|
||||
@@ -1,38 +0,0 @@
|
||||
// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
|
||||
|
||||
exports[`latest model per provider snapshot > matches snapshot 1`] = `
|
||||
{
|
||||
"anthropic": {
|
||||
"modelId": "claude-opus-4-7",
|
||||
"releaseDate": "2026-04-16",
|
||||
},
|
||||
"deepseek": {
|
||||
"modelId": "deepseek-v4-pro",
|
||||
"releaseDate": "2026-04-24",
|
||||
},
|
||||
"google": {
|
||||
"modelId": "gemini-3.1-flash-lite",
|
||||
"releaseDate": "2026-05-07",
|
||||
},
|
||||
"moonshotai": {
|
||||
"modelId": "kimi-k2.6",
|
||||
"releaseDate": "2026-04-21",
|
||||
},
|
||||
"openai": {
|
||||
"modelId": "gpt-5.5-pro",
|
||||
"releaseDate": "2026-04-23",
|
||||
},
|
||||
"opencode": {
|
||||
"modelId": "gpt-5.5-pro",
|
||||
"releaseDate": "2026-04-24",
|
||||
},
|
||||
"openrouter": {
|
||||
"modelId": "x-ai/grok-4.3",
|
||||
"releaseDate": "2026-05-01",
|
||||
},
|
||||
"xai": {
|
||||
"modelId": "grok-4.3",
|
||||
"releaseDate": "2026-05-01",
|
||||
},
|
||||
}
|
||||
`;
|
||||
@@ -0,0 +1,75 @@
|
||||
import type { AgentResult, TestRunnerOptions, ValidationCheck } from "../utils.ts";
|
||||
import { defineFixture, getAgentOutput } from "../utils.ts";
|
||||
|
||||
/**
|
||||
* BYOK-no-keys fallback test — proves that an account configured for a
|
||||
* BYOK model (here: `anthropic/claude-opus`) but with no provider API
|
||||
* keys present in the runner env still gets a successful run by falling
|
||||
* back to a free OpenCode model.
|
||||
*
|
||||
* This was the structural failure that took out 15 accounts post-launch
|
||||
* before the fallback shipped: GH Actions secret references resolved to
|
||||
* empty strings (because the secrets didn't exist), the action launched
|
||||
* Claude Code with no key, the LLM provider 401'd, and the run died in
|
||||
* 20s with a synthesized "Invalid API key" message.
|
||||
*
|
||||
* The env block below empty-strings every known provider key — that's
|
||||
* exactly what GitHub Actions does when a `${{ secrets.X }}` reference
|
||||
* resolves to a missing secret. We verify:
|
||||
* 1. the run succeeded
|
||||
* 2. the fallback log line was emitted (proves the swap happened)
|
||||
*/
|
||||
const fixture = defineFixture(
|
||||
{
|
||||
prompt: "Reply with exactly the single character: 4",
|
||||
timeout: "5m",
|
||||
},
|
||||
{ localOnly: true }
|
||||
);
|
||||
|
||||
function validator(result: AgentResult): ValidationCheck[] {
|
||||
const output = getAgentOutput(result);
|
||||
const fellBack = /fell back from .* to opencode\/minimax-m2\.5-free/.test(output);
|
||||
return [
|
||||
{ name: "run_succeeded", passed: result.success },
|
||||
{ name: "fallback_logged", passed: fellBack },
|
||||
];
|
||||
}
|
||||
|
||||
export const test: TestRunnerOptions = {
|
||||
name: "byok-no-keys-fallback",
|
||||
fixture,
|
||||
validator,
|
||||
env: {
|
||||
// simulate every BYOK provider's secret being absent — same shape as
|
||||
// a fresh-install account whose user never configured any keys.
|
||||
ANTHROPIC_API_KEY: "",
|
||||
CLAUDE_CODE_OAUTH_TOKEN: "",
|
||||
OPENAI_API_KEY: "",
|
||||
OPENROUTER_API_KEY: "",
|
||||
GEMINI_API_KEY: "",
|
||||
GOOGLE_GENERATIVE_AI_API_KEY: "",
|
||||
XAI_API_KEY: "",
|
||||
DEEPSEEK_API_KEY: "",
|
||||
MOONSHOT_API_KEY: "",
|
||||
OPENCODE_API_KEY: "",
|
||||
AWS_BEARER_TOKEN_BEDROCK: "",
|
||||
AWS_ACCESS_KEY_ID: "",
|
||||
AWS_SECRET_ACCESS_KEY: "",
|
||||
BEDROCK_MODEL_ID: "",
|
||||
// configure a model that requires a BYOK key — the fallback only
|
||||
// engages when there's a configured model whose provider key is
|
||||
// absent, so we have to pin one. anthropic/claude-opus is the
|
||||
// most common first-run choice (it's the catalog "preferred" for
|
||||
// the anthropic provider).
|
||||
PULLFROG_MODEL: "anthropic/claude-opus",
|
||||
},
|
||||
tags: ["agnostic"],
|
||||
coverage: [
|
||||
"action/utils/byokFallback.ts",
|
||||
"action/utils/apiKeys.ts",
|
||||
"action/utils/agent.ts",
|
||||
"action/main.ts",
|
||||
"action/models.ts",
|
||||
],
|
||||
};
|
||||
@@ -96,4 +96,10 @@ export const test: TestRunnerOptions = {
|
||||
repoSetup,
|
||||
env: { PULLFROG_DISABLE_SECURITY_INSTRUCTIONS: "1" },
|
||||
tags: ["agnostic", "security"],
|
||||
coverage: [
|
||||
"action/utils/gitAuth.ts",
|
||||
"action/utils/gitAuthServer.ts",
|
||||
"action/mcp/git.ts",
|
||||
"action/mcp/checkout.ts",
|
||||
],
|
||||
};
|
||||
|
||||
@@ -104,4 +104,12 @@ export const test: TestRunnerOptions = {
|
||||
agentEnv,
|
||||
env: { PULLFROG_DISABLE_SECURITY_INSTRUCTIONS: "1" },
|
||||
tags: ["agnostic"],
|
||||
coverage: [
|
||||
"action/utils/gitAuth.ts",
|
||||
"action/utils/gitAuthServer.ts",
|
||||
"action/utils/lifecycle.ts",
|
||||
"action/toolState.ts",
|
||||
"action/mcp/git.ts",
|
||||
"action/mcp/checkout.ts",
|
||||
],
|
||||
};
|
||||
|
||||
@@ -92,4 +92,5 @@ export const test: TestRunnerOptions = {
|
||||
validator,
|
||||
env: { PULLFROG_DISABLE_SECURITY_INSTRUCTIONS: "1" },
|
||||
tags: ["agnostic", "security"],
|
||||
coverage: ["action/mcp/dependencies.ts", "action/utils/install.ts"],
|
||||
};
|
||||
|
||||
@@ -62,4 +62,12 @@ export const test: TestRunnerOptions = {
|
||||
agentEnv,
|
||||
env: { PULLFROG_DISABLE_SECURITY_INSTRUCTIONS: "1" },
|
||||
tags: ["agnostic"],
|
||||
coverage: [
|
||||
"action/utils/gitAuth.ts",
|
||||
"action/utils/gitAuthServer.ts",
|
||||
"action/utils/lifecycle.ts",
|
||||
"action/toolState.ts",
|
||||
"action/mcp/git.ts",
|
||||
"action/mcp/checkout.ts",
|
||||
],
|
||||
};
|
||||
|
||||
@@ -18,8 +18,8 @@ const fixture = defineFixture(
|
||||
3. Report if it succeeded
|
||||
|
||||
## Test 2: Tag Operations
|
||||
1. Create a local tag using the git MCP tool: git tag -a test-tag-enabled -m "test tag"
|
||||
2. Try push_tags tool with tag "test-tag-enabled"
|
||||
1. Create a local tag using the git MCP tool: git tag -a test-tag-enabled-\${RANDOM} -m "test tag"
|
||||
2. Try push_tags tool with the tag you just created
|
||||
3. Report if tag push succeeded
|
||||
|
||||
## Test 3: Branch Deletion (cleanup)
|
||||
@@ -74,4 +74,12 @@ export const test: TestRunnerOptions = {
|
||||
validator,
|
||||
env: { PULLFROG_DISABLE_SECURITY_INSTRUCTIONS: "1" },
|
||||
tags: ["agnostic"],
|
||||
coverage: [
|
||||
"action/utils/gitAuth.ts",
|
||||
"action/utils/gitAuthServer.ts",
|
||||
"action/utils/lifecycle.ts",
|
||||
"action/toolState.ts",
|
||||
"action/mcp/git.ts",
|
||||
"action/mcp/checkout.ts",
|
||||
],
|
||||
};
|
||||
|
||||
@@ -67,4 +67,12 @@ export const test: TestRunnerOptions = {
|
||||
validator,
|
||||
env: { PULLFROG_DISABLE_SECURITY_INSTRUCTIONS: "1" },
|
||||
tags: ["agnostic"],
|
||||
coverage: [
|
||||
"action/utils/gitAuth.ts",
|
||||
"action/utils/gitAuthServer.ts",
|
||||
"action/utils/lifecycle.ts",
|
||||
"action/toolState.ts",
|
||||
"action/mcp/git.ts",
|
||||
"action/mcp/checkout.ts",
|
||||
],
|
||||
};
|
||||
|
||||
@@ -29,4 +29,11 @@ export const test: TestRunnerOptions = {
|
||||
expectFailure: true,
|
||||
env: { PULLFROG_DISABLE_SECURITY_INSTRUCTIONS: "1" },
|
||||
tags: ["agnostic"],
|
||||
coverage: [
|
||||
"action/utils/timer.ts",
|
||||
"action/utils/subprocess.ts",
|
||||
"action/utils/exitHandler.ts",
|
||||
"action/utils/activity.ts",
|
||||
"action/mcp/selectMode.ts",
|
||||
],
|
||||
};
|
||||
|
||||
@@ -1,70 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# determines which agents need testing based on changed files.
|
||||
# reads changed file paths from stdin (JSON array or newline-delimited).
|
||||
# outputs a JSON array of agent names to stdout.
|
||||
#
|
||||
# only agents whose harness file changed AND are exported from index.ts are included.
|
||||
# shared.ts/index.ts/postRun.ts and other non-harness action changes fall back to opencode as a canary.
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
AGENTS_INDEX="$SCRIPT_DIR/../agents/index.ts"
|
||||
|
||||
# build the set of active agents from index.ts imports (portable, no -P)
|
||||
active_agents=()
|
||||
while IFS= read -r line; do
|
||||
[[ -n "$line" ]] && active_agents+=("$line")
|
||||
done < <(sed -n 's/.*from "\.\/\([^"]*\)\.ts".*/\1/p' "$AGENTS_INDEX" | grep -v shared)
|
||||
|
||||
# read stdin - auto-detect JSON array vs newline-delimited
|
||||
input=$(cat)
|
||||
if echo "$input" | jq -e 'type == "array"' > /dev/null 2>&1; then
|
||||
files=$(echo "$input" | jq -r '.[]')
|
||||
else
|
||||
files="$input"
|
||||
fi
|
||||
|
||||
is_active_agent() {
|
||||
local name="$1"
|
||||
for a in "${active_agents[@]}"; do
|
||||
[[ "$a" == "$name" ]] && return 0
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
# find which agent harness files changed
|
||||
changed_agents=()
|
||||
has_non_agent_change=false
|
||||
|
||||
while IFS= read -r file; do
|
||||
[[ -z "$file" ]] && continue
|
||||
case "$file" in
|
||||
action/agents/shared.ts|action/agents/index.ts|action/agents/postRun.ts)
|
||||
has_non_agent_change=true
|
||||
;;
|
||||
action/agents/*.ts)
|
||||
agent_name="$(basename "$file" .ts)"
|
||||
if is_active_agent "$agent_name"; then
|
||||
changed_agents+=("$agent_name")
|
||||
else
|
||||
# legacy/inactive agent file changed — treat as non-agent change
|
||||
has_non_agent_change=true
|
||||
fi
|
||||
;;
|
||||
action/*)
|
||||
has_non_agent_change=true
|
||||
;;
|
||||
esac
|
||||
done <<< "$files"
|
||||
|
||||
# output agents based on change type.
|
||||
# non-agent action changes always include opencode as a canary.
|
||||
if $has_non_agent_change; then
|
||||
changed_agents+=("opencode")
|
||||
fi
|
||||
|
||||
if [[ ${#changed_agents[@]} -gt 0 ]]; then
|
||||
printf '%s\n' "${changed_agents[@]}" | sort -u | jq -R . | jq -sc .
|
||||
else
|
||||
echo '[]'
|
||||
fi
|
||||
+25
-53
@@ -1,4 +1,3 @@
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { readdirSync, readFileSync } from "node:fs";
|
||||
import { dirname, join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
@@ -16,7 +15,7 @@ type WorkflowJob = {
|
||||
"runs-on": string;
|
||||
"timeout-minutes"?: number;
|
||||
permissions?: WorkflowPermissions;
|
||||
strategy?: { "fail-fast": boolean; matrix: Record<string, string[]> };
|
||||
strategy?: { "fail-fast": boolean; matrix: Record<string, unknown> };
|
||||
env?: Record<string, string>;
|
||||
steps?: unknown[];
|
||||
};
|
||||
@@ -57,12 +56,14 @@ const expectedAgents = Object.keys(agents).sort();
|
||||
const crossagentTests = getTestNamesFromDir("crossagent");
|
||||
const agnosticTests = getTestNamesFromDir("agnostic");
|
||||
const adhocTests = getTestNamesFromDir("adhoc");
|
||||
const dynamicAgentsExpression = "$" + "{{ fromJSON(needs.changes.outputs.agents) }}";
|
||||
|
||||
// all provider API key names + GITHUB_TOKEN + model overrides
|
||||
// all provider API key names + managed credentials (e.g. Codex auth blob)
|
||||
// + GITHUB_TOKEN + model overrides
|
||||
const expectedAgentEnvVars = [
|
||||
"GITHUB_TOKEN",
|
||||
...new Set(Object.values(providers).flatMap((p) => [...p.envVars])),
|
||||
...new Set(
|
||||
Object.values(providers).flatMap((p) => [...p.envVars, ...(p.managedCredentials ?? [])])
|
||||
),
|
||||
"PULLFROG_MODEL",
|
||||
].sort();
|
||||
|
||||
@@ -83,53 +84,22 @@ describe("ci workflow consistency", () => {
|
||||
const rootJob = rootWorkflow.jobs["action-agents"];
|
||||
const actionJob = actionWorkflow.jobs.agents;
|
||||
|
||||
it("root agent matrix uses dynamic output from changes job", () => {
|
||||
expect(rootJob.strategy!.matrix.agent).toBe(dynamicAgentsExpression);
|
||||
});
|
||||
|
||||
it("changed-agents.sh falls back to opencode when shared agent code changed", () => {
|
||||
const input = JSON.stringify(["action/agents/shared.ts"]);
|
||||
const output = execFileSync("bash", [join(__dirname, "changed-agents.sh")], {
|
||||
input,
|
||||
encoding: "utf-8",
|
||||
});
|
||||
expect(JSON.parse(output)).toEqual(["opencode"]);
|
||||
});
|
||||
|
||||
it("changed-agents.sh falls back to opencode for non-agent action changes", () => {
|
||||
const output = execFileSync("bash", [join(__dirname, "changed-agents.sh")], {
|
||||
input: JSON.stringify(["action/mcp/server.ts"]),
|
||||
encoding: "utf-8",
|
||||
});
|
||||
expect(JSON.parse(output)).toEqual(["opencode"]);
|
||||
});
|
||||
|
||||
it("changed-agents.sh includes opencode canary alongside changed agents", () => {
|
||||
const output = execFileSync("bash", [join(__dirname, "changed-agents.sh")], {
|
||||
input: JSON.stringify(["action/agents/opencode.ts", "action/mcp/server.ts"]),
|
||||
encoding: "utf-8",
|
||||
});
|
||||
expect(JSON.parse(output)).toEqual(["opencode"]);
|
||||
});
|
||||
|
||||
it("changed-agents.sh treats legacy agent files as non-agent changes", () => {
|
||||
const output = execFileSync("bash", [join(__dirname, "changed-agents.sh")], {
|
||||
input: JSON.stringify(["action/agents/codex.ts", "action/agents/gemini.ts"]),
|
||||
encoding: "utf-8",
|
||||
});
|
||||
expect(JSON.parse(output)).toEqual(["opencode"]);
|
||||
it("root agents matrix is wired to the dynamic matrix output", () => {
|
||||
const include = rootJob.strategy?.matrix.include;
|
||||
expect(typeof include).toBe("string");
|
||||
expect(include as string).toContain("fromJSON(needs.changes.outputs.matrix).agents");
|
||||
});
|
||||
|
||||
it("action agent matrix matches agents map", () => {
|
||||
expect([...actionJob.strategy!.matrix.agent].sort()).toEqual(expectedAgents);
|
||||
});
|
||||
|
||||
it("root test matrix matches crossagent/ directory", () => {
|
||||
expect([...rootJob.strategy!.matrix.test].sort()).toEqual(crossagentTests);
|
||||
expect((actionJob.strategy?.matrix.agent as string[])?.slice().sort()).toEqual(
|
||||
expectedAgents
|
||||
);
|
||||
});
|
||||
|
||||
it("action test matrix matches crossagent/ directory", () => {
|
||||
expect([...actionJob.strategy!.matrix.test].sort()).toEqual(crossagentTests);
|
||||
expect((actionJob.strategy?.matrix.test as string[])?.slice().sort()).toEqual(
|
||||
crossagentTests
|
||||
);
|
||||
});
|
||||
|
||||
it("permissions match between root and action", () => {
|
||||
@@ -149,8 +119,8 @@ describe("ci workflow consistency", () => {
|
||||
});
|
||||
|
||||
it("fail-fast is enabled in both", () => {
|
||||
expect(rootJob.strategy!["fail-fast"]).toBe(true);
|
||||
expect(actionJob.strategy!["fail-fast"]).toBe(true);
|
||||
expect(rootJob.strategy?.["fail-fast"]).toBe(true);
|
||||
expect(actionJob.strategy?.["fail-fast"]).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -158,12 +128,14 @@ describe("ci workflow consistency", () => {
|
||||
const rootJob = rootWorkflow.jobs["action-agnostic"];
|
||||
const actionJob = actionWorkflow.jobs.agnostic;
|
||||
|
||||
it("root test matrix matches agnostic/ directory", () => {
|
||||
expect([...rootJob.strategy!.matrix.test].sort()).toEqual(agnosticTests);
|
||||
it("root agnostic matrix is wired to the dynamic matrix output", () => {
|
||||
const include = rootJob.strategy?.matrix.include;
|
||||
expect(typeof include).toBe("string");
|
||||
expect(include as string).toContain("fromJSON(needs.changes.outputs.matrix).agnostic");
|
||||
});
|
||||
|
||||
it("action test matrix matches agnostic/ directory", () => {
|
||||
expect([...actionJob.strategy!.matrix.test].sort()).toEqual(agnosticTests);
|
||||
expect((actionJob.strategy?.matrix.test as string[])?.slice().sort()).toEqual(agnosticTests);
|
||||
});
|
||||
|
||||
it("permissions match between root and action", () => {
|
||||
@@ -183,8 +155,8 @@ describe("ci workflow consistency", () => {
|
||||
});
|
||||
|
||||
it("fail-fast is enabled in both", () => {
|
||||
expect(rootJob.strategy!["fail-fast"]).toBe(true);
|
||||
expect(actionJob.strategy!["fail-fast"]).toBe(true);
|
||||
expect(rootJob.strategy?.["fail-fast"]).toBe(true);
|
||||
expect(actionJob.strategy?.["fail-fast"]).toBe(true);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
/**
|
||||
* shared coverage / glob plumbing for the matrix builder.
|
||||
*
|
||||
* every test (`crossagent/`, `agnostic/`) and every provider entry
|
||||
* (`providers.ts`) declares a `coverage` array of repo-relative globs. on a PR
|
||||
* push, the `changes` job feeds the changed-file list into `matrix.ts`, which
|
||||
* intersects each entry's globs against the diff and emits only the entries
|
||||
* that need to run.
|
||||
*
|
||||
* `ALWAYS_RUN_ALL` is the escape hatch: any change to a file matched here
|
||||
* forces the full matrix (every test, every flagship, every alias). it
|
||||
* captures cross-cutting infrastructure where fan-out is unpredictable —
|
||||
* agent loader, MCP server boot, test runner itself. if a per-test glob
|
||||
* goes stale, this list and the on-`main`-full-matrix policy are the safety
|
||||
* nets — there's no completeness lint.
|
||||
*
|
||||
* `coverage` is optional on tests/providers; missing = always run (treat as
|
||||
* "any code change touches me"). default to defensive — opt into precision
|
||||
* by adding globs.
|
||||
*/
|
||||
|
||||
/** patterns that, when matched by any changed file, force the full matrix. */
|
||||
export const ALWAYS_RUN_ALL: string[] = [
|
||||
// agent loader + cross-agent shared code
|
||||
"action/agents/shared.ts",
|
||||
"action/agents/index.ts",
|
||||
"action/agents/postRun.ts",
|
||||
// test harness — changing these can affect every test
|
||||
"action/test/run.ts",
|
||||
"action/test/utils.ts",
|
||||
"action/test/matrix.ts",
|
||||
"action/test/list-aliases.ts",
|
||||
"action/test/coverage.ts",
|
||||
"action/test/providers.ts",
|
||||
// boot + lifecycle
|
||||
"action/main.ts",
|
||||
"action/index.ts",
|
||||
"action/cli.ts",
|
||||
"action/utils/setup.ts",
|
||||
"action/utils/install.ts",
|
||||
"action/utils/runFixture.ts",
|
||||
"action/utils/globals.ts",
|
||||
// local docker container plumbing (changes invalidate every test's environment)
|
||||
"action/Dockerfile",
|
||||
"action/docker-entrypoint.sh",
|
||||
"action/docker.ts",
|
||||
// MCP orchestrator (every test runs through it)
|
||||
"action/mcp/server.ts",
|
||||
"action/mcp/shared.ts",
|
||||
// dependency graph
|
||||
"action/package.json",
|
||||
"action/pnpm-lock.yaml",
|
||||
// workflow itself
|
||||
".github/workflows/test.yml",
|
||||
];
|
||||
|
||||
/**
|
||||
* expand a single brace group like `{a,b,c}` into an array of patterns.
|
||||
*
|
||||
* intentionally minimal: nested braces (`{a,{b,c}}`) and escaped braces are
|
||||
* NOT supported — coverage globs in this repo only need flat brace groups
|
||||
* (`{claude,opencode}.ts`). add complexity if a real use case emerges.
|
||||
*/
|
||||
function expandBraces(pattern: string): string[] {
|
||||
const m = pattern.match(/\{([^{}]+)\}/);
|
||||
if (!m || m.index === undefined) return [pattern];
|
||||
const before = pattern.slice(0, m.index);
|
||||
const after = pattern.slice(m.index + m[0].length);
|
||||
const opts = m[1].split(",");
|
||||
return opts.flatMap((opt) => expandBraces(`${before}${opt}${after}`));
|
||||
}
|
||||
|
||||
/** convert a glob pattern to a regex anchored at start + end. */
|
||||
function globToRegex(pattern: string): RegExp {
|
||||
const DSTAR = "\u0000DSTAR\u0000";
|
||||
let s = pattern.replace(/\*\*/g, DSTAR);
|
||||
s = s.replace(/[.+^$()|[\]\\]/g, "\\$&");
|
||||
s = s.replace(/\*/g, "[^/]*");
|
||||
s = s.replace(/\?/g, "[^/]");
|
||||
s = s.replaceAll(DSTAR, ".*");
|
||||
return new RegExp(`^${s}$`);
|
||||
}
|
||||
|
||||
/** does any path in `paths` match any glob in `patterns`? */
|
||||
export function anyMatch(paths: string[], patterns: string[]): boolean {
|
||||
if (patterns.length === 0) return false;
|
||||
const regexes = patterns.flatMap((p) => expandBraces(p)).map(globToRegex);
|
||||
return paths.some((path) => regexes.some((r) => r.test(path)));
|
||||
}
|
||||
|
||||
/**
|
||||
* decide whether an entry runs given changed files + its coverage globs.
|
||||
*
|
||||
* three short-circuits:
|
||||
* 1. `full` flag (e.g. main pushes, workflow_dispatch) → always run
|
||||
* 2. any changed file matches `ALWAYS_RUN_ALL` → run everything
|
||||
* 3. coverage missing or empty on the entry → run (defensive default)
|
||||
*
|
||||
* otherwise: run iff any changed file matches the entry's coverage globs.
|
||||
*
|
||||
* `coverage: []` is treated identically to `coverage: undefined` to avoid the
|
||||
* footgun where a future test author intends "skip on PRs" by passing an
|
||||
* empty array — silently skipping CI on every PR is worse than always running.
|
||||
*/
|
||||
export type ShouldRunInput = {
|
||||
changedFiles: string[];
|
||||
coverage: string[] | undefined;
|
||||
full: boolean;
|
||||
};
|
||||
|
||||
export function shouldRun(input: ShouldRunInput): boolean {
|
||||
if (input.full) return true;
|
||||
if (anyMatch(input.changedFiles, ALWAYS_RUN_ALL)) return true;
|
||||
if (input.coverage === undefined || input.coverage.length === 0) return true;
|
||||
return anyMatch(input.changedFiles, input.coverage);
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { detectCodexRefresh } from "../../utils/codexHome.ts";
|
||||
import type { AgentResult, TestRunnerOptions, ValidationCheck } from "../utils.ts";
|
||||
import { defineFixture } from "../utils.ts";
|
||||
|
||||
/**
|
||||
* codex-auth test — end-to-end Codex ChatGPT-subscription auth smoke.
|
||||
*
|
||||
* Pins openai/gpt-5.5 (in upstream opencode's Codex `ALLOWED_MODELS` allow
|
||||
* list) and runs the full opencode harness against the developer's / CI's
|
||||
* `CODEX_AUTH_JSON`. Exercises:
|
||||
*
|
||||
* - installCodexAuth() materializes auth.json at $HOME/.local/share/opencode/
|
||||
* with `expires: 0` (forces refresh on first request).
|
||||
* - opencode's CodexAuthPlugin routes openai requests through the ChatGPT
|
||||
* subscription instead of needing OPENAI_API_KEY.
|
||||
* - the refresh chain advances during the run (proving the refresh path
|
||||
* works end-to-end against live Codex auth servers).
|
||||
* - detectCodexRefresh() would surface the rotation to entryPost.ts for
|
||||
* write-back to Pullfrog's secret store.
|
||||
*
|
||||
* the post-hook itself runs in a separate GHA `post:` step and is not
|
||||
* invoked by `pnpm runtest`. instead, this test asserts the on-disk auth.json
|
||||
* state that the post-hook would consume, which is the genuine integration
|
||||
* boundary (everything past `detectCodexRefresh` is a single fetch + unit-
|
||||
* tested in codexHome.test.ts).
|
||||
*
|
||||
* requires `CODEX_AUTH_JSON` in the environment. dev-local: put it in
|
||||
* `.env`. CI: provisioned as `secrets.CODEX_AUTH_JSON` and forwarded by the
|
||||
* `action-agents` job env block in `.github/workflows/test.yml`.
|
||||
*/
|
||||
|
||||
const token = randomUUID();
|
||||
|
||||
const fixture = defineFixture(
|
||||
{
|
||||
prompt: `Call set_output with exactly this token and nothing else: ${token}`,
|
||||
shell: "restricted",
|
||||
push: "disabled",
|
||||
timeout: "4m",
|
||||
},
|
||||
{ localOnly: true }
|
||||
);
|
||||
|
||||
function parseOriginalRefresh(): string | null {
|
||||
const raw = process.env.CODEX_AUTH_JSON;
|
||||
if (!raw) return null;
|
||||
try {
|
||||
const parsed = JSON.parse(raw) as { tokens?: { refresh_token?: unknown } };
|
||||
const rt = parsed?.tokens?.refresh_token;
|
||||
return typeof rt === "string" && rt.length > 0 ? rt : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function validator(result: AgentResult): ValidationCheck[] {
|
||||
const setOutputCalled = result.structuredOutput !== null;
|
||||
const tokenMatches = result.structuredOutput === token;
|
||||
|
||||
// installCodexAuth() emits this log line with the absolute path; we use it
|
||||
// to find the per-test HOME (randomized inside runAgentStreaming).
|
||||
const pathMatch = result.output.match(/installed Codex auth at (\S+)/);
|
||||
const authPath = pathMatch?.[1];
|
||||
|
||||
let authMaterialized = false;
|
||||
let refreshRotated = false;
|
||||
|
||||
if (authPath) {
|
||||
try {
|
||||
const content = readFileSync(authPath, "utf8");
|
||||
authMaterialized = true;
|
||||
const originalRefresh = parseOriginalRefresh();
|
||||
if (originalRefresh) {
|
||||
refreshRotated = detectCodexRefresh({ authFileContent: content, originalRefresh }) !== null;
|
||||
}
|
||||
} catch {
|
||||
// authMaterialized stays false
|
||||
}
|
||||
}
|
||||
|
||||
return [
|
||||
{ name: "set_output", passed: setOutputCalled },
|
||||
{ name: "token_matches", passed: tokenMatches },
|
||||
{ name: "auth_materialized", passed: authMaterialized },
|
||||
{ name: "refresh_rotated", passed: refreshRotated },
|
||||
];
|
||||
}
|
||||
|
||||
export const test: TestRunnerOptions = {
|
||||
name: "codex-auth",
|
||||
fixture,
|
||||
validator,
|
||||
agents: ["opencode"],
|
||||
env: {
|
||||
PULLFROG_MODEL: "openai/gpt",
|
||||
PULLFROG_DISABLE_SECURITY_INSTRUCTIONS: "1",
|
||||
},
|
||||
coverage: [
|
||||
"action/utils/codexHome.ts",
|
||||
"action/entryPost.ts",
|
||||
"action/agents/{opencode,opencode_v2}.ts",
|
||||
],
|
||||
// forks + contributors without the Codex secret skip cleanly rather than
|
||||
// failing on `auth_materialized=✗` and (with fail-fast: true) cascading
|
||||
// cancellation across the rest of the matrix. CI on `pullfrog/app` and
|
||||
// dev-local with `.env` both have the secret and run the test as normal.
|
||||
skipIf: () => (process.env.CODEX_AUTH_JSON ? null : "CODEX_AUTH_JSON unset"),
|
||||
};
|
||||
@@ -43,4 +43,6 @@ export const test: TestRunnerOptions = {
|
||||
},
|
||||
repoSetup:
|
||||
'mkdir -p /tmp/pullfrog-mcp-secret && printf "%s" "$PULLFROG_MCP_SECRET" > /tmp/pullfrog-mcp-secret/secret.txt',
|
||||
// any MCP-layer change can affect repo-MCP merging; agents own MCP wiring.
|
||||
coverage: ["action/mcp/**", "action/agents/{claude,opencode,opencode_v2}.ts"],
|
||||
};
|
||||
|
||||
@@ -43,4 +43,5 @@ export const test: TestRunnerOptions = {
|
||||
validator,
|
||||
agentEnv,
|
||||
env: { PULLFROG_DISABLE_SECURITY_INSTRUCTIONS: "1" },
|
||||
coverage: ["action/mcp/shell.ts", "action/agents/{claude,opencode,opencode_v2}.ts"],
|
||||
};
|
||||
|
||||
@@ -52,4 +52,9 @@ export const test: TestRunnerOptions = {
|
||||
validator,
|
||||
agentEnv,
|
||||
env: { PULLFROG_DISABLE_SECURITY_INSTRUCTIONS: "1" },
|
||||
coverage: [
|
||||
"action/utils/normalizeEnv.ts",
|
||||
"action/mcp/shell.ts",
|
||||
"action/agents/{claude,opencode,opencode_v2}.ts",
|
||||
],
|
||||
};
|
||||
|
||||
@@ -44,4 +44,5 @@ export const test: TestRunnerOptions = {
|
||||
PULLFROG_DISABLE_SECURITY_INSTRUCTIONS: "1",
|
||||
PULLFROG_MODEL: "anthropic/claude-sonnet-4-6",
|
||||
},
|
||||
coverage: ["action/agents/claude.ts"],
|
||||
};
|
||||
|
||||
@@ -44,4 +44,9 @@ export const test: TestRunnerOptions = {
|
||||
PULLFROG_DISABLE_SECURITY_INSTRUCTIONS: "1",
|
||||
PULLFROG_MODEL: "anthropic/claude-sonnet-4-6",
|
||||
},
|
||||
coverage: [
|
||||
"action/agents/opencode.ts",
|
||||
"action/agents/opencode_v2.ts",
|
||||
"action/agents/opencodePlugin.ts",
|
||||
],
|
||||
};
|
||||
|
||||
@@ -2,13 +2,16 @@ import type { AgentResult, TestRunnerOptions, ValidationCheck } from "../utils.t
|
||||
import { defineFixture } from "../utils.ts";
|
||||
|
||||
/**
|
||||
* smoke test - validates agent can connect to API and call MCP tools.
|
||||
* verifies set_output tool is called with correct value.
|
||||
* smoke test — validates agent can connect to the API and call MCP tools.
|
||||
*
|
||||
* two tool calls (not one) on purpose: this is the canary that exercises the
|
||||
* 2nd model→agent round-trip across every providers-live flagship. bugs like
|
||||
* the Gemini `thought_signature` echo only fire after the first tool result
|
||||
* comes back. do not collapse to a single tool call.
|
||||
*/
|
||||
|
||||
const fixture = defineFixture(
|
||||
{
|
||||
prompt: `Call set_output with "SMOKE TEST PASSED".`,
|
||||
prompt: `First call the git tool with command "status" to confirm the repo is reachable. Then call set_output with exactly the literal string "SMOKE TEST PASSED".`,
|
||||
},
|
||||
{ localOnly: true }
|
||||
);
|
||||
@@ -29,4 +32,7 @@ export const test: TestRunnerOptions = {
|
||||
fixture,
|
||||
validator,
|
||||
env: { PULLFROG_DISABLE_SECURITY_INSTRUCTIONS: "1" },
|
||||
// canary: any agent harness change runs the smoke. shared MCP set_output
|
||||
// surface is also captured.
|
||||
coverage: ["action/agents/{claude,opencode,opencode_v2}.ts", "action/mcp/output.ts"],
|
||||
};
|
||||
|
||||
@@ -58,4 +58,9 @@ export const test: TestRunnerOptions = {
|
||||
validator,
|
||||
agentEnv,
|
||||
env: { PULLFROG_DISABLE_SECURITY_INSTRUCTIONS: "1" },
|
||||
coverage: [
|
||||
"action/utils/normalizeEnv.ts",
|
||||
"action/mcp/shell.ts",
|
||||
"action/agents/{claude,opencode,opencode_v2}.ts",
|
||||
],
|
||||
};
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
import type { AgentResult, TestRunnerOptions, ValidationCheck } from "../utils.ts";
|
||||
import { defineFixture } from "../utils.ts";
|
||||
|
||||
const fixture = defineFixture(
|
||||
{
|
||||
prompt: `Call set_output with "VERTEX CLAUDE SMOKE PASSED".`,
|
||||
},
|
||||
{ localOnly: true }
|
||||
);
|
||||
|
||||
function validator(result: AgentResult): ValidationCheck[] {
|
||||
const output = result.structuredOutput;
|
||||
const setOutputCalled = output !== null;
|
||||
const correctValue = setOutputCalled && /VERTEX CLAUDE SMOKE PASSED/i.test(output);
|
||||
|
||||
return [
|
||||
{ name: "set_output", passed: setOutputCalled },
|
||||
{ name: "correct_value", passed: correctValue },
|
||||
];
|
||||
}
|
||||
|
||||
export const test: TestRunnerOptions = {
|
||||
name: "vertex-claude",
|
||||
agents: ["claude"],
|
||||
fixture,
|
||||
validator,
|
||||
env: {
|
||||
PULLFROG_DISABLE_SECURITY_INSTRUCTIONS: "1",
|
||||
PULLFROG_MODEL: "vertex/byok",
|
||||
VERTEX_MODEL_ID: "claude-haiku-4-5@20251001",
|
||||
VERTEX_LOCATION: "global",
|
||||
},
|
||||
coverage: [
|
||||
"action/models.ts",
|
||||
"action/main.ts",
|
||||
"action/agents/claude.ts",
|
||||
"action/utils/{agent,apiKeys,vertex}.ts",
|
||||
],
|
||||
};
|
||||
@@ -0,0 +1,39 @@
|
||||
import type { AgentResult, TestRunnerOptions, ValidationCheck } from "../utils.ts";
|
||||
import { defineFixture } from "../utils.ts";
|
||||
|
||||
const fixture = defineFixture(
|
||||
{
|
||||
prompt: `Call set_output with "VERTEX OPENCODE SMOKE PASSED".`,
|
||||
},
|
||||
{ localOnly: true }
|
||||
);
|
||||
|
||||
function validator(result: AgentResult): ValidationCheck[] {
|
||||
const output = result.structuredOutput;
|
||||
const setOutputCalled = output !== null;
|
||||
const correctValue = setOutputCalled && /VERTEX OPENCODE SMOKE PASSED/i.test(output);
|
||||
|
||||
return [
|
||||
{ name: "set_output", passed: setOutputCalled },
|
||||
{ name: "correct_value", passed: correctValue },
|
||||
];
|
||||
}
|
||||
|
||||
export const test: TestRunnerOptions = {
|
||||
name: "vertex-opencode",
|
||||
agents: ["opencode"],
|
||||
fixture,
|
||||
validator,
|
||||
env: {
|
||||
PULLFROG_DISABLE_SECURITY_INSTRUCTIONS: "1",
|
||||
PULLFROG_MODEL: "vertex/byok",
|
||||
VERTEX_MODEL_ID: "gemini-2.5-flash",
|
||||
VERTEX_LOCATION: "global",
|
||||
},
|
||||
coverage: [
|
||||
"action/models.ts",
|
||||
"action/main.ts",
|
||||
"action/agents/opencode.ts",
|
||||
"action/utils/{agent,apiKeys,vertex}.ts",
|
||||
],
|
||||
};
|
||||
+73
-39
@@ -1,54 +1,88 @@
|
||||
/**
|
||||
* emits a JSON array of { slug, agent, name } entries for the `models-live`
|
||||
* matrix job. `agent` is auto-derived from the alias provider and matches the
|
||||
* harness the runtime would pick in production.
|
||||
* emits a JSON array of { slug, agent, name } entries for one of two CI matrix
|
||||
* jobs. `agent` mirrors the harness the runtime would pick in production
|
||||
* (anthropic/* → claude, everything else → opencode).
|
||||
*
|
||||
* set MATRIX_FILTER to a substring to restrict the matrix to matching aliases
|
||||
* — useful for iterating on a single provider without paying for every model.
|
||||
* MODE=aliases (default) — every alias. consumed by `models-live`, which runs
|
||||
* the cheap top-level CLI smoke per alias (`action/test/model-smoke.ts`) to
|
||||
* validate resolution + auth.
|
||||
*
|
||||
* passthrough pruning: openrouter/* aliases and keyed opencode/* aliases are
|
||||
* just routing-layer wrappers around models we already smoke-test directly
|
||||
* (anthropic/*, openai/*, google/*, etc). running every passthrough burns CI
|
||||
* minutes without catching anything the direct smoke doesn't. we keep one
|
||||
* canary per routing layer to validate the routing layer itself is alive;
|
||||
* slug-drift is caught separately by the `models-catalog` job. set
|
||||
* INCLUDE_ALL_PASSTHROUGHS=1 to bypass this for full validation.
|
||||
* MODE=flagships — one standard-tier model per provider. consumed by
|
||||
* `providers-live`, which runs the full harness smoke
|
||||
* (`pnpm runtest smoke <agent>`) to validate provider-class tool-calling
|
||||
* (e.g. Gemini schema sanitizer, OpenAI tool-call format). flagship slugs
|
||||
* live in `providers.ts` alongside their per-provider coverage globs.
|
||||
*
|
||||
* Every keyed alias is smoked — including `openrouter/*` and keyed `opencode/*`
|
||||
* passthroughs. They look like routing-layer wrappers but each one is a
|
||||
* distinct catalog entry on models.dev (under the `openrouter` / `opencode`
|
||||
* provider sections) that can drift independently of the upstream provider
|
||||
* mirror — testing the direct google entry tells you nothing about whether
|
||||
* the openrouter mirror has the same model id. The only entries pruned are
|
||||
* routing slugs (bedrock/byok) whose `resolve` is a sentinel that picks the
|
||||
* actual model id from a per-run env var.
|
||||
*
|
||||
* usage:
|
||||
* node action/test/list-aliases.ts
|
||||
* MODE=flagships node action/test/list-aliases.ts
|
||||
* MATRIX_FILTER=gemini node action/test/list-aliases.ts
|
||||
* INCLUDE_ALL_PASSTHROUGHS=1 node action/test/list-aliases.ts
|
||||
*
|
||||
* NOTE: the per-PR-precision matrix lives in `matrix.ts`, which calls into
|
||||
* this file. raw invocation here emits the unfiltered matrix.
|
||||
*/
|
||||
import { modelAliases } from "../models.ts";
|
||||
import { providers } from "./providers.ts";
|
||||
|
||||
function agentForSlug(slug: string): "claude" | "opencode" {
|
||||
return slug.startsWith("anthropic/") ? "claude" : "opencode";
|
||||
}
|
||||
export type MatrixEntry = {
|
||||
slug: string;
|
||||
agent: string;
|
||||
name: string;
|
||||
};
|
||||
|
||||
// one canary per routing layer — proves the routing surface (auth, tool-call
|
||||
// translation) is alive without re-testing every underlying model.
|
||||
const ROUTING_CANARIES = new Set(["openrouter/claude-sonnet", "opencode/claude-sonnet"]);
|
||||
|
||||
function isPrunablePassthrough(alias: (typeof modelAliases)[number]): boolean {
|
||||
if (ROUTING_CANARIES.has(alias.slug)) return false;
|
||||
if (alias.provider === "openrouter") return true;
|
||||
// opencode FREE models (big-pickle, mimo, minimax, gpt-5-nano) are unique
|
||||
// to opencode and used in prod — keep them. only prune the keyed mirrors.
|
||||
if (alias.provider === "opencode" && !alias.isFree) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
const filter = process.env.MATRIX_FILTER?.trim() ?? "";
|
||||
const includeAllPassthroughs = process.env.INCLUDE_ALL_PASSTHROUGHS === "1";
|
||||
|
||||
const matrix = modelAliases
|
||||
.filter((alias) => (filter ? alias.slug.toLowerCase().includes(filter.toLowerCase()) : true))
|
||||
.filter((alias) => includeAllPassthroughs || !isPrunablePassthrough(alias))
|
||||
.map((alias) => ({
|
||||
function toMatrixEntry(alias: (typeof modelAliases)[number]): MatrixEntry {
|
||||
return {
|
||||
slug: alias.slug,
|
||||
agent: agentForSlug(alias.slug),
|
||||
agent: alias.slug.startsWith("anthropic/") ? "claude" : "opencode",
|
||||
// readable display name (GHA renders slashes awkwardly in matrix job titles)
|
||||
name: alias.slug.replace("/", "-"),
|
||||
}));
|
||||
};
|
||||
}
|
||||
|
||||
process.stdout.write(JSON.stringify(matrix));
|
||||
const aliasBySlug = new Map(modelAliases.map((a) => [a.slug, a]));
|
||||
|
||||
export function buildAliasMatrix(opts: { filter?: string }): MatrixEntry[] {
|
||||
const filter = opts.filter ?? "";
|
||||
return modelAliases
|
||||
.filter((alias) => {
|
||||
if (filter && !alias.slug.toLowerCase().includes(filter)) return false;
|
||||
// routing slugs (bedrock/byok) need a per-run env var to pick the actual
|
||||
// model — there's no generic smoke test.
|
||||
if (alias.routing) return false;
|
||||
return true;
|
||||
})
|
||||
.map(toMatrixEntry);
|
||||
}
|
||||
|
||||
export function buildFlagshipMatrix(opts: { filter?: string }): MatrixEntry[] {
|
||||
const filter = opts.filter ?? "";
|
||||
return providers
|
||||
.map((p) => {
|
||||
const alias = aliasBySlug.get(p.flagship);
|
||||
if (!alias) {
|
||||
throw new Error(
|
||||
`list-aliases: flagship "${p.flagship}" missing from modelAliases — update providers.ts`
|
||||
);
|
||||
}
|
||||
return alias;
|
||||
})
|
||||
.filter((alias) => !filter || alias.slug.toLowerCase().includes(filter))
|
||||
.map(toMatrixEntry);
|
||||
}
|
||||
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
const mode = process.env.MODE === "flagships" ? "flagships" : "aliases";
|
||||
const filter = process.env.MATRIX_FILTER?.trim().toLowerCase() ?? "";
|
||||
const matrix =
|
||||
mode === "flagships" ? buildFlagshipMatrix({ filter }) : buildAliasMatrix({ filter });
|
||||
process.stdout.write(JSON.stringify(matrix));
|
||||
}
|
||||
|
||||
+227
@@ -0,0 +1,227 @@
|
||||
/**
|
||||
* unified CI matrix builder. emits the four matrices consumed by
|
||||
* `.github/workflows/test.yml`:
|
||||
*
|
||||
* - agents: crossagent tests × eligible agents (fan-out)
|
||||
* - agnostic: agnostic infrastructure tests (run with opencode)
|
||||
* - flagships: one harness smoke per provider (providers-live)
|
||||
* - aliases: one CLI smoke per model alias (models-live)
|
||||
*
|
||||
* input: a JSON array of repo-relative changed paths on stdin (the
|
||||
* `paths-filter` action's `*_files` output). PR pushes pass the diff;
|
||||
* `main` pushes and `workflow_dispatch` set FULL=1 to skip filtering and
|
||||
* emit every entry.
|
||||
*
|
||||
* each test/provider declares its own `coverage` globs colocated with the
|
||||
* test (`crossagent/`, `agnostic/`) or provider (`providers.ts`). the matrix
|
||||
* builder intersects coverage against the diff. a top-level `ALWAYS_RUN_ALL`
|
||||
* (see `coverage.ts`) bypasses filtering when test-harness or cross-cutting
|
||||
* agent code changes — keeps stale globs from silently skipping critical
|
||||
* tests on test runner / shared.ts churn.
|
||||
*
|
||||
* usage:
|
||||
* echo '["action/agents/opencode.ts"]' | node action/test/matrix.ts
|
||||
* FULL=1 node action/test/matrix.ts < /dev/null
|
||||
* MATRIX_FILTER=gemini FULL=1 node action/test/matrix.ts < /dev/null
|
||||
*/
|
||||
|
||||
import { existsSync, readdirSync, readFileSync } from "node:fs";
|
||||
import { dirname, join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { shouldRun } from "./coverage.ts";
|
||||
import { buildAliasMatrix, buildFlagshipMatrix } from "./list-aliases.ts";
|
||||
import { providers } from "./providers.ts";
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
type AgentEntry = { agent: string; test: string; name: string };
|
||||
type AgnosticEntry = { test: string; name: string };
|
||||
type SlugEntry = { slug: string; agent: string; name: string };
|
||||
|
||||
type MatrixOutput = {
|
||||
agents: AgentEntry[];
|
||||
agnostic: AgnosticEntry[];
|
||||
flagships: SlugEntry[];
|
||||
aliases: SlugEntry[];
|
||||
};
|
||||
|
||||
/**
|
||||
* extracted test metadata. parsed via regex from the test source — see
|
||||
* `parseTestFile`. dynamic-import is intentionally avoided: the GHA `changes`
|
||||
* job runs without `pnpm install`, and the real test modules transitively
|
||||
* import `@actions/core` etc. parsing keeps `matrix.ts` zero-dep.
|
||||
*/
|
||||
type ParsedTest = {
|
||||
name: string;
|
||||
agents: string[] | undefined;
|
||||
coverage: string[] | undefined;
|
||||
};
|
||||
|
||||
const STRING_LITERAL = /"((?:\\.|[^"\\])*)"/g;
|
||||
|
||||
function extractStringLiterals(source: string): string[] {
|
||||
const out: string[] = [];
|
||||
STRING_LITERAL.lastIndex = 0;
|
||||
let m: RegExpExecArray | null;
|
||||
// biome-ignore lint/suspicious/noAssignInExpressions: idiomatic regex iteration
|
||||
while ((m = STRING_LITERAL.exec(source))) {
|
||||
out.push(m[1]);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* extract a `key: [...]` array literal of strings from a test object. matches
|
||||
* line-leading indented `key:` to avoid colliding with the same word inside
|
||||
* prompts / template literals.
|
||||
*/
|
||||
function extractStringArray(source: string, key: string): string[] | undefined {
|
||||
const re = new RegExp(`^\\s+${key}:\\s*\\[([\\s\\S]*?)\\]`, "m");
|
||||
const m = source.match(re);
|
||||
if (!m) return undefined;
|
||||
return extractStringLiterals(m[1]);
|
||||
}
|
||||
|
||||
function parseTestFile(source: string): ParsedTest | null {
|
||||
// strip line comments — `//` inside string literals is rare in test files,
|
||||
// and the static parser doesn't need to be perfect (defensive default of
|
||||
// "missing coverage = always run" covers parse misses).
|
||||
const stripped = source.replace(/\/\/[^\n]*$/gm, "");
|
||||
const nameMatch = stripped.match(/^\s+name:\s*"([^"]+)"/m);
|
||||
if (!nameMatch) return null;
|
||||
return {
|
||||
name: nameMatch[1],
|
||||
agents: extractStringArray(stripped, "agents"),
|
||||
coverage: extractStringArray(stripped, "coverage"),
|
||||
};
|
||||
}
|
||||
|
||||
function loadDir(dir: string): ParsedTest[] {
|
||||
const dirPath = join(__dirname, dir);
|
||||
if (!existsSync(dirPath)) return [];
|
||||
const files = readdirSync(dirPath).filter((f) => f.endsWith(".ts"));
|
||||
const out: ParsedTest[] = [];
|
||||
for (const file of files) {
|
||||
const source = readFileSync(join(dirPath, file), "utf8");
|
||||
const parsed = parseTestFile(source);
|
||||
if (parsed) out.push(parsed);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* derive the active agent list from `agents/index.ts` so adding a new harness
|
||||
* file automatically wires it into the matrix. avoids dynamic-import
|
||||
* (transitively pulls `@actions/core` etc. — would explode in the no-install
|
||||
* `changes` job) by regex-parsing the imports the same way `parseTestFile`
|
||||
* handles tests.
|
||||
*/
|
||||
function loadAgents(): string[] {
|
||||
const indexPath = join(__dirname, "..", "agents", "index.ts");
|
||||
const source = readFileSync(indexPath, "utf8");
|
||||
const out: string[] = [];
|
||||
const re = /^\s*import\s+\{\s*(\w+)\s*\}\s+from\s+"\.\/(\w+)\.ts"/gm;
|
||||
let m: RegExpExecArray | null;
|
||||
// biome-ignore lint/suspicious/noAssignInExpressions: idiomatic regex iteration
|
||||
while ((m = re.exec(source))) {
|
||||
if (m[2] === "shared") continue;
|
||||
out.push(m[1]);
|
||||
}
|
||||
return out.sort();
|
||||
}
|
||||
|
||||
function readChangedFiles(): string[] {
|
||||
const raw = readFileSync(0, "utf8").trim();
|
||||
if (!raw) return [];
|
||||
const parsed: unknown = JSON.parse(raw);
|
||||
if (!Array.isArray(parsed)) {
|
||||
throw new Error("matrix: stdin must be a JSON array of changed paths");
|
||||
}
|
||||
return parsed.map((p) => {
|
||||
if (typeof p !== "string") {
|
||||
throw new Error(`matrix: non-string entry in changed paths: ${JSON.stringify(p)}`);
|
||||
}
|
||||
return p;
|
||||
});
|
||||
}
|
||||
|
||||
function buildAgentsMatrix(input: { changedFiles: string[]; full: boolean }): AgentEntry[] {
|
||||
const tests = loadDir("crossagent");
|
||||
const allAgents = loadAgents();
|
||||
const out: AgentEntry[] = [];
|
||||
for (const t of tests) {
|
||||
if (!shouldRun({ changedFiles: input.changedFiles, coverage: t.coverage, full: input.full })) {
|
||||
continue;
|
||||
}
|
||||
const agents = t.agents ?? allAgents;
|
||||
for (const agent of agents) {
|
||||
out.push({ agent, test: t.name, name: `${t.name}-${agent}` });
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function buildAgnosticMatrix(input: { changedFiles: string[]; full: boolean }): AgnosticEntry[] {
|
||||
const tests = loadDir("agnostic");
|
||||
const out: AgnosticEntry[] = [];
|
||||
for (const t of tests) {
|
||||
if (!shouldRun({ changedFiles: input.changedFiles, coverage: t.coverage, full: input.full })) {
|
||||
continue;
|
||||
}
|
||||
out.push({ test: t.name, name: t.name });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function buildFlagshipsMatrix(input: {
|
||||
changedFiles: string[];
|
||||
full: boolean;
|
||||
filter: string;
|
||||
}): SlugEntry[] {
|
||||
const all = buildFlagshipMatrix({ filter: input.filter });
|
||||
const byName = new Map(providers.map((p) => [p.flagship, p]));
|
||||
return all.filter((entry) => {
|
||||
const provider = byName.get(entry.slug);
|
||||
return shouldRun({
|
||||
changedFiles: input.changedFiles,
|
||||
coverage: provider?.coverage,
|
||||
full: input.full,
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function buildAliasesMatrix(input: {
|
||||
changedFiles: string[];
|
||||
full: boolean;
|
||||
filter: string;
|
||||
}): SlugEntry[] {
|
||||
const all = buildAliasMatrix({ filter: input.filter });
|
||||
const coverageByProvider = new Map(providers.map((p) => [p.name, p.coverage]));
|
||||
return all.filter((entry) => {
|
||||
const provider = entry.slug.split("/")[0];
|
||||
return shouldRun({
|
||||
changedFiles: input.changedFiles,
|
||||
coverage: coverageByProvider.get(provider),
|
||||
full: input.full,
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function main(): void {
|
||||
const full = process.env.FULL === "1";
|
||||
const filter = process.env.MATRIX_FILTER?.trim().toLowerCase() ?? "";
|
||||
const changedFiles = full ? [] : readChangedFiles();
|
||||
|
||||
const output: MatrixOutput = {
|
||||
agents: buildAgentsMatrix({ changedFiles, full }),
|
||||
agnostic: buildAgnosticMatrix({ changedFiles, full }),
|
||||
flagships: buildFlagshipsMatrix({ changedFiles, full, filter }),
|
||||
aliases: buildAliasesMatrix({ changedFiles, full, filter }),
|
||||
};
|
||||
|
||||
process.stdout.write(JSON.stringify(output));
|
||||
}
|
||||
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
main();
|
||||
}
|
||||
@@ -0,0 +1,182 @@
|
||||
/**
|
||||
* model-smoke: per-alias resolution + auth check that bypasses the Pullfrog
|
||||
* harness. resolves a model alias to its concrete provider/model + agent CLI,
|
||||
* invokes the CLI directly with a trivial "reply OK" prompt, and asserts the
|
||||
* provider replied. validates exactly the surface that changes when models.ts
|
||||
* changes — alias → resolve mapping, agent classification, env-var wiring —
|
||||
* without booting Docker, MCP, or the full agent runtime.
|
||||
*
|
||||
* tool-calling correctness is a property of the underlying model, not the
|
||||
* alias; the `providers-live` job runs the full harness smoke once per
|
||||
* provider (one standard-tier model each), which is enough.
|
||||
*
|
||||
* usage:
|
||||
* node action/test/model-smoke.ts --slug openai/gpt
|
||||
* PULLFROG_MODEL=openai/gpt node action/test/model-smoke.ts
|
||||
*/
|
||||
import { spawn } from "node:child_process";
|
||||
import { mkdtempSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { config } from "dotenv";
|
||||
import { modelAliases, resolveCliModel } from "../models.ts";
|
||||
import { installFromNpmTarball } from "../utils/install.ts";
|
||||
import { getDevDependencyVersion } from "../utils/version.ts";
|
||||
|
||||
config({ path: join(import.meta.dirname, "..", ".env") });
|
||||
config({ path: join(import.meta.dirname, "..", "..", ".env") });
|
||||
|
||||
const PROMPT = "Reply with exactly OK and nothing else.";
|
||||
const MATCH = /\bOK\b/i;
|
||||
// xai is the slowest provider in the matrix — winning xai/grok-4.3 jobs land
|
||||
// at 42-67s wall time (vs 23-41s for every other provider), brushing a 60s
|
||||
// ceiling and intermittently crossing it. 120s gives ~2x headroom on the
|
||||
// slowest provider observed in CI, with no downside on the fast-path
|
||||
// providers since the timer only fires on actual hangs.
|
||||
const TIMEOUT_MS = 120_000;
|
||||
|
||||
function parseSlug(): string {
|
||||
const argIdx = process.argv.indexOf("--slug");
|
||||
if (argIdx >= 0 && process.argv[argIdx + 1]) return process.argv[argIdx + 1];
|
||||
if (process.env.PULLFROG_MODEL) return process.env.PULLFROG_MODEL;
|
||||
throw new Error("model-smoke: pass --slug <alias> or set PULLFROG_MODEL");
|
||||
}
|
||||
|
||||
type Plan =
|
||||
| { agent: "opencode"; cliPath: string; args: string[] }
|
||||
| { agent: "claude"; cliPath: string; args: string[] };
|
||||
|
||||
async function plan(slug: string): Promise<Plan> {
|
||||
const alias = modelAliases.find((a) => a.slug === slug);
|
||||
if (!alias) throw new Error(`model-smoke: unknown alias "${slug}"`);
|
||||
if (alias.routing) {
|
||||
throw new Error(
|
||||
`model-smoke: ${slug} is a routing slug (no fixed model). pass an explicit Bedrock model ID via PULLFROG_MODEL or the workflow env block.`
|
||||
);
|
||||
}
|
||||
|
||||
// walk the fallback chain so deprecated aliases (those with `fallback` set,
|
||||
// e.g. opencode/mimo-v2-pro-free → opencode/big-pickle) hit their replacement
|
||||
// instead of the dead resolve target. mirrors production via resolveCliModel.
|
||||
const cliModel = resolveCliModel(slug);
|
||||
if (!cliModel) throw new Error(`model-smoke: fallback chain for "${slug}" is broken or cyclic`);
|
||||
|
||||
// anthropic/* aliases run through claude-code in production; everything else
|
||||
// (openai, google, xai, deepseek, moonshot, opencode, openrouter) runs through
|
||||
// opencode. mirrors the inline classification in list-aliases.ts toMatrixEntry().
|
||||
if (slug.startsWith("anthropic/")) {
|
||||
const cliPath = await installFromNpmTarball({
|
||||
packageName: "@anthropic-ai/claude-code",
|
||||
version: getDevDependencyVersion("@anthropic-ai/claude-code"),
|
||||
executablePath: "cli.js",
|
||||
installDependencies: false,
|
||||
});
|
||||
// claude expects a bare model id (e.g. "claude-sonnet-4-6"), not "anthropic/claude-sonnet-4-6"
|
||||
const bareModel = cliModel.split("/").slice(1).join("/");
|
||||
return {
|
||||
agent: "claude",
|
||||
cliPath,
|
||||
args: [cliPath, "-p", PROMPT, "--model", bareModel],
|
||||
};
|
||||
}
|
||||
|
||||
const cliPath = await installFromNpmTarball({
|
||||
packageName: "opencode-ai",
|
||||
version: getDevDependencyVersion("opencode-ai"),
|
||||
// v1.14+: postinstall.mjs renames the platform-specific binary to
|
||||
// `bin/opencode.exe` for every OS — see action/agents/opencode_v2.ts.
|
||||
executablePath: "bin/opencode.exe",
|
||||
installDependencies: true,
|
||||
});
|
||||
return {
|
||||
agent: "opencode",
|
||||
cliPath,
|
||||
args: ["run", "--model", cliModel, PROMPT],
|
||||
};
|
||||
}
|
||||
|
||||
type SpawnResult = { ok: boolean; output: string; reason: string };
|
||||
|
||||
function runCli(p: Plan, env: NodeJS.ProcessEnv): Promise<SpawnResult> {
|
||||
// claude's cli.js shebangs to env node, but we invoke node explicitly to
|
||||
// avoid PATH-resolution surprises in CI runners; opencode is a real binary.
|
||||
const command = p.agent === "claude" ? "node" : p.cliPath;
|
||||
|
||||
return new Promise((resolve) => {
|
||||
const child = spawn(command, p.args, { env, stdio: ["ignore", "pipe", "pipe"] });
|
||||
|
||||
let stdout = "";
|
||||
let stderr = "";
|
||||
child.stdout?.on("data", (chunk: Buffer) => {
|
||||
stdout += chunk.toString();
|
||||
});
|
||||
child.stderr?.on("data", (chunk: Buffer) => {
|
||||
stderr += chunk.toString();
|
||||
});
|
||||
|
||||
const timer = setTimeout(() => {
|
||||
child.kill("SIGKILL");
|
||||
}, TIMEOUT_MS);
|
||||
|
||||
child.on("close", (code, signal) => {
|
||||
clearTimeout(timer);
|
||||
const output = stdout + (stderr ? `\n---stderr---\n${stderr}` : "");
|
||||
if (signal === "SIGKILL") {
|
||||
resolve({ ok: false, output, reason: `timed out after ${TIMEOUT_MS / 1000}s` });
|
||||
return;
|
||||
}
|
||||
if (code !== 0) {
|
||||
resolve({ ok: false, output, reason: `exit ${code}` });
|
||||
return;
|
||||
}
|
||||
if (!MATCH.test(stdout)) {
|
||||
resolve({ ok: false, output, reason: "no OK in stdout" });
|
||||
return;
|
||||
}
|
||||
resolve({ ok: true, output, reason: "ok" });
|
||||
});
|
||||
|
||||
child.on("error", (err) => {
|
||||
clearTimeout(timer);
|
||||
resolve({ ok: false, output: stderr, reason: `spawn error: ${err.message}` });
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const slug = parseSlug();
|
||||
const tempDir = mkdtempSync(join(tmpdir(), "model-smoke-"));
|
||||
const homeDir = join(tempDir, "home");
|
||||
|
||||
// installFromNpmTarball reads PULLFROG_TEMP_DIR from process.env, not from
|
||||
// the spawn env, so we mutate process.env up-front. HOME/XDG_CONFIG_HOME are
|
||||
// redirected to keep the agent CLIs from picking up the dev user's config.
|
||||
process.env.PULLFROG_TEMP_DIR = tempDir;
|
||||
process.env.HOME = homeDir;
|
||||
process.env.XDG_CONFIG_HOME = join(homeDir, ".config");
|
||||
// opencode reads GOOGLE_GENERATIVE_AI_API_KEY for gemini; mirror the harness fallback.
|
||||
if (!process.env.GOOGLE_GENERATIVE_AI_API_KEY && process.env.GEMINI_API_KEY) {
|
||||
process.env.GOOGLE_GENERATIVE_AI_API_KEY = process.env.GEMINI_API_KEY;
|
||||
}
|
||||
|
||||
console.log(`» model-smoke ${slug}`);
|
||||
const p = await plan(slug);
|
||||
console.log(
|
||||
`» agent=${p.agent} cmd=${[p.agent === "claude" ? "node" : p.cliPath, ...p.args].join(" ")}`
|
||||
);
|
||||
|
||||
const result = await runCli(p, process.env);
|
||||
if (result.ok) {
|
||||
console.log(`✓ ${slug} (${p.agent})`);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
console.error(`✗ ${slug} (${p.agent}): ${result.reason}`);
|
||||
if (result.output) console.error(result.output);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
main().catch((err: unknown) => {
|
||||
console.error(err instanceof Error ? (err.stack ?? err.message) : String(err));
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -1,20 +1,30 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { type ModelProvider, modelAliases, providers } from "../models.ts";
|
||||
import { modelAliases, resolveDisplayAlias } from "../models.ts";
|
||||
|
||||
// ── catalog drift tests — main-only ─────────────────────────────────────────────
|
||||
// ── catalog drift tests ─────────────────────────────────────────────────────
|
||||
//
|
||||
// these tests fetch models.dev and openrouter.ai to verify that every alias in
|
||||
// models.ts still corresponds to a live, non-deprecated upstream model. upstream
|
||||
// catalog drift (new model ships, old model deprecated, etc.) causes failures
|
||||
// that are unrelated to any code change in the PR — so these run only on main.
|
||||
// that are unrelated to any code change in a typical PR — so these are gated
|
||||
// off for normal PRs and run only on main pushes plus PRs from the
|
||||
// `pullfrog/models-bump` branch (the bot-authored bump PR — this test IS the
|
||||
// integrity gate for its edits, so it has to run on the PR itself, not just
|
||||
// post-merge).
|
||||
//
|
||||
// the registry is kept in sync with upstreams by the `models-bump` cron
|
||||
// (`.github/workflows/models-bump.yml`), which scans models.dev every 12h and
|
||||
// opens a PR bumping `resolve` / `openRouterResolve` for any alias whose
|
||||
// upstream has shipped a newer GA version. these tests are the integrity gate
|
||||
// for that PR — they catch typos, removed models, and openrouter mismatches.
|
||||
//
|
||||
// run locally with `pnpm test:catalog`.
|
||||
// in CI, gated to push events on main.
|
||||
|
||||
type ModelsDevModel = {
|
||||
name: string;
|
||||
status?: string;
|
||||
release_date?: string;
|
||||
cost?: { input?: number; output?: number };
|
||||
};
|
||||
|
||||
type ModelsDevProvider = {
|
||||
@@ -35,6 +45,17 @@ describe("models.dev validity", async () => {
|
||||
const data = await api;
|
||||
|
||||
for (const alias of modelAliases) {
|
||||
// routing slugs (e.g. bedrock/byok) have no fixed `resolve` — the actual
|
||||
// model ID is read from a separate env var at run time. skip drift checks
|
||||
// since there's no models.dev entry to validate against.
|
||||
if (alias.routing) continue;
|
||||
|
||||
// aliases with a `fallback` are deprecated entries that legitimately point
|
||||
// at dead resolve targets — the fallback chain redirects callers to a live
|
||||
// model. skip both existence and deprecation checks; the terminal-fallback
|
||||
// is validated separately by the Zen served-list test below.
|
||||
if (alias.fallback) continue;
|
||||
|
||||
const parsed = parseResolve(alias.resolve);
|
||||
|
||||
it(`${alias.resolve} exists on models.dev`, () => {
|
||||
@@ -47,13 +68,11 @@ describe("models.dev validity", async () => {
|
||||
).toBeDefined();
|
||||
});
|
||||
|
||||
if (!alias.fallback) {
|
||||
it(`${alias.resolve} is not deprecated`, () => {
|
||||
const model = data[parsed.provider]?.models[parsed.modelId];
|
||||
if (!model) return; // covered by existence test above
|
||||
expect(model.status, `${alias.resolve} is deprecated on models.dev`).not.toBe("deprecated");
|
||||
});
|
||||
}
|
||||
it(`${alias.resolve} is not deprecated`, () => {
|
||||
const model = data[parsed.provider]?.models[parsed.modelId];
|
||||
if (!model) return; // covered by existence test above
|
||||
expect(model.status, `${alias.resolve} is deprecated on models.dev`).not.toBe("deprecated");
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
@@ -107,39 +126,69 @@ describe("openRouterResolve OpenRouter API validity", async () => {
|
||||
}
|
||||
});
|
||||
|
||||
describe("latest model per provider snapshot", async () => {
|
||||
const data = await api;
|
||||
const providerKeys = Object.keys(providers) as ModelProvider[];
|
||||
// ── OpenCode Zen served-list + free-cost checks ────────────────────────────────
|
||||
//
|
||||
// these enforce the two dynamic conditions for "this opencode alias works for a
|
||||
// user without OPENCODE_API_KEY" — the gap that let issue #691 ship:
|
||||
// 1. the alias's terminal-fallback resolve appears in Zen's /v1/models (Zen
|
||||
// actually serves it). caught nothing in #691 because mimo had a fallback
|
||||
// to big-pickle which IS served, but would catch any future alias that
|
||||
// points at a Zen-removed model without a fallback.
|
||||
// 2. for isFree aliases, the terminal-fallback's models.dev `cost.input` is
|
||||
// zero. caught the gpt-5-nano regression: $0.05/M input on models.dev,
|
||||
// marked isFree in our catalog.
|
||||
//
|
||||
// we check the terminal-fallback (via resolveDisplayAlias) because deprecated
|
||||
// aliases legitimately point at dead resolve targets — the terminal is what
|
||||
// actually runs at the agent CLI.
|
||||
|
||||
const latestByProvider: Record<string, { modelId: string; releaseDate: string }> = {};
|
||||
type ZenModel = { id: string };
|
||||
type ZenModelsResponse = { data: ZenModel[] };
|
||||
|
||||
for (const key of providerKeys) {
|
||||
const providerData = data[key];
|
||||
if (!providerData) continue;
|
||||
const zenApi = fetch("https://opencode.ai/zen/v1/models").then(
|
||||
(r) => r.json() as Promise<ZenModelsResponse>
|
||||
);
|
||||
|
||||
let latest: { modelId: string; releaseDate: string } | undefined;
|
||||
for (const [modelId, model] of Object.entries(providerData.models)) {
|
||||
// skip non-GA models so beta/nightly churn doesn't break the snapshot
|
||||
if (model.status) continue;
|
||||
const rd = model.release_date;
|
||||
if (!rd) continue;
|
||||
// tiebreak by modelId for stable ordering when release dates match
|
||||
if (
|
||||
!latest ||
|
||||
rd > latest.releaseDate ||
|
||||
(rd === latest.releaseDate && modelId > latest.modelId)
|
||||
) {
|
||||
latest = { modelId, releaseDate: rd };
|
||||
}
|
||||
}
|
||||
if (latest) {
|
||||
latestByProvider[key] = latest;
|
||||
}
|
||||
describe("opencode Zen served list", async () => {
|
||||
const zenData = await zenApi;
|
||||
const zenIds = new Set(zenData.data.map((m) => m.id));
|
||||
const seen = new Set<string>();
|
||||
|
||||
for (const alias of modelAliases) {
|
||||
const terminal = resolveDisplayAlias(alias.slug);
|
||||
if (!terminal) continue;
|
||||
const parsed = parseResolve(terminal.resolve);
|
||||
if (parsed.provider !== "opencode") continue;
|
||||
if (seen.has(terminal.resolve)) continue;
|
||||
seen.add(terminal.resolve);
|
||||
|
||||
it(`${alias.slug} terminal resolve ${terminal.resolve} is served by Zen`, () => {
|
||||
expect(
|
||||
zenIds.has(parsed.modelId),
|
||||
`terminal resolve "${terminal.resolve}" for alias "${alias.slug}" is not in https://opencode.ai/zen/v1/models — Zen no longer serves it. either point a fallback at a Zen-served alias or remove the entry.`
|
||||
).toBe(true);
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
describe("isFree models.dev cost", async () => {
|
||||
const data = await api;
|
||||
const seen = new Set<string>();
|
||||
|
||||
for (const alias of modelAliases.filter((a) => a.isFree)) {
|
||||
const terminal = resolveDisplayAlias(alias.slug);
|
||||
if (!terminal) continue;
|
||||
const parsed = parseResolve(terminal.resolve);
|
||||
if (seen.has(terminal.resolve)) continue;
|
||||
seen.add(terminal.resolve);
|
||||
|
||||
it(`${alias.slug} terminal resolve ${terminal.resolve} has cost.input === 0`, () => {
|
||||
const model = data[parsed.provider]?.models[parsed.modelId];
|
||||
expect(model, `terminal resolve "${terminal.resolve}" missing on models.dev`).toBeDefined();
|
||||
expect(
|
||||
model?.cost?.input,
|
||||
`isFree alias "${alias.slug}" walks to "${terminal.resolve}" which reports cost.input=${model?.cost?.input} on models.dev — either repoint the fallback or drop \`isFree\``
|
||||
).toBe(0);
|
||||
});
|
||||
}
|
||||
|
||||
// when this fails, a provider shipped a new model. check whether we need
|
||||
// to add or update an alias in models.ts before updating the snapshot.
|
||||
it("matches snapshot", () => {
|
||||
expect(latestByProvider).toMatchSnapshot();
|
||||
});
|
||||
});
|
||||
|
||||
+60
-2
@@ -1,11 +1,12 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { modelAliases, resolveCliModel } from "../models.ts";
|
||||
import { getModelEnvVars, modelAliases, resolveCliModel, resolveDisplayAlias } from "../models.ts";
|
||||
|
||||
// ── pure alias-registry invariants ──────────────────────────────────────────────
|
||||
//
|
||||
// these tests validate our alias data structure without hitting external APIs.
|
||||
// network-dependent checks (models.dev / OpenRouter catalog drift, latest-model
|
||||
// snapshot) live in models-catalog.main.test.ts and run only on main.
|
||||
// snapshot) live in models-catalog.main.test.ts and run on main pushes plus
|
||||
// `pullfrog/models-bump` PRs (the bot's bump branch, gated in test.yml).
|
||||
|
||||
// models that have no OpenRouter equivalent and require BYOK.
|
||||
// add a model here ONLY when it genuinely doesn't exist on both models.dev and OpenRouter.
|
||||
@@ -14,6 +15,10 @@ const BYOK_ONLY_MODELS = new Set(["openai/o3"]);
|
||||
describe("openRouterResolve completeness", () => {
|
||||
for (const alias of modelAliases) {
|
||||
if (alias.isFree) continue;
|
||||
// routing slugs (e.g. bedrock/byok) are inherently BYOK — there's no
|
||||
// single model to map to OpenRouter because the actual model ID is read
|
||||
// from a per-run env var.
|
||||
if (alias.routing) continue;
|
||||
if (BYOK_ONLY_MODELS.has(alias.slug)) continue;
|
||||
it(`${alias.slug} has openRouterResolve`, () => {
|
||||
expect(
|
||||
@@ -29,6 +34,13 @@ describe("openRouterResolve completeness", () => {
|
||||
expect(alias.openRouterResolve).toBeUndefined();
|
||||
});
|
||||
}
|
||||
|
||||
for (const alias of modelAliases) {
|
||||
if (!alias.routing) continue;
|
||||
it(`${alias.slug} (routing slug) has no openRouterResolve`, () => {
|
||||
expect(alias.openRouterResolve).toBeUndefined();
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
describe("fallback chain resolution", () => {
|
||||
@@ -42,3 +54,49 @@ describe("fallback chain resolution", () => {
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// ── isFree invariants — sanity-check the catalog data shape ─────────────────────
|
||||
//
|
||||
// these catch the latent regressions that produced issue #691:
|
||||
// - opencode/gpt-5-nano was marked `isFree` despite costing $0.05/M
|
||||
// (no static check existed; demoted to paid in the same PR adding these tests)
|
||||
// - opencode/mimo-v2-pro-free was free + fallback to big-pickle (correct shape),
|
||||
// but nothing enforced that the terminal of an isFree fallback chain is itself
|
||||
// free. if someone repointed big-pickle's fallback at a paid model, all of mimo
|
||||
// and big-pickle's users would silently start hitting a paid endpoint.
|
||||
//
|
||||
// the cost.input check itself is network-dependent (lives in
|
||||
// models-catalog.main.test.ts); these are the static sibling that runs on every PR.
|
||||
describe("isFree invariants", () => {
|
||||
for (const alias of modelAliases.filter((a) => a.isFree)) {
|
||||
it(`${alias.slug} lives under the opencode provider`, () => {
|
||||
expect(
|
||||
alias.provider,
|
||||
`isFree alias "${alias.slug}" must be under "opencode" (Zen's keyless gate is opencode-only)`
|
||||
).toBe("opencode");
|
||||
});
|
||||
|
||||
it(`${alias.slug} has empty envVars`, () => {
|
||||
expect(
|
||||
getModelEnvVars(alias.slug),
|
||||
`isFree alias "${alias.slug}" must declare \`envVars: []\` so validateAgentApiKey doesn't demand OPENCODE_API_KEY`
|
||||
).toEqual([]);
|
||||
});
|
||||
|
||||
it(`${alias.slug} has no openRouterResolve`, () => {
|
||||
expect(
|
||||
alias.openRouterResolve,
|
||||
`isFree alias "${alias.slug}" must omit \`openRouterResolve\` — free Zen models don't exist on OpenRouter`
|
||||
).toBeUndefined();
|
||||
});
|
||||
|
||||
it(`${alias.slug} fallback chain terminates at an isFree alias`, () => {
|
||||
const terminal = resolveDisplayAlias(alias.slug);
|
||||
expect(terminal, `fallback chain for "${alias.slug}" is broken`).toBeDefined();
|
||||
expect(
|
||||
terminal?.isFree,
|
||||
`isFree alias "${alias.slug}" walks to "${terminal?.slug}" which is NOT isFree — users would silently start paying`
|
||||
).toBe(true);
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
/**
|
||||
* provider catalog — the source of truth for `providers-live` (full harness
|
||||
* smoke per provider) and the per-provider coverage globs that scope `models-live`
|
||||
* (per-alias CLI smoke).
|
||||
*
|
||||
* each entry pins one standard-tier flagship slug per provider — not the
|
||||
* pro/opus tier (too expensive for per-push) and not the free/experimental
|
||||
* tier (too flaky). these flagships catch provider-class regressions like
|
||||
* Gemini schema sanitization or OpenAI tool-call format drift that the cheap
|
||||
* per-alias CLI smoke can't see.
|
||||
*
|
||||
* `coverage` lists the source files that, when changed, should rerun this
|
||||
* provider's flagship + every alias of this provider. `action/models.ts` is
|
||||
* included on every entry — touching the resolution table reruns all model
|
||||
* tests (simple model; matches the per-PR-precision answer from planning).
|
||||
*
|
||||
* adding a new provider:
|
||||
* 1. add an entry here with the flagship slug, agent harness, coverage globs
|
||||
* 2. add a row to wiki/models-catalog.md "To add a provider"
|
||||
* 3. CI picks it up automatically — no workflow change
|
||||
*/
|
||||
|
||||
export type ProviderEntry = {
|
||||
name: string;
|
||||
/** flagship slug for `providers-live` full-harness smoke. */
|
||||
flagship: string;
|
||||
/** harness used by the runtime for this provider's models. */
|
||||
agent: "claude" | "opencode";
|
||||
/** repo-relative globs that invalidate this provider's matrix entries. */
|
||||
coverage: string[];
|
||||
};
|
||||
|
||||
const SHARED_OPENCODE_COVERAGE = [
|
||||
"action/models.ts",
|
||||
"action/agents/opencode.ts",
|
||||
"action/agents/opencode_v2.ts",
|
||||
"action/agents/opencodePlugin.ts",
|
||||
];
|
||||
|
||||
export const providers: ProviderEntry[] = [
|
||||
{
|
||||
name: "anthropic",
|
||||
flagship: "anthropic/claude-sonnet",
|
||||
agent: "claude",
|
||||
coverage: ["action/models.ts", "action/agents/claude.ts"],
|
||||
},
|
||||
{
|
||||
name: "openai",
|
||||
flagship: "openai/gpt",
|
||||
agent: "opencode",
|
||||
coverage: SHARED_OPENCODE_COVERAGE,
|
||||
},
|
||||
{
|
||||
name: "google",
|
||||
flagship: "google/gemini-pro",
|
||||
agent: "opencode",
|
||||
coverage: [...SHARED_OPENCODE_COVERAGE, "action/mcp/geminiSanitizer.ts"],
|
||||
},
|
||||
{
|
||||
name: "xai",
|
||||
flagship: "xai/grok",
|
||||
agent: "opencode",
|
||||
coverage: SHARED_OPENCODE_COVERAGE,
|
||||
},
|
||||
{
|
||||
name: "deepseek",
|
||||
flagship: "deepseek/deepseek-pro",
|
||||
agent: "opencode",
|
||||
coverage: SHARED_OPENCODE_COVERAGE,
|
||||
},
|
||||
{
|
||||
name: "moonshotai",
|
||||
flagship: "moonshotai/kimi-k2",
|
||||
agent: "opencode",
|
||||
coverage: SHARED_OPENCODE_COVERAGE,
|
||||
},
|
||||
{
|
||||
name: "opencode",
|
||||
flagship: "opencode/big-pickle",
|
||||
agent: "opencode",
|
||||
coverage: SHARED_OPENCODE_COVERAGE,
|
||||
},
|
||||
{
|
||||
name: "openrouter",
|
||||
flagship: "openrouter/claude-sonnet",
|
||||
agent: "opencode",
|
||||
coverage: SHARED_OPENCODE_COVERAGE,
|
||||
},
|
||||
];
|
||||
+34
-41
@@ -2,9 +2,6 @@ import { existsSync, readdirSync } from "node:fs";
|
||||
import { dirname, join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { config } from "dotenv";
|
||||
import { runInDocker } from "../utils/docker.ts";
|
||||
import { ensureGitHubToken } from "../utils/github.ts";
|
||||
import { isInsideDocker } from "../utils/globals.ts";
|
||||
import { killTrackedChildren, setSignalHandler } from "../utils/subprocess.ts";
|
||||
import {
|
||||
type AgentResult,
|
||||
@@ -22,32 +19,33 @@ import {
|
||||
/**
|
||||
* unified test runner for all agent tests.
|
||||
*
|
||||
* usage: node test/run.ts [filters...]
|
||||
* invoke from the repo root:
|
||||
* pnpm runtest [filters…] # host, in-process — fast iteration (default)
|
||||
* pnpm runtest:docker [filters…] # local docker container that mocks GHA
|
||||
* pnpm docker test/run.ts [filters…] # explicit container form (equivalent to `pnpm runtest:docker`)
|
||||
*
|
||||
* filters can be test names, tags, or agent names:
|
||||
* node test/run.ts # run all tests (excludes adhoc-tagged tests)
|
||||
* node test/run.ts smoke # run tests named "smoke" or tagged "smoke"
|
||||
* node test/run.ts opencode # run all tests for opencode only
|
||||
* node test/run.ts security # run all tests tagged "security"
|
||||
* node test/run.ts agnostic # run all agnostic-tagged tests (with opencode)
|
||||
* node test/run.ts adhoc # run all adhoc-tagged tests
|
||||
* node test/run.ts smoke opencode # run smoke tests for opencode only
|
||||
* pnpm runtest # run all tests (excludes adhoc-tagged tests)
|
||||
* pnpm runtest smoke # run tests named "smoke" or tagged "smoke"
|
||||
* pnpm runtest opencode # run all tests for opencode only
|
||||
* pnpm runtest security # run all tests tagged "security"
|
||||
* pnpm runtest agnostic # run all agnostic-tagged tests (with opencode)
|
||||
* pnpm runtest adhoc # run all adhoc-tagged tests
|
||||
* pnpm runtest smoke opencode # run smoke tests for opencode only
|
||||
*
|
||||
* special tags:
|
||||
* - "agnostic": runs with opencode only, excluded when filtering by agent
|
||||
* - "adhoc": excluded from default runs, must be explicitly requested
|
||||
*
|
||||
* by default, runs in a Docker container for isolation.
|
||||
* see wiki/docker.md for when host vs container matters.
|
||||
*/
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
export const actionDir = join(__dirname, "..");
|
||||
|
||||
// load .env files
|
||||
config({ path: join(actionDir, ".env") });
|
||||
config({ path: join(actionDir, "..", ".env") });
|
||||
|
||||
const nodeModulesVolume = "pullfrog-action-test-node-modules";
|
||||
const mcpPortBase = 49000;
|
||||
let nextMcpPort = mcpPortBase;
|
||||
|
||||
@@ -57,25 +55,6 @@ function allocateMcpPort(): number {
|
||||
return port;
|
||||
}
|
||||
|
||||
function buildNodeCmd(args: string[]): string {
|
||||
const passArgs = args.map((arg) => `'${arg.replace(/'/g, "'\\''")}'`).join(" ");
|
||||
return `node test/run.ts ${passArgs}`;
|
||||
}
|
||||
|
||||
// run the test runner inside docker
|
||||
function runTestsInDocker(args: string[]): never {
|
||||
const result = runInDocker({
|
||||
actionDir,
|
||||
args,
|
||||
nodeCmd: buildNodeCmd(args),
|
||||
volumeName: nodeModulesVolume,
|
||||
envFilterMode: "allowlist",
|
||||
onStart: () => console.log("» running tests in docker container...\n"),
|
||||
});
|
||||
|
||||
process.exit(result.status ?? 1);
|
||||
}
|
||||
|
||||
type TestInfo = {
|
||||
name: string;
|
||||
config: TestRunnerOptions;
|
||||
@@ -282,6 +261,28 @@ function shouldRetry(result: AgentResult, validation: ValidationResult): RetryDe
|
||||
|
||||
async function runTestForAgent(ctx: RunContext): Promise<ValidationResult> {
|
||||
const testConfig = ctx.testInfo.config;
|
||||
|
||||
// runtime-evaluated skip: gate on env (e.g. CODEX_AUTH_JSON for codex-auth).
|
||||
// skipped runs short-circuit before any agent spawn AND count as passing so
|
||||
// a missing optional secret doesn't fail-fast cancel the rest of the matrix.
|
||||
const skipReason = testConfig.skipIf?.();
|
||||
if (skipReason) {
|
||||
const prefix = getPrefix({ test: ctx.testInfo.name, agent: ctx.agent });
|
||||
console.log(`${prefix} ⏭ skipped: ${skipReason}`);
|
||||
const skipped: ValidationResult = {
|
||||
test: ctx.testInfo.name,
|
||||
agent: ctx.agent,
|
||||
passed: true,
|
||||
canceled: false,
|
||||
checks: [],
|
||||
output: `skipped: ${skipReason}`,
|
||||
skipped: true,
|
||||
skipReason,
|
||||
};
|
||||
ctx.results.set(getRunKey(ctx.testInfo.name, ctx.agent), skipped);
|
||||
return skipped;
|
||||
}
|
||||
|
||||
const env: Record<string, string> = {};
|
||||
if (testConfig.env) {
|
||||
const entries = Object.entries(testConfig.env);
|
||||
@@ -393,14 +394,6 @@ async function runTestForAgent(ctx: RunContext): Promise<ValidationResult> {
|
||||
async function main(): Promise<void> {
|
||||
const args = process.argv.slice(2);
|
||||
|
||||
// run in Docker unless already inside
|
||||
if (!isInsideDocker) {
|
||||
// acquire token for docker if needed
|
||||
await ensureGitHubToken();
|
||||
runTestsInDocker(args);
|
||||
}
|
||||
|
||||
// load all tests
|
||||
const allTests = await loadAllTests();
|
||||
const parsed = parseArgs(args, allTests);
|
||||
|
||||
|
||||
@@ -28,6 +28,14 @@ const results: { model: string; status: "pass" | "fail" | "skip"; detail?: strin
|
||||
const seen = new Set<string>();
|
||||
|
||||
for (const alias of modelAliases) {
|
||||
// routing slugs (bedrock/byok) have no fixed `resolve` to test against —
|
||||
// the model ID is supplied at run time via a per-run env var. skipping
|
||||
// here matches the bumps cron + catalog drift test.
|
||||
if (alias.routing) {
|
||||
results.push({ model: alias.slug, status: "skip", detail: "routing slug (no fixed resolve)" });
|
||||
continue;
|
||||
}
|
||||
|
||||
if (seen.has(alias.resolve)) {
|
||||
results.push({ model: alias.resolve, status: "skip", detail: "duplicate resolve" });
|
||||
continue;
|
||||
|
||||
+27
-4
@@ -152,6 +152,8 @@ export interface ValidationResult {
|
||||
canceled: boolean;
|
||||
checks: ValidationCheck[];
|
||||
output: string;
|
||||
skipped?: boolean;
|
||||
skipReason?: string;
|
||||
}
|
||||
|
||||
export type ValidatorFn = (result: AgentResult) => ValidationCheck[];
|
||||
@@ -328,6 +330,16 @@ export interface TestRunnerOptions {
|
||||
// - "agnostic": runs with opencode only, excluded when filtering by agent
|
||||
// - "adhoc": excluded from default runs, must be explicitly requested
|
||||
tags?: TestTag[];
|
||||
// repo-relative globs of source files that, when changed in a PR, should
|
||||
// trigger this test in CI. omit to opt out of filtering (test always runs
|
||||
// — the defensive default). see action/test/coverage.ts.
|
||||
coverage?: string[];
|
||||
/** evaluated at test-runtime (after `pnpm install`, before agent spawn).
|
||||
* return a non-empty reason string to skip the test entirely — the runner
|
||||
* records a passing-with-skipped result so the matrix doesn't fail-fast
|
||||
* cancel the rest of the jobs. used to gate tests on optional secrets
|
||||
* (e.g. codex-auth needs `CODEX_AUTH_JSON`, which forks won't have). */
|
||||
skipIf?: () => string | null;
|
||||
}
|
||||
|
||||
export type TestTag = "adhoc" | "agnostic" | "security";
|
||||
@@ -336,8 +348,9 @@ export function printSingleValidation(validation: ValidationResult): void {
|
||||
const checksStr = validation.checks.map((c) => `${c.name}=${c.passed ? "✓" : "✗"}`).join(" ");
|
||||
const color = AGENT_COLORS[validation.agent] ?? "";
|
||||
const canceledNote = validation.canceled ? " (canceled)" : "";
|
||||
const skippedNote = validation.skipped ? ` (skipped: ${validation.skipReason ?? ""})` : "";
|
||||
console.log(
|
||||
`\n${color}[${validation.test}][${validation.agent}]${RESET} ${checksStr}${canceledNote}`
|
||||
`\n${color}[${validation.test}][${validation.agent}]${RESET} ${checksStr}${canceledNote}${skippedNote}`
|
||||
);
|
||||
}
|
||||
|
||||
@@ -349,8 +362,16 @@ export function printResults(validations: ValidationResult[]): void {
|
||||
|
||||
for (const v of validations) {
|
||||
const color = AGENT_COLORS[v.agent] ?? "";
|
||||
const status = v.canceled ? "❌ canceled" : v.passed ? "✅ pass" : "❌ fail";
|
||||
const checkCols = v.checks.map((c) => `${c.name}=${c.passed ? "✓" : "✗"}`).join(" ");
|
||||
const status = v.canceled
|
||||
? "❌ canceled"
|
||||
: v.skipped
|
||||
? "⏭ skipped"
|
||||
: v.passed
|
||||
? "✅ pass"
|
||||
: "❌ fail";
|
||||
const checkCols = v.skipped
|
||||
? `(skipped: ${v.skipReason ?? ""})`
|
||||
: v.checks.map((c) => `${c.name}=${c.passed ? "✓" : "✗"}`).join(" ");
|
||||
console.log(
|
||||
`${status} ${v.test.padEnd(12)} ${color}${v.agent.padEnd(10)}${RESET} ${checkCols}`
|
||||
);
|
||||
@@ -358,5 +379,7 @@ export function printResults(validations: ValidationResult[]): void {
|
||||
console.log("-".repeat(70));
|
||||
|
||||
const passed = validations.filter((v) => v.passed);
|
||||
console.log(`\n${passed.length}/${validations.length} passed`);
|
||||
const skipped = validations.filter((v) => v.skipped).length;
|
||||
const skippedNote = skipped > 0 ? ` (${skipped} skipped)` : "";
|
||||
console.log(`\n${passed.length}/${validations.length} passed${skippedNote}`);
|
||||
}
|
||||
|
||||
+197
@@ -0,0 +1,197 @@
|
||||
import type { AgentUsage } from "./agents/shared.ts";
|
||||
import type { PrepResult } from "./prep/types.ts";
|
||||
import type { AgentDiagnostic } from "./utils/agentHangReport.ts";
|
||||
import { log } from "./utils/cli.ts";
|
||||
import type { DiffCoverageState } from "./utils/diffCoverage.ts";
|
||||
import {
|
||||
type ProgressComment,
|
||||
type ProgressCommentType,
|
||||
parseProgressComment,
|
||||
} from "./utils/progressComment.ts";
|
||||
import type { TodoTracker } from "./utils/todoTracking.ts";
|
||||
|
||||
export type BackgroundProcess = {
|
||||
pid: number;
|
||||
outputPath: string;
|
||||
pidPath: string;
|
||||
};
|
||||
|
||||
export type BrowserDaemon = { binDir: string; error?: never } | { binDir?: never; error: string };
|
||||
|
||||
export type StoredPushDest = {
|
||||
remoteName: string;
|
||||
remoteBranch: string;
|
||||
localBranch: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* Valid inline-comment anchor lines per side at a particular checkout SHA.
|
||||
* Lives here (not in `mcp/review.ts`) so `ToolState` — which caches
|
||||
* `Map<path, CommentableLines>` per checkout — does not pull the MCP server
|
||||
* graph into every consumer of run state (the action's main loop, agent
|
||||
* harnesses, cf-worker indexing).
|
||||
*/
|
||||
export type CommentableLines = { RIGHT: Set<number>; LEFT: Set<number> };
|
||||
|
||||
/**
|
||||
* mutable per-run record of facts that occurred during execution. shared
|
||||
* between the action process and the MCP server (one process — toolState is
|
||||
* just a JS object passed by reference into both surfaces).
|
||||
*
|
||||
* design rule: ToolState is LITERAL. each field records a thing that
|
||||
* happened — `review` is set when `create_pull_request_review` succeeded,
|
||||
* `finalSummaryWritten` flips when `report_progress` wrote a non-plan body,
|
||||
* `selectedMode` is set when `select_mode` was called. fields should never
|
||||
* encode the absence of an event ("unsubmittedReview", "missingArtifact"),
|
||||
* speculative state, or values derived from other fields.
|
||||
*
|
||||
* any predicate the rest of the code needs ("the agent picked review mode but
|
||||
* never produced a review or progress write") is computed inline at the call
|
||||
* site, not stored. derived state in this struct invariably drifts from the
|
||||
* literal fields under refactors and is the wrong layer for the check.
|
||||
*
|
||||
* write narrowly: prefer adding state inside the tool that mutates it (e.g.
|
||||
* `create_pull_request_review` populates `toolState.review`) and reading
|
||||
* narrowly elsewhere. don't introduce flags from main.ts that mirror what an
|
||||
* MCP tool already records.
|
||||
*/
|
||||
export interface ToolState {
|
||||
// where we're allowed to push - base repo initially, fork URL for fork PRs
|
||||
// set by setupGit, updated by checkout_pr. always set before push validation.
|
||||
pushUrl?: string;
|
||||
// push destination set by checkout_pr - used as primary source in push_branch
|
||||
// because git config reads can fail in certain environments
|
||||
pushDest?: StoredPushDest;
|
||||
// issue or PR number (same number space in GitHub)
|
||||
issueNumber?: number;
|
||||
// PR HEAD sha at checkout time — used to detect new commits pushed during a review
|
||||
checkoutSha?: string;
|
||||
// commentable lines per file at checkoutSha — captured during checkout_pr so
|
||||
// review-time inline-comment validation matches the diff GitHub will anchor
|
||||
// to (commit_id=checkoutSha). without this, a PR update between checkout and
|
||||
// review would make listFiles (latest HEAD) disagree with the anchor,
|
||||
// silently dropping valid comments or letting invalid ones through.
|
||||
//
|
||||
// commentableLinesPullNumber records WHICH PR this snapshot belongs to. if
|
||||
// the agent checks out PR B and then reviews PR A in the same session, the
|
||||
// cached snapshot for B would silently mis-validate A's comments — keying
|
||||
// by PR number forces a re-fetch when the target changes.
|
||||
//
|
||||
// commentableLinesCheckoutSha pins the snapshot to the SHA it was built
|
||||
// against. if a second checkout_pr for the SAME PR bumps checkoutSha but
|
||||
// fails before repopulating the cache (e.g., listFiles rate-limits), the
|
||||
// stale snapshot would silently mis-validate comments against the new SHA.
|
||||
// comparing both fields forces a re-fetch when either moves.
|
||||
commentableLinesByFile?: Map<string, CommentableLines>;
|
||||
commentableLinesPullNumber?: number;
|
||||
commentableLinesCheckoutSha?: string | undefined;
|
||||
// SHA to diff incrementally against — set from event payload on first checkout,
|
||||
// then from checkoutSha when review.ts detects new commits mid-review
|
||||
beforeSha?: string;
|
||||
selectedMode?: string;
|
||||
// number of prepush hook failures this run. push_branch runs the hook
|
||||
// while this is 0 and skips it once non-zero; never decremented within
|
||||
// a run.
|
||||
prepushFailureCount: number;
|
||||
backgroundProcesses: Map<string, BackgroundProcess>;
|
||||
browserDaemon?: BrowserDaemon | undefined;
|
||||
review?: {
|
||||
id: number;
|
||||
nodeId: string;
|
||||
reviewedSha: string | undefined;
|
||||
};
|
||||
// dedupe key: parent review comment_id → most-recent reply written this
|
||||
// session by reply_to_review_comment. used by duplicateReplyDecision to
|
||||
// skip identical-body re-emissions of the same call (PR #610 root cause).
|
||||
// body-keyed (not just id-keyed) so legitimate follow-up replies with
|
||||
// different content still go through.
|
||||
reviewReplies?: Map<
|
||||
number,
|
||||
{ commentId: number; url: string | undefined; bodyWithFooter: string }
|
||||
>;
|
||||
dependencyInstallation?: {
|
||||
status: "not_started" | "in_progress" | "completed" | "failed";
|
||||
promise: Promise<PrepResult[]> | undefined;
|
||||
results: PrepResult[] | undefined;
|
||||
};
|
||||
// undefined = no comment yet, object = active comment, null = deliberately deleted
|
||||
progressComment: ProgressComment | null | undefined;
|
||||
// immutable snapshot: true if a progress comment was pre-created at init time.
|
||||
// survives deleteProgressComment so handleAgentResult can still detect "expected but never reported".
|
||||
hadProgressComment: boolean;
|
||||
lastProgressBody?: string;
|
||||
wasUpdated?: boolean;
|
||||
// set after a non-plan report_progress successfully writes the final summary.
|
||||
// decoupled from todoTracker.enabled so cleanup detection survives API failures.
|
||||
finalSummaryWritten?: boolean;
|
||||
// set by select_mode when Plan + issue_number and plan-comment API returns existing plan (for report_progress target_plan_comment)
|
||||
existingPlanCommentId?: number;
|
||||
previousPlanBody?: string;
|
||||
// absolute path to the PR summary markdown file the agent edits in place.
|
||||
// seeded by main.ts before the agent starts when payload.generateSummary is set;
|
||||
// read back at end-of-run to persist to DB.
|
||||
summaryFilePath?: string;
|
||||
// exact bytes of the seeded snapshot file at run start. compared against
|
||||
// the file content at end-of-run to detect "agent never touched it" — in
|
||||
// that case persistSummary skips the DB write (saving the seed verbatim
|
||||
// would either re-write what the DB already has, on incremental runs, or
|
||||
// serialize the placeholder scaffold, on first runs).
|
||||
summarySeed?: string;
|
||||
// set to true after persistSummary completes once. prevents the error-path
|
||||
// call (which exists so a successful agent edit before a crash still gets
|
||||
// persisted) from redundantly re-running the DB PATCH on the
|
||||
// success-then-late-throw path.
|
||||
summaryPersistAttempted?: boolean;
|
||||
// absolute path to the rolling repo-level learnings markdown file the
|
||||
// agent reads at startup and may edit at end-of-run. seeded by main.ts
|
||||
// for every run from `Repo.learnings` (empty file when no learnings
|
||||
// exist yet); read back at end-of-run to persist any edits.
|
||||
learningsFilePath?: string;
|
||||
// exact bytes of the seeded learnings file at run start. compared
|
||||
// against the file content at end-of-run to detect "agent never touched
|
||||
// it" — in that case persistLearnings skips the DB PATCH (saving the
|
||||
// identical content would be a no-op write that wastes a LearningsRevision
|
||||
// row and the API round-trip).
|
||||
learningsSeed?: string;
|
||||
// mirror of `summaryPersistAttempted` for the learnings tmpfile — guards
|
||||
// the error-path / exit-signal callers from a redundant second PATCH
|
||||
// after the success path already persisted.
|
||||
learningsPersistAttempted?: boolean;
|
||||
output?: string;
|
||||
usageEntries: AgentUsage[];
|
||||
model?: string | undefined;
|
||||
// set by main.ts when the BYOK fallback engaged (configured model needed
|
||||
// a provider key the runner didn't have). carried into PR-comment footers
|
||||
// so users can see "Using <free model> (credentials for <configured> not
|
||||
// configured)" rather than just being silently downgraded. literal record
|
||||
// of an event that happened — matches the ToolState design rule.
|
||||
modelFallback?: { from: string } | undefined;
|
||||
todoTracker?: TodoTracker | undefined;
|
||||
diffCoverage?: DiffCoverageState | undefined;
|
||||
// mutable handle the agent harness writes to as a run progresses (recent
|
||||
// stderr ring buffer reference, last provider-error label, event count).
|
||||
// read by main.ts's outer catch so a watchdog-fired activity timeout still
|
||||
// surfaces the same agent-side context the harness's own catch path returns
|
||||
// via `result.error`. see `utils/agentHangReport.ts`.
|
||||
agentDiagnostic?: AgentDiagnostic | undefined;
|
||||
}
|
||||
|
||||
interface InitToolStateParams {
|
||||
progressComment: { id: string; type: ProgressCommentType } | undefined;
|
||||
}
|
||||
|
||||
export function initToolState(params: InitToolStateParams): ToolState {
|
||||
const resolved = parseProgressComment(params.progressComment);
|
||||
|
||||
if (resolved) {
|
||||
log.info(`» using pre-created progress comment: ${resolved.id} (${resolved.type})`);
|
||||
}
|
||||
|
||||
return {
|
||||
progressComment: resolved,
|
||||
hadProgressComment: !!resolved,
|
||||
prepushFailureCount: 0,
|
||||
backgroundProcesses: new Map(),
|
||||
usageEntries: [],
|
||||
};
|
||||
}
|
||||
+3
-2
@@ -19,7 +19,8 @@
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"stripInternal": true,
|
||||
"moduleDetection": "force",
|
||||
"useUnknownInCatchVariables": true
|
||||
"useUnknownInCatchVariables": true,
|
||||
"noEmit": true
|
||||
},
|
||||
"exclude": []
|
||||
"exclude": ["dist"]
|
||||
}
|
||||
|
||||
+58
-1
@@ -1,4 +1,5 @@
|
||||
import { performance } from "node:perf_hooks";
|
||||
import { log } from "./log.ts";
|
||||
|
||||
function isMonitorDebugEnabled(): boolean {
|
||||
return (
|
||||
@@ -79,6 +80,19 @@ type WriteFunction = {
|
||||
// module-level activity tracking - allows agents to mark activity on any event
|
||||
let _lastActivity = performance.now();
|
||||
|
||||
/**
|
||||
* upper bound on how long a single tool call can suspend the activity
|
||||
* watchdog. matched against the typical worst-case `checkout_pr`
|
||||
* fetch+deepen on a large monorepo (issue #760: 4-5min) plus generous
|
||||
* headroom for slower MCP tools, while still bounding the worst case if
|
||||
* a tool genuinely hangs and `tool_result` never arrives — auto-resume
|
||||
* fires here and the normal idle clock takes over from a fresh baseline.
|
||||
*/
|
||||
export const MAX_TOOL_CALL_SUSPENSION_MS = 15 * 60 * 1000;
|
||||
|
||||
let _suspendedAt: number | null = null;
|
||||
let _suspensionTimer: NodeJS.Timeout | null = null;
|
||||
|
||||
/**
|
||||
* mark activity to reset the no-output timeout.
|
||||
* call this whenever the agent emits any event, even if it isn't logged to stdout.
|
||||
@@ -88,12 +102,55 @@ export function markActivity(): void {
|
||||
}
|
||||
|
||||
/**
|
||||
* get the time since last activity in milliseconds
|
||||
* get the time since last activity in milliseconds.
|
||||
* returns 0 while the watchdog is suspended (issue #760).
|
||||
*/
|
||||
export function getIdleMs(): number {
|
||||
if (_suspendedAt !== null) return 0;
|
||||
return Math.round(performance.now() - _lastActivity);
|
||||
}
|
||||
|
||||
/**
|
||||
* suspend the activity watchdog while a long-running, in-flight unit of
|
||||
* work is happening (e.g. an MCP `tools/call` that synchronously awaits
|
||||
* a multi-minute git fetch). bracket calls with `resumeActivity()` from
|
||||
* the agent harness's `tool_use` / `tool_result` event handlers.
|
||||
*
|
||||
* - idempotent: nested suspends are no-ops; the first resume wins.
|
||||
* - bounded: auto-resumes after `maxMs` so a buggy tool that never
|
||||
* produces a `tool_result` can't pin the watchdog open forever.
|
||||
* - safe: only the *agent harness* (claude.ts / opencode.ts) on explicit,
|
||||
* paired CLI events should call this. NEVER blanket-suspend on internal
|
||||
* noise — that would resurrect issue #12 zombie runs.
|
||||
*/
|
||||
export function suspendActivity(maxMs: number = MAX_TOOL_CALL_SUSPENSION_MS): void {
|
||||
if (_suspendedAt !== null) return;
|
||||
_suspendedAt = performance.now();
|
||||
_suspensionTimer = setTimeout(() => {
|
||||
log.warning(`activity watchdog suspended >${Math.round(maxMs / 1000)}s — auto-resuming`);
|
||||
resumeActivity();
|
||||
}, maxMs);
|
||||
_suspensionTimer.unref?.();
|
||||
}
|
||||
|
||||
/**
|
||||
* resume the activity watchdog. resets the idle baseline so a stale
|
||||
* idle window before the suspend can't immediately re-fire.
|
||||
*/
|
||||
export function resumeActivity(): void {
|
||||
if (_suspendedAt === null) return;
|
||||
_suspendedAt = null;
|
||||
if (_suspensionTimer) {
|
||||
clearTimeout(_suspensionTimer);
|
||||
_suspensionTimer = null;
|
||||
}
|
||||
_lastActivity = performance.now();
|
||||
}
|
||||
|
||||
export function isActivitySuspended(): boolean {
|
||||
return _suspendedAt !== null;
|
||||
}
|
||||
|
||||
function wrapWrite(original: WriteFunction, onActivity: () => void): WriteFunction {
|
||||
const wrapped: WriteFunction = (
|
||||
chunk: string | Uint8Array,
|
||||
|
||||
+194
-5
@@ -1,9 +1,198 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { resolveAgent } from "./agent.ts";
|
||||
import { mkdtempSync, readFileSync, rmSync, statSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
||||
import { resolveAgent, resolveModel } from "./agent.ts";
|
||||
import { cleanupVertexCredentials, materializeVertexCredentials } from "./vertex.ts";
|
||||
|
||||
const savedEnv = { ...process.env };
|
||||
|
||||
const STRIPPED = [
|
||||
/_API_KEY$/,
|
||||
/^CLAUDE_CODE_OAUTH_TOKEN$/,
|
||||
/^AWS_BEARER_TOKEN_BEDROCK$/,
|
||||
/^AWS_ACCESS_KEY_ID$/,
|
||||
/^AWS_SECRET_ACCESS_KEY$/,
|
||||
/^AWS_SESSION_TOKEN$/,
|
||||
/^AWS_REGION$/,
|
||||
/^BEDROCK_MODEL_ID$/,
|
||||
/^GOOGLE_APPLICATION_CREDENTIALS$/,
|
||||
/^GOOGLE_CLOUD_PROJECT$/,
|
||||
/^VERTEX_SERVICE_ACCOUNT_JSON$/,
|
||||
/^VERTEX_LOCATION$/,
|
||||
/^VERTEX_MODEL_ID$/,
|
||||
/^PULLFROG_SECRET_HOME$/,
|
||||
/^PULLFROG_MODEL$/,
|
||||
/^PULLFROG_AGENT$/,
|
||||
];
|
||||
|
||||
beforeEach(() => {
|
||||
for (const key of Object.keys(process.env)) {
|
||||
if (STRIPPED.some((re) => re.test(key))) delete process.env[key];
|
||||
}
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
process.env = { ...savedEnv };
|
||||
});
|
||||
|
||||
describe("resolveAgent", () => {
|
||||
it("returns opencode", () => {
|
||||
const agent = resolveAgent({});
|
||||
expect(agent.name).toBe("opencode");
|
||||
it("returns opencode by default", () => {
|
||||
expect(resolveAgent({}).name).toBe("opencode");
|
||||
});
|
||||
|
||||
it("routes anthropic/* to claude when ANTHROPIC_API_KEY is set", () => {
|
||||
process.env.ANTHROPIC_API_KEY = "sk-test";
|
||||
expect(resolveAgent({ model: "anthropic/claude-opus-4-7" }).name).toBe("claude");
|
||||
});
|
||||
|
||||
it("falls back to opencode for anthropic/* without claude-code creds", () => {
|
||||
expect(resolveAgent({ model: "anthropic/claude-opus-4-7" }).name).toBe("opencode");
|
||||
});
|
||||
|
||||
describe("bedrock routing", () => {
|
||||
it("routes Anthropic Bedrock IDs to claude", () => {
|
||||
process.env.AWS_BEARER_TOKEN_BEDROCK = "bedrock-token";
|
||||
process.env.BEDROCK_MODEL_ID = "us.anthropic.claude-opus-4-7";
|
||||
expect(resolveAgent({ model: "us.anthropic.claude-opus-4-7" }).name).toBe("claude");
|
||||
});
|
||||
|
||||
it("routes Anthropic Bedrock IDs (no region prefix) to claude", () => {
|
||||
process.env.AWS_BEARER_TOKEN_BEDROCK = "bedrock-token";
|
||||
process.env.BEDROCK_MODEL_ID = "anthropic.claude-haiku-4-5-20251001-v1:0";
|
||||
expect(resolveAgent({ model: "anthropic.claude-haiku-4-5-20251001-v1:0" }).name).toBe(
|
||||
"claude"
|
||||
);
|
||||
});
|
||||
|
||||
it("routes non-Anthropic Bedrock IDs to opencode", () => {
|
||||
process.env.AWS_BEARER_TOKEN_BEDROCK = "bedrock-token";
|
||||
process.env.BEDROCK_MODEL_ID = "amazon.nova-pro-v1:0";
|
||||
expect(resolveAgent({ model: "amazon.nova-pro-v1:0" }).name).toBe("opencode");
|
||||
});
|
||||
|
||||
it("routes Llama IDs to opencode", () => {
|
||||
process.env.AWS_BEARER_TOKEN_BEDROCK = "bedrock-token";
|
||||
process.env.BEDROCK_MODEL_ID = "us.meta.llama4-scout-17b-instruct-v1:0";
|
||||
expect(resolveAgent({ model: "us.meta.llama4-scout-17b-instruct-v1:0" }).name).toBe(
|
||||
"opencode"
|
||||
);
|
||||
});
|
||||
|
||||
it("accepts AWS access keys as auth", () => {
|
||||
process.env.AWS_ACCESS_KEY_ID = "AKIA-test";
|
||||
process.env.AWS_SECRET_ACCESS_KEY = "secret-test";
|
||||
process.env.BEDROCK_MODEL_ID = "us.anthropic.claude-opus-4-7";
|
||||
expect(resolveAgent({ model: "us.anthropic.claude-opus-4-7" }).name).toBe("claude");
|
||||
});
|
||||
|
||||
it("PULLFROG_AGENT override wins over Anthropic auto-routing", () => {
|
||||
process.env.PULLFROG_AGENT = "opencode";
|
||||
process.env.AWS_BEARER_TOKEN_BEDROCK = "bedrock-token";
|
||||
process.env.BEDROCK_MODEL_ID = "us.anthropic.claude-opus-4-7";
|
||||
expect(resolveAgent({ model: "us.anthropic.claude-opus-4-7" }).name).toBe("opencode");
|
||||
});
|
||||
});
|
||||
|
||||
describe("vertex routing", () => {
|
||||
it("routes Anthropic Vertex IDs to claude", () => {
|
||||
process.env.VERTEX_SERVICE_ACCOUNT_JSON = "{}";
|
||||
process.env.VERTEX_MODEL_ID = "claude-opus-4-1@20250805";
|
||||
expect(resolveAgent({ model: "claude-opus-4-1@20250805" }).name).toBe("claude");
|
||||
});
|
||||
|
||||
it("routes Gemini Vertex IDs to opencode", () => {
|
||||
process.env.VERTEX_SERVICE_ACCOUNT_JSON = "{}";
|
||||
process.env.VERTEX_MODEL_ID = "gemini-2.5-pro";
|
||||
expect(resolveAgent({ model: "gemini-2.5-pro" }).name).toBe("opencode");
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("resolveModel", () => {
|
||||
it("PULLFROG_MODEL override wins", () => {
|
||||
process.env.PULLFROG_MODEL = "anthropic/claude-opus";
|
||||
expect(resolveModel({ slug: "openai/gpt" })).toBe("anthropic/claude-opus-4-7");
|
||||
});
|
||||
|
||||
it("PULLFROG_MODEL bypasses bedrock routing entirely", () => {
|
||||
process.env.PULLFROG_MODEL = "openai/gpt";
|
||||
process.env.BEDROCK_MODEL_ID = "us.anthropic.claude-opus-4-7";
|
||||
expect(resolveModel({ slug: "bedrock/byok" })).toBe("openai/gpt-5.5");
|
||||
});
|
||||
|
||||
it("resolves bedrock/byok to BEDROCK_MODEL_ID", () => {
|
||||
process.env.BEDROCK_MODEL_ID = "us.anthropic.claude-opus-4-7";
|
||||
expect(resolveModel({ slug: "bedrock/byok" })).toBe("us.anthropic.claude-opus-4-7");
|
||||
});
|
||||
|
||||
it("throws when bedrock/byok is selected without BEDROCK_MODEL_ID", () => {
|
||||
expect(() => resolveModel({ slug: "bedrock/byok" })).toThrow("BEDROCK_MODEL_ID");
|
||||
});
|
||||
|
||||
it("returns the alias resolve for normal slugs", () => {
|
||||
expect(resolveModel({ slug: "openai/gpt" })).toBe("openai/gpt-5.5");
|
||||
});
|
||||
|
||||
it("returns undefined for no slug + no PULLFROG_MODEL", () => {
|
||||
expect(resolveModel({})).toBeUndefined();
|
||||
});
|
||||
|
||||
// regression: PR #720 review caught that `resolveCliModel("bedrock/byok")`
|
||||
// returns the literal sentinel `"bedrock"` from the alias's `resolve`
|
||||
// field. Without routing-aware handling, PULLFROG_MODEL=bedrock/byok would
|
||||
// leak that sentinel downstream and break agent dispatch.
|
||||
it("PULLFROG_MODEL=bedrock/byok defers to BEDROCK_MODEL_ID, not the sentinel", () => {
|
||||
process.env.PULLFROG_MODEL = "bedrock/byok";
|
||||
process.env.BEDROCK_MODEL_ID = "us.anthropic.claude-opus-4-7";
|
||||
expect(resolveModel({ slug: "openai/gpt" })).toBe("us.anthropic.claude-opus-4-7");
|
||||
});
|
||||
|
||||
it("PULLFROG_MODEL=bedrock/byok throws if BEDROCK_MODEL_ID is missing", () => {
|
||||
process.env.PULLFROG_MODEL = "bedrock/byok";
|
||||
expect(() => resolveModel({ slug: "openai/gpt" })).toThrow("BEDROCK_MODEL_ID");
|
||||
});
|
||||
|
||||
it("resolves vertex/byok to VERTEX_MODEL_ID", () => {
|
||||
process.env.VERTEX_MODEL_ID = "claude-opus-4-1@20250805";
|
||||
expect(resolveModel({ slug: "vertex/byok" })).toBe("claude-opus-4-1@20250805");
|
||||
});
|
||||
|
||||
it("throws when vertex/byok is selected without VERTEX_MODEL_ID", () => {
|
||||
expect(() => resolveModel({ slug: "vertex/byok" })).toThrow("VERTEX_MODEL_ID");
|
||||
});
|
||||
|
||||
it("PULLFROG_MODEL=vertex/byok defers to VERTEX_MODEL_ID, not the sentinel", () => {
|
||||
process.env.PULLFROG_MODEL = "vertex/byok";
|
||||
process.env.VERTEX_MODEL_ID = "gemini-2.5-pro";
|
||||
expect(resolveModel({ slug: "openai/gpt" })).toBe("gemini-2.5-pro");
|
||||
});
|
||||
});
|
||||
|
||||
describe("materializeVertexCredentials", () => {
|
||||
it("writes service-account JSON outside tmpdir and defaults project from project_id", () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "vertex-creds-test-"));
|
||||
process.env.VERTEX_MODEL_ID = "claude-opus-4-1@20250805";
|
||||
process.env.PULLFROG_SECRET_HOME = dir;
|
||||
process.env.VERTEX_SERVICE_ACCOUNT_JSON = JSON.stringify({
|
||||
project_id: "test-project",
|
||||
client_email: "pullfrog@test-project.iam.gserviceaccount.com",
|
||||
});
|
||||
|
||||
try {
|
||||
const credentials = materializeVertexCredentials({ model: "claude-opus-4-1@20250805" });
|
||||
|
||||
if (!credentials) throw new Error("expected vertex credentials");
|
||||
expect(credentials.credentialsPath).toContain(join(dir, ".pullfrog", "secrets"));
|
||||
expect(process.env.GOOGLE_APPLICATION_CREDENTIALS).toBe(credentials.credentialsPath);
|
||||
expect(process.env.GOOGLE_CLOUD_PROJECT).toBe("test-project");
|
||||
expect(readFileSync(credentials.credentialsPath, "utf8")).toBe(
|
||||
process.env.VERTEX_SERVICE_ACCOUNT_JSON
|
||||
);
|
||||
expect(statSync(credentials.credentialsPath).mode & 0o777).toBe(0o600);
|
||||
cleanupVertexCredentials(credentials);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
+80
-7
@@ -1,7 +1,16 @@
|
||||
import type { Agent } from "../agents/index.ts";
|
||||
import { agents } from "../agents/index.ts";
|
||||
import { getModelProvider, resolveCliModel } from "../models.ts";
|
||||
import {
|
||||
BEDROCK_MODEL_ID_ENV,
|
||||
getModelProvider,
|
||||
isBedrockAnthropicId,
|
||||
isVertexAnthropicId,
|
||||
resolveCliModel,
|
||||
resolveDisplayAlias,
|
||||
VERTEX_MODEL_ID_ENV,
|
||||
} from "../models.ts";
|
||||
import { log } from "./cli.ts";
|
||||
import { VERTEX_SERVICE_ACCOUNT_JSON_ENV } from "./vertex.ts";
|
||||
|
||||
function hasEnvVar(name: string): boolean {
|
||||
const val = process.env[name];
|
||||
@@ -12,6 +21,52 @@ function hasClaudeCodeAuth(): boolean {
|
||||
return hasEnvVar("CLAUDE_CODE_OAUTH_TOKEN") || hasEnvVar("ANTHROPIC_API_KEY");
|
||||
}
|
||||
|
||||
function hasBedrockAuth(): boolean {
|
||||
return (
|
||||
hasEnvVar("AWS_BEARER_TOKEN_BEDROCK") ||
|
||||
(hasEnvVar("AWS_ACCESS_KEY_ID") && hasEnvVar("AWS_SECRET_ACCESS_KEY"))
|
||||
);
|
||||
}
|
||||
|
||||
function hasVertexAuth(): boolean {
|
||||
return hasEnvVar(VERTEX_SERVICE_ACCOUNT_JSON_ENV);
|
||||
}
|
||||
|
||||
/**
|
||||
* resolve a single slug to its CLI-ready model string. routing aliases
|
||||
* (e.g. `bedrock/byok`) defer to their backing env var instead of the
|
||||
* sentinel stored in `resolve`. shared between PULLFROG_MODEL override
|
||||
* and repo-config slug resolution so both paths get the same routing
|
||||
* semantics — without this helper, `PULLFROG_MODEL=bedrock/byok` would
|
||||
* leak the literal sentinel string `"bedrock"` downstream.
|
||||
*/
|
||||
function resolveSlug(slug: string): string | undefined {
|
||||
const alias = resolveDisplayAlias(slug);
|
||||
if (alias?.routing === "bedrock") {
|
||||
const bedrockId = process.env[BEDROCK_MODEL_ID_ENV]?.trim();
|
||||
if (!bedrockId) {
|
||||
throw new Error(
|
||||
`${BEDROCK_MODEL_ID_ENV} env var is required when the model is set to "${slug}". ` +
|
||||
`set it to an AWS Bedrock model ID from the Bedrock console. ` +
|
||||
`see https://docs.pullfrog.com/bedrock for setup.`
|
||||
);
|
||||
}
|
||||
return bedrockId;
|
||||
}
|
||||
if (alias?.routing === "vertex") {
|
||||
const vertexId = process.env[VERTEX_MODEL_ID_ENV]?.trim();
|
||||
if (!vertexId) {
|
||||
throw new Error(
|
||||
`${VERTEX_MODEL_ID_ENV} env var is required when the model is set to "${slug}". ` +
|
||||
`set it to a Google Vertex AI model ID from Model Garden. ` +
|
||||
`see https://docs.pullfrog.com/vertex for setup.`
|
||||
);
|
||||
}
|
||||
return vertexId;
|
||||
}
|
||||
return resolveCliModel(slug);
|
||||
}
|
||||
|
||||
/**
|
||||
* resolve the effective model for this run.
|
||||
*
|
||||
@@ -19,17 +74,20 @@ function hasClaudeCodeAuth(): boolean {
|
||||
* 1. PULLFROG_MODEL env var — resolved through the alias registry first,
|
||||
* so values like "anthropic/claude-opus" become "anthropic/claude-opus-4-7".
|
||||
* raw specifiers (e.g. "anthropic/claude-opus-4-6") pass through unchanged.
|
||||
* 2. slug from repo config / payload → alias registry
|
||||
* 3. undefined — agent will auto-select
|
||||
* always wins — bypasses Bedrock routing entirely. to test a different
|
||||
* Bedrock model, change `BEDROCK_MODEL_ID`, not `PULLFROG_MODEL`.
|
||||
* 2. slug from repo config / payload → alias registry. routing slugs
|
||||
* (e.g. `bedrock/byok`) defer to a separate env var (`BEDROCK_MODEL_ID`).
|
||||
* 3. undefined — agent will auto-select.
|
||||
*/
|
||||
export function resolveModel(ctx: { slug?: string | undefined }): string | undefined {
|
||||
const envModel = process.env.PULLFROG_MODEL?.trim();
|
||||
if (envModel) {
|
||||
return resolveCliModel(envModel) ?? envModel;
|
||||
return resolveSlug(envModel) ?? envModel;
|
||||
}
|
||||
|
||||
if (ctx.slug) {
|
||||
const resolved = resolveCliModel(ctx.slug);
|
||||
const resolved = resolveSlug(ctx.slug);
|
||||
if (resolved) {
|
||||
return resolved;
|
||||
}
|
||||
@@ -49,7 +107,22 @@ export function resolveAgent(ctx: { model?: string | undefined }): Agent {
|
||||
log.warning(`» unknown PULLFROG_AGENT="${envAgent}" — falling through to auto-select`);
|
||||
}
|
||||
|
||||
// 2. if model is Anthropic and Claude Code credentials are available, use Claude Code
|
||||
// 2. Bedrock routing: when BEDROCK_MODEL_ID is the resolved model, route
|
||||
// Anthropic IDs through claude-code (which supports Bedrock natively
|
||||
// once CLAUDE_CODE_USE_BEDROCK=1) and everything else through opencode's
|
||||
// `amazon-bedrock` provider.
|
||||
if (ctx.model && hasBedrockAuth() && process.env[BEDROCK_MODEL_ID_ENV]?.trim() === ctx.model) {
|
||||
return isBedrockAnthropicId(ctx.model) ? agents.claude : agents.opencode;
|
||||
}
|
||||
|
||||
// 3. Vertex routing: same shape as Bedrock, but Anthropic Vertex IDs are
|
||||
// anchored `claude-*` IDs and non-Anthropic models use opencode's
|
||||
// `google-vertex` provider.
|
||||
if (ctx.model && hasVertexAuth() && process.env[VERTEX_MODEL_ID_ENV]?.trim() === ctx.model) {
|
||||
return isVertexAnthropicId(ctx.model) ? agents.claude : agents.opencode;
|
||||
}
|
||||
|
||||
// 4. if model is Anthropic and Claude Code credentials are available, use Claude Code
|
||||
if (ctx.model) {
|
||||
try {
|
||||
const provider = getModelProvider(ctx.model);
|
||||
@@ -61,6 +134,6 @@ export function resolveAgent(ctx: { model?: string | undefined }): Agent {
|
||||
}
|
||||
}
|
||||
|
||||
// 3. default: OpenCode (universal, supports all providers)
|
||||
// 5. default: OpenCode (universal, supports all providers)
|
||||
return agents.opencode;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,170 @@
|
||||
const MAX_STDERR_BYTES = 3000;
|
||||
|
||||
/**
|
||||
* mutable per-run handle the agent harness writes to as a run progresses.
|
||||
* the action's outer try/catch in `main.ts` reads this off `toolState` when
|
||||
* the activity-timeout watchdog wins the race against the harness's own
|
||||
* catch — the bare timer reject reason ("activity timeout: no output for
|
||||
* 302s") tells the user nothing actionable, but `recentStderr` +
|
||||
* `lastProviderError` together usually point straight at the upstream cause.
|
||||
*
|
||||
* `recentStderr` is shared by reference with the harness's bounded ring
|
||||
* buffer, so the diagnostic always reflects the latest captured tail.
|
||||
*/
|
||||
export type AgentDiagnostic = {
|
||||
/** display label for the agent, e.g. "Pullfrog". used in the headline. */
|
||||
label: string;
|
||||
/** shared reference to the harness's bounded stderr ring buffer. */
|
||||
recentStderr: string[];
|
||||
/** most-recent provider-error label from `detectProviderError`, if any. */
|
||||
lastProviderError: string | undefined;
|
||||
/** count of stdout events successfully parsed before the failure. */
|
||||
eventCount: number;
|
||||
};
|
||||
|
||||
/**
|
||||
* Build a user-facing markdown body for an agent hang or failure.
|
||||
*
|
||||
* Rendered into both the PR progress comment and the GitHub Actions job
|
||||
* summary. Returns `null` when no diagnostic is available, which signals to
|
||||
* the caller to fall back to its bare-error rendering.
|
||||
*
|
||||
* `errorMessage` is the underlying timer / spawn reject string (e.g.
|
||||
* `activity timeout: no output for 301s`). The idle seconds are parsed out
|
||||
* of it for the hang explanation — total runtime would overstate the stall
|
||||
* for runs that streamed for a long time before going quiet.
|
||||
*/
|
||||
export function formatAgentHangBody(input: {
|
||||
diagnostic: AgentDiagnostic | undefined;
|
||||
isHang: boolean;
|
||||
errorMessage: string;
|
||||
}): string | null {
|
||||
if (!input.diagnostic) return null;
|
||||
|
||||
// billing exhaustion (CreditsError / FreeUsageLimitError / spending cap /
|
||||
// Insufficient balance) is mis-classified as transient by upstream harnesses
|
||||
// and the run only ends when the activity-timeout watchdog fires (see #778).
|
||||
// when we recognise the billing label, replace the generic "stalled — auth
|
||||
// error" headline with a billing-specific CTA that names the actual remedy.
|
||||
if (input.diagnostic.lastProviderError === "provider billing exhausted") {
|
||||
return formatBillingExhaustedBody(input.diagnostic);
|
||||
}
|
||||
|
||||
const verb = input.isHang ? "stalled" : "failed";
|
||||
const cause = input.diagnostic.lastProviderError
|
||||
? ` — likely cause: \`${input.diagnostic.lastProviderError}\``
|
||||
: "";
|
||||
const headline = `**${input.diagnostic.label} ${verb}**${cause}`;
|
||||
|
||||
const explanation = formatExplanation({
|
||||
isHang: input.isHang,
|
||||
errorMessage: input.errorMessage,
|
||||
});
|
||||
const parts = [headline, "", `${explanation} ${formatEventsPart(input.diagnostic)}`];
|
||||
|
||||
const tail = renderStderrTail(input.diagnostic.recentStderr);
|
||||
if (tail) {
|
||||
// pick a fence longer than any backtick run in the body so a stderr line
|
||||
// containing ``` (provider error JSON occasionally embeds it) can't
|
||||
// terminate the fence early and corrupt the rest of the markdown.
|
||||
const fence = pickFence(tail);
|
||||
parts.push(
|
||||
"",
|
||||
"<details><summary>Recent agent stderr</summary>",
|
||||
"",
|
||||
fence,
|
||||
tail,
|
||||
fence,
|
||||
"",
|
||||
"</details>"
|
||||
);
|
||||
}
|
||||
|
||||
return parts.join("\n");
|
||||
}
|
||||
|
||||
function formatExplanation(input: { isHang: boolean; errorMessage: string }): string {
|
||||
if (!input.isHang) return `The agent exited unexpectedly: ${input.errorMessage}`;
|
||||
const idleSec = parseIdleSec(input.errorMessage);
|
||||
if (idleSec === undefined) {
|
||||
return "The agent stopped emitting events and was killed by the activity-timeout watchdog.";
|
||||
}
|
||||
return `The agent stopped emitting events for ${idleSec}s and was killed by the activity-timeout watchdog.`;
|
||||
}
|
||||
|
||||
function parseIdleSec(message: string): number | undefined {
|
||||
const match = /no output for (\d+)s/.exec(message);
|
||||
return match ? Number(match[1]) : undefined;
|
||||
}
|
||||
|
||||
function formatEventsPart(diagnostic: AgentDiagnostic): string {
|
||||
if (diagnostic.eventCount > 0) {
|
||||
return `${diagnostic.eventCount} events were processed before the failure.`;
|
||||
}
|
||||
// when the provider-error label already names the cause in the headline,
|
||||
// the reachability nudge below contradicts it (e.g. an immediate 401 also
|
||||
// produces zero events but isn't a reachability problem). suppress it.
|
||||
if (diagnostic.lastProviderError) return "No events were emitted before the failure.";
|
||||
return "No events were emitted — check whether the model provider is reachable.";
|
||||
}
|
||||
|
||||
function renderStderrTail(lines: readonly string[]): string {
|
||||
if (lines.length === 0) return "";
|
||||
const joined = lines.join("\n");
|
||||
if (joined.length <= MAX_STDERR_BYTES) return joined;
|
||||
return `... (older lines truncated)\n${joined.slice(-MAX_STDERR_BYTES)}`;
|
||||
}
|
||||
|
||||
function pickFence(content: string): string {
|
||||
let max = 0;
|
||||
for (const match of content.matchAll(/`+/g)) {
|
||||
if (match[0].length > max) max = match[0].length;
|
||||
}
|
||||
return "`".repeat(Math.max(3, max + 1));
|
||||
}
|
||||
|
||||
/**
|
||||
* Pull a billing URL out of the captured stderr if the provider helpfully
|
||||
* embedded one (OpenCode Zen does — Anthropic and Gemini do not). Restricted
|
||||
* to known billing/console hosts so a stray URL elsewhere in the buffer
|
||||
* can't masquerade as the remedy link.
|
||||
*/
|
||||
function extractBillingUrl(lines: readonly string[]): string | undefined {
|
||||
const urlPattern =
|
||||
/https:\/\/(?:opencode\.ai\/[^\s"]*billing[^\s"]*|console\.anthropic\.com[^\s"]*|console\.cloud\.google\.com[^\s"]*billing[^\s"]*)/i;
|
||||
for (let i = lines.length - 1; i >= 0; i--) {
|
||||
const m = urlPattern.exec(lines[i] ?? "");
|
||||
if (m) return m[0];
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function formatBillingExhaustedBody(diagnostic: AgentDiagnostic): string {
|
||||
const headline = `**${diagnostic.label} stopped** — your model provider returned a billing-exhausted response.`;
|
||||
|
||||
const billingUrl = extractBillingUrl(diagnostic.recentStderr);
|
||||
const cta = billingUrl
|
||||
? `Top up your provider balance, then re-run: [${billingUrl}](${billingUrl})`
|
||||
: "Top up your model-provider balance (or rotate to a key with remaining credits) and re-run.";
|
||||
const explanation =
|
||||
"The agent kept retrying the request because the provider marked the failure as transient. Pullfrog's activity-timeout watchdog ended the run after no further events were emitted.";
|
||||
|
||||
const parts = [headline, "", explanation, "", cta];
|
||||
|
||||
const tail = renderStderrTail(diagnostic.recentStderr);
|
||||
if (tail) {
|
||||
const fence = pickFence(tail);
|
||||
parts.push(
|
||||
"",
|
||||
"<details><summary>Recent agent stderr</summary>",
|
||||
"",
|
||||
fence,
|
||||
tail,
|
||||
fence,
|
||||
"",
|
||||
"</details>"
|
||||
);
|
||||
}
|
||||
|
||||
return parts.join("\n");
|
||||
}
|
||||
@@ -37,6 +37,18 @@ export async function apiFetch(options: ApiFetchOptions): Promise<Response> {
|
||||
headers["x-vercel-protection-bypass"] = bypassSecret;
|
||||
}
|
||||
|
||||
// never send Content-Type on body-less requests. empirically, Vercel's
|
||||
// Next.js lambda adapter (Next 16.1.x) throws `SyntaxError: Unexpected
|
||||
// end of data` before delegating to the route handler — returning a 500 —
|
||||
// when Content-Type is set but no body is present. exact mechanism is
|
||||
// unverified (minified runtime frame), but Content-Type on a body-less
|
||||
// request has no defined semantics per RFC 9110 §8.3 anyway. see #692.
|
||||
if (!options.body) {
|
||||
for (const key of Object.keys(headers)) {
|
||||
if (key.toLowerCase() === "content-type") delete headers[key];
|
||||
}
|
||||
}
|
||||
|
||||
log.debug(`api fetch: ${options.method ?? "GET"} ${url.pathname}`);
|
||||
|
||||
const init: RequestInit = {
|
||||
|
||||
+230
-8
@@ -1,5 +1,5 @@
|
||||
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
||||
import { validateAgentApiKey } from "./apiKeys.ts";
|
||||
import { formatApiKeyErrorSummary, isApiKeyAuthError, validateAgentApiKey } from "./apiKeys.ts";
|
||||
|
||||
const base = {
|
||||
agent: { name: "opencode" },
|
||||
@@ -9,10 +9,29 @@ const base = {
|
||||
|
||||
const savedEnv = { ...process.env };
|
||||
|
||||
// keys that count as provider auth in `knownApiKeys` and would let the
|
||||
// auto-select path pass without our intent. strip all of them at test setup
|
||||
// so each `it` starts from a clean slate regardless of what's in the dev `.env`.
|
||||
const STRIPPED_PREFIXES_OR_NAMES = [
|
||||
/_API_KEY$/,
|
||||
/^CLAUDE_CODE_OAUTH_TOKEN$/,
|
||||
/^CODEX_AUTH_JSON$/,
|
||||
/^AWS_BEARER_TOKEN_BEDROCK$/,
|
||||
/^AWS_ACCESS_KEY_ID$/,
|
||||
/^AWS_SECRET_ACCESS_KEY$/,
|
||||
/^AWS_SESSION_TOKEN$/,
|
||||
/^AWS_REGION$/,
|
||||
/^BEDROCK_MODEL_ID$/,
|
||||
/^GOOGLE_APPLICATION_CREDENTIALS$/,
|
||||
/^GOOGLE_CLOUD_PROJECT$/,
|
||||
/^VERTEX_SERVICE_ACCOUNT_JSON$/,
|
||||
/^VERTEX_LOCATION$/,
|
||||
/^VERTEX_MODEL_ID$/,
|
||||
];
|
||||
|
||||
beforeEach(() => {
|
||||
// strip all known provider keys so tests start clean
|
||||
for (const key of Object.keys(process.env)) {
|
||||
if (key.endsWith("_API_KEY") || key === "CLAUDE_CODE_OAUTH_TOKEN") delete process.env[key];
|
||||
if (STRIPPED_PREFIXES_OR_NAMES.some((re) => re.test(key))) delete process.env[key];
|
||||
}
|
||||
});
|
||||
|
||||
@@ -27,11 +46,7 @@ describe("validateAgentApiKey", () => {
|
||||
});
|
||||
|
||||
it("passes for other free opencode models", () => {
|
||||
for (const slug of [
|
||||
"opencode/gpt-5-nano",
|
||||
"opencode/mimo-v2-pro-free",
|
||||
"opencode/minimax-m2.5-free",
|
||||
]) {
|
||||
for (const slug of ["opencode/mimo-v2-pro-free", "opencode/minimax-m2.5-free"]) {
|
||||
expect(() => validateAgentApiKey({ ...base, model: slug })).not.toThrow();
|
||||
}
|
||||
});
|
||||
@@ -59,6 +74,12 @@ describe("validateAgentApiKey", () => {
|
||||
"no API key found"
|
||||
);
|
||||
});
|
||||
|
||||
it("throws for opencode/gpt-5-nano without OPENCODE_API_KEY (paid Zen alias)", () => {
|
||||
expect(() => validateAgentApiKey({ ...base, model: "opencode/gpt-5-nano" })).toThrow(
|
||||
"no API key found"
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("no model (auto-select)", () => {
|
||||
@@ -71,4 +92,205 @@ describe("validateAgentApiKey", () => {
|
||||
expect(() => validateAgentApiKey({ ...base, model: undefined })).toThrow("no API key found");
|
||||
});
|
||||
});
|
||||
|
||||
describe("bedrock routing slug", () => {
|
||||
it("passes with AWS_BEARER_TOKEN_BEDROCK + AWS_REGION + BEDROCK_MODEL_ID", () => {
|
||||
process.env.AWS_BEARER_TOKEN_BEDROCK = "bedrock-token";
|
||||
process.env.AWS_REGION = "us-east-1";
|
||||
process.env.BEDROCK_MODEL_ID = "us.anthropic.claude-opus-4-7";
|
||||
expect(() => validateAgentApiKey({ ...base, model: "bedrock/byok" })).not.toThrow();
|
||||
});
|
||||
|
||||
it("passes with AWS access keys + region + model id", () => {
|
||||
process.env.AWS_ACCESS_KEY_ID = "AKIA-test";
|
||||
process.env.AWS_SECRET_ACCESS_KEY = "secret-test";
|
||||
process.env.AWS_REGION = "us-east-1";
|
||||
process.env.BEDROCK_MODEL_ID = "amazon.nova-pro-v1:0";
|
||||
expect(() => validateAgentApiKey({ ...base, model: "bedrock/byok" })).not.toThrow();
|
||||
});
|
||||
|
||||
it("throws when BEDROCK_MODEL_ID is missing", () => {
|
||||
process.env.AWS_BEARER_TOKEN_BEDROCK = "bedrock-token";
|
||||
process.env.AWS_REGION = "us-east-1";
|
||||
expect(() => validateAgentApiKey({ ...base, model: "bedrock/byok" })).toThrow(
|
||||
"BEDROCK_MODEL_ID"
|
||||
);
|
||||
});
|
||||
|
||||
it("throws when AWS_REGION is missing", () => {
|
||||
process.env.AWS_BEARER_TOKEN_BEDROCK = "bedrock-token";
|
||||
process.env.BEDROCK_MODEL_ID = "us.anthropic.claude-opus-4-7";
|
||||
expect(() => validateAgentApiKey({ ...base, model: "bedrock/byok" })).toThrow("AWS_REGION");
|
||||
});
|
||||
|
||||
it("throws when no auth is set", () => {
|
||||
process.env.AWS_REGION = "us-east-1";
|
||||
process.env.BEDROCK_MODEL_ID = "us.anthropic.claude-opus-4-7";
|
||||
expect(() => validateAgentApiKey({ ...base, model: "bedrock/byok" })).toThrow(
|
||||
"AWS_BEARER_TOKEN_BEDROCK"
|
||||
);
|
||||
});
|
||||
|
||||
it("throws when only AWS_ACCESS_KEY_ID is set (missing secret)", () => {
|
||||
process.env.AWS_ACCESS_KEY_ID = "AKIA-test";
|
||||
process.env.AWS_REGION = "us-east-1";
|
||||
process.env.BEDROCK_MODEL_ID = "us.anthropic.claude-opus-4-7";
|
||||
expect(() => validateAgentApiKey({ ...base, model: "bedrock/byok" })).toThrow(
|
||||
"AWS_BEARER_TOKEN_BEDROCK"
|
||||
);
|
||||
});
|
||||
|
||||
// regression: main.ts passes the resolved model into validateAgentApiKey
|
||||
// (`payload.proxyModel ?? resolvedModel ?? payload.model`), which for
|
||||
// bedrock is the raw AWS model ID and has no `/`. parseModel would throw.
|
||||
// see PR #720 e2e run 25821218139 for the original failure mode.
|
||||
it("accepts a raw Bedrock model ID (post-resolveModel) without throwing", () => {
|
||||
process.env.AWS_BEARER_TOKEN_BEDROCK = "bedrock-token";
|
||||
process.env.AWS_REGION = "us-east-1";
|
||||
process.env.BEDROCK_MODEL_ID = "us.anthropic.claude-opus-4-6-v1";
|
||||
expect(() =>
|
||||
validateAgentApiKey({ ...base, model: "us.anthropic.claude-opus-4-6-v1" })
|
||||
).not.toThrow();
|
||||
});
|
||||
|
||||
it("throws on raw Bedrock model ID when AWS auth is missing", () => {
|
||||
process.env.AWS_REGION = "us-east-1";
|
||||
process.env.BEDROCK_MODEL_ID = "us.anthropic.claude-opus-4-6-v1";
|
||||
expect(() =>
|
||||
validateAgentApiKey({ ...base, model: "us.anthropic.claude-opus-4-6-v1" })
|
||||
).toThrow("AWS_BEARER_TOKEN_BEDROCK");
|
||||
});
|
||||
});
|
||||
|
||||
describe("vertex routing slug", () => {
|
||||
it("passes with service-account JSON + project + location + model id", () => {
|
||||
process.env.VERTEX_SERVICE_ACCOUNT_JSON = "{}";
|
||||
process.env.GOOGLE_CLOUD_PROJECT = "test-project";
|
||||
process.env.VERTEX_LOCATION = "us-east5";
|
||||
process.env.VERTEX_MODEL_ID = "claude-opus-4-1@20250805";
|
||||
expect(() => validateAgentApiKey({ ...base, model: "vertex/byok" })).not.toThrow();
|
||||
});
|
||||
|
||||
it("passes when project is derivable from service-account JSON", () => {
|
||||
process.env.VERTEX_SERVICE_ACCOUNT_JSON = JSON.stringify({ project_id: "test-project" });
|
||||
process.env.VERTEX_LOCATION = "us-east5";
|
||||
process.env.VERTEX_MODEL_ID = "gemini-2.5-pro";
|
||||
expect(() => validateAgentApiKey({ ...base, model: "vertex/byok" })).not.toThrow();
|
||||
});
|
||||
|
||||
it("throws when VERTEX_MODEL_ID is missing", () => {
|
||||
process.env.VERTEX_SERVICE_ACCOUNT_JSON = "{}";
|
||||
process.env.GOOGLE_CLOUD_PROJECT = "test-project";
|
||||
process.env.VERTEX_LOCATION = "us-east5";
|
||||
expect(() => validateAgentApiKey({ ...base, model: "vertex/byok" })).toThrow(
|
||||
"VERTEX_MODEL_ID"
|
||||
);
|
||||
});
|
||||
|
||||
it("throws when VERTEX_LOCATION is missing", () => {
|
||||
process.env.VERTEX_SERVICE_ACCOUNT_JSON = "{}";
|
||||
process.env.GOOGLE_CLOUD_PROJECT = "test-project";
|
||||
process.env.VERTEX_MODEL_ID = "claude-opus-4-1@20250805";
|
||||
expect(() => validateAgentApiKey({ ...base, model: "vertex/byok" })).toThrow(
|
||||
"VERTEX_LOCATION"
|
||||
);
|
||||
});
|
||||
|
||||
it("throws when GOOGLE_CLOUD_PROJECT is missing and not derivable", () => {
|
||||
process.env.VERTEX_SERVICE_ACCOUNT_JSON = "{}";
|
||||
process.env.VERTEX_LOCATION = "us-east5";
|
||||
process.env.VERTEX_MODEL_ID = "claude-opus-4-1@20250805";
|
||||
expect(() => validateAgentApiKey({ ...base, model: "vertex/byok" })).toThrow(
|
||||
"GOOGLE_CLOUD_PROJECT"
|
||||
);
|
||||
});
|
||||
|
||||
it("throws when no auth path is set", () => {
|
||||
process.env.GOOGLE_CLOUD_PROJECT = "test-project";
|
||||
process.env.VERTEX_LOCATION = "us-east5";
|
||||
process.env.VERTEX_MODEL_ID = "claude-opus-4-1@20250805";
|
||||
expect(() => validateAgentApiKey({ ...base, model: "vertex/byok" })).toThrow(
|
||||
"VERTEX_SERVICE_ACCOUNT_JSON"
|
||||
);
|
||||
});
|
||||
|
||||
it("accepts a raw Vertex model ID (post-resolveModel) without throwing", () => {
|
||||
process.env.VERTEX_SERVICE_ACCOUNT_JSON = "{}";
|
||||
process.env.GOOGLE_CLOUD_PROJECT = "test-project";
|
||||
process.env.VERTEX_LOCATION = "us-east5";
|
||||
process.env.VERTEX_MODEL_ID = "gemini-2.5-pro";
|
||||
expect(() => validateAgentApiKey({ ...base, model: "gemini-2.5-pro" })).not.toThrow();
|
||||
});
|
||||
|
||||
it("throws on raw Vertex model ID when auth is missing", () => {
|
||||
process.env.GOOGLE_CLOUD_PROJECT = "test-project";
|
||||
process.env.VERTEX_LOCATION = "us-east5";
|
||||
process.env.VERTEX_MODEL_ID = "gemini-2.5-pro";
|
||||
expect(() => validateAgentApiKey({ ...base, model: "gemini-2.5-pro" })).toThrow(
|
||||
"VERTEX_SERVICE_ACCOUNT_JSON"
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("isApiKeyAuthError", () => {
|
||||
it("matches the missing-key marker thrown by validateAgentApiKey", () => {
|
||||
expect(isApiKeyAuthError("no API key found. Pullfrog needs ...")).toBe(true);
|
||||
});
|
||||
|
||||
it("matches Claude CLI 401 strings", () => {
|
||||
expect(isApiKeyAuthError("Invalid API key · Fix external API key")).toBe(true);
|
||||
});
|
||||
|
||||
it("matches OpenAI / OpenRouter 401 phrasings", () => {
|
||||
expect(isApiKeyAuthError("ProviderAuthError: User not found")).toBe(true);
|
||||
expect(isApiKeyAuthError("401 Invalid authentication")).toBe(true);
|
||||
});
|
||||
|
||||
// see #782 — direct-Anthropic 401 shape (revoked / mistyped / rotated
|
||||
// ANTHROPIC_API_KEY) reaches us via Claude CLI as a JSON dump, not as
|
||||
// any of the canonical "Invalid API key" strings. these matchers ensure
|
||||
// the formatted CTA fires instead of the raw 401 JSON blob.
|
||||
it("matches direct-Anthropic 401 shapes", () => {
|
||||
expect(
|
||||
isApiKeyAuthError(
|
||||
'Failed to authenticate. API Error: 401 {"type":"error","error":{"type":"authentication_error","message":"Invalid bearer token"}}'
|
||||
)
|
||||
).toBe(true);
|
||||
expect(
|
||||
isApiKeyAuthError(
|
||||
"» Pullfrog result error: subtype=success, api_error_status=401, message=Failed to authenticate."
|
||||
)
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("ignores unrelated errors", () => {
|
||||
expect(isApiKeyAuthError("git fetch failed")).toBe(false);
|
||||
expect(isApiKeyAuthError("")).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("formatApiKeyErrorSummary", () => {
|
||||
it("renders the missing-key body when the raw error contains the marker", () => {
|
||||
const msg = formatApiKeyErrorSummary({
|
||||
owner: "acme",
|
||||
name: "repo",
|
||||
raw: "no API key found in this run",
|
||||
});
|
||||
expect(msg).toContain("no API key found");
|
||||
expect(msg).toContain("https://github.com/acme/repo/settings/secrets/actions");
|
||||
expect(msg).toContain("/console/acme/repo");
|
||||
expect(msg).toContain("https://discord.gg/8y96raFg8e");
|
||||
});
|
||||
|
||||
it("renders the invalid-key body for any other auth error", () => {
|
||||
const msg = formatApiKeyErrorSummary({
|
||||
owner: "acme",
|
||||
name: "repo",
|
||||
raw: "Invalid API key · Fix external API key",
|
||||
});
|
||||
expect(msg).toContain("rejected (401)");
|
||||
expect(msg).toContain("https://github.com/acme/repo/settings/secrets/actions");
|
||||
expect(msg).toContain("https://discord.gg/8y96raFg8e");
|
||||
});
|
||||
});
|
||||
|
||||
+169
-15
@@ -1,28 +1,70 @@
|
||||
import { getModelEnvVars, providers } from "../models.ts";
|
||||
import {
|
||||
BEDROCK_MODEL_ID_ENV,
|
||||
getModelEnvVars,
|
||||
providers,
|
||||
resolveDisplayAlias,
|
||||
VERTEX_MODEL_ID_ENV,
|
||||
} from "../models.ts";
|
||||
import { getApiUrl } from "./apiUrl.ts";
|
||||
import {
|
||||
GOOGLE_CLOUD_PROJECT_ENV,
|
||||
readProjectIdFromVertexServiceAccountJson,
|
||||
VERTEX_LOCATION_ENV,
|
||||
VERTEX_SERVICE_ACCOUNT_JSON_ENV,
|
||||
} from "./vertex.ts";
|
||||
|
||||
const knownApiKeys: Set<string> = new Set(Object.values(providers).flatMap((p) => [...p.envVars]));
|
||||
const knownApiKeys: Set<string> = new Set(
|
||||
Object.values(providers).flatMap((p) => [...p.envVars, ...(p.managedCredentials ?? [])])
|
||||
);
|
||||
|
||||
/** marker prefix on the throw message for the catch-side reclassification path */
|
||||
const MISSING_KEY_MARKER = "no API key found";
|
||||
|
||||
/** Markdown body used for both the thrown error and the formatted PR comment summary. */
|
||||
function buildMissingApiKeyError(params: { owner: string; name: string }): string {
|
||||
const apiUrl = getApiUrl();
|
||||
const settingsUrl = `${apiUrl}/console/${params.owner}/${params.name}`;
|
||||
const githubSecretsUrl = `https://github.com/${params.owner}/${params.name}/settings/secrets/actions`;
|
||||
const settingsUrl = `${getApiUrl()}/console/${params.owner}/${params.name}`;
|
||||
|
||||
const githubRepoUrl = `https://github.com/${params.owner}/${params.name}`;
|
||||
const githubSecretsUrl = `${githubRepoUrl}/settings/secrets/actions`;
|
||||
return [
|
||||
`**${MISSING_KEY_MARKER}** — Pullfrog needs at least one LLM provider API key (e.g. \`ANTHROPIC_API_KEY\`, \`OPENAI_API_KEY\`, \`GEMINI_API_KEY\`) configured as a GitHub Actions secret.`,
|
||||
"",
|
||||
`[Open repo secrets →](${githubSecretsUrl}) · [Configure model →](${settingsUrl}) · [Setup docs →](https://docs.pullfrog.com/keys) · [Ask in Discord →](https://discord.gg/8y96raFg8e)`,
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
return `no API key found. Pullfrog requires at least one LLM provider API key.
|
||||
function buildBedrockSetupError(params: {
|
||||
owner: string;
|
||||
name: string;
|
||||
missing: string[];
|
||||
}): string {
|
||||
const githubSecretsUrl = `https://github.com/${params.owner}/${params.name}/settings/secrets/actions`;
|
||||
|
||||
to fix this, add the required secret to your GitHub repository:
|
||||
return `Bedrock model selected but required configuration is missing: ${params.missing.join(", ")}.
|
||||
|
||||
1. go to: ${githubSecretsUrl}
|
||||
2. click "New repository secret"
|
||||
3. set the name to your provider's key (e.g., \`ANTHROPIC_API_KEY\`, \`OPENAI_API_KEY\`, \`GEMINI_API_KEY\`)
|
||||
4. set the value to your API key
|
||||
5. click "Add secret"
|
||||
add the missing secret(s) to your GitHub repository at ${githubSecretsUrl}, then reference them in your workflow's \`env:\` block:
|
||||
|
||||
configure your model at ${settingsUrl}
|
||||
AWS_BEARER_TOKEN_BEDROCK: \${{ secrets.AWS_BEARER_TOKEN_BEDROCK }}
|
||||
AWS_REGION: \${{ secrets.AWS_REGION }}
|
||||
${BEDROCK_MODEL_ID_ENV}: \${{ secrets.${BEDROCK_MODEL_ID_ENV} }}
|
||||
|
||||
for full setup instructions, see https://docs.pullfrog.com/keys`;
|
||||
\`AWS_BEARER_TOKEN_BEDROCK\` may be substituted with \`AWS_ACCESS_KEY_ID\` + \`AWS_SECRET_ACCESS_KEY\` (and optional \`AWS_SESSION_TOKEN\`) if you prefer access keys.
|
||||
|
||||
for full setup instructions, see https://docs.pullfrog.com/bedrock`;
|
||||
}
|
||||
|
||||
function buildVertexSetupError(params: { owner: string; name: string; missing: string[] }): string {
|
||||
const githubSecretsUrl = `https://github.com/${params.owner}/${params.name}/settings/secrets/actions`;
|
||||
|
||||
return `Google Vertex AI model selected but required configuration is missing: ${params.missing.join(", ")}.
|
||||
|
||||
add the missing secret(s) to your GitHub repository at ${githubSecretsUrl}, then reference them in your workflow's \`env:\` block:
|
||||
|
||||
${VERTEX_SERVICE_ACCOUNT_JSON_ENV}: \${{ secrets.${VERTEX_SERVICE_ACCOUNT_JSON_ENV} }}
|
||||
${GOOGLE_CLOUD_PROJECT_ENV}: my-project
|
||||
${VERTEX_LOCATION_ENV}: global
|
||||
${VERTEX_MODEL_ID_ENV}: <vertex-model-id>
|
||||
|
||||
for full setup instructions, see https://docs.pullfrog.com/vertex`;
|
||||
}
|
||||
|
||||
function hasEnvVar(name: string): boolean {
|
||||
@@ -37,6 +79,39 @@ export function hasProviderKey(model: string): boolean {
|
||||
return requiredVars.some((v) => hasEnvVar(v));
|
||||
}
|
||||
|
||||
function validateBedrockSetup(params: { owner: string; name: string }): void {
|
||||
const hasAuth =
|
||||
hasEnvVar("AWS_BEARER_TOKEN_BEDROCK") ||
|
||||
(hasEnvVar("AWS_ACCESS_KEY_ID") && hasEnvVar("AWS_SECRET_ACCESS_KEY"));
|
||||
|
||||
const missing: string[] = [];
|
||||
if (!hasAuth)
|
||||
missing.push("AWS_BEARER_TOKEN_BEDROCK (or AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY)");
|
||||
if (!hasEnvVar("AWS_REGION")) missing.push("AWS_REGION");
|
||||
if (!hasEnvVar(BEDROCK_MODEL_ID_ENV)) missing.push(BEDROCK_MODEL_ID_ENV);
|
||||
|
||||
if (missing.length > 0) {
|
||||
throw new Error(buildBedrockSetupError({ owner: params.owner, name: params.name, missing }));
|
||||
}
|
||||
}
|
||||
|
||||
function validateVertexSetup(params: { owner: string; name: string }): void {
|
||||
const hasAuth = hasEnvVar(VERTEX_SERVICE_ACCOUNT_JSON_ENV);
|
||||
const hasProject =
|
||||
hasEnvVar(GOOGLE_CLOUD_PROJECT_ENV) ||
|
||||
readProjectIdFromVertexServiceAccountJson() !== undefined;
|
||||
|
||||
const missing: string[] = [];
|
||||
if (!hasAuth) missing.push(VERTEX_SERVICE_ACCOUNT_JSON_ENV);
|
||||
if (!hasProject) missing.push(GOOGLE_CLOUD_PROJECT_ENV);
|
||||
if (!hasEnvVar(VERTEX_LOCATION_ENV)) missing.push(VERTEX_LOCATION_ENV);
|
||||
if (!hasEnvVar(VERTEX_MODEL_ID_ENV)) missing.push(VERTEX_MODEL_ID_ENV);
|
||||
|
||||
if (missing.length > 0) {
|
||||
throw new Error(buildVertexSetupError({ owner: params.owner, name: params.name, missing }));
|
||||
}
|
||||
}
|
||||
|
||||
export function validateAgentApiKey(params: {
|
||||
agent: { name: string };
|
||||
model: string | undefined;
|
||||
@@ -45,6 +120,36 @@ export function validateAgentApiKey(params: {
|
||||
}): void {
|
||||
// if a specific model is configured, only check that model's required env vars
|
||||
if (params.model) {
|
||||
// routing slugs (e.g. bedrock) get a tailored validation path because
|
||||
// their auth shape doesn't match the standard "any one envVar present"
|
||||
// rule (Bedrock needs auth + region + model-id, with auth being either
|
||||
// a bearer token OR an access-key pair).
|
||||
const alias = resolveDisplayAlias(params.model);
|
||||
if (alias?.routing === "bedrock") {
|
||||
validateBedrockSetup({ owner: params.owner, name: params.name });
|
||||
return;
|
||||
}
|
||||
if (alias?.routing === "vertex") {
|
||||
validateVertexSetup({ owner: params.owner, name: params.name });
|
||||
return;
|
||||
}
|
||||
|
||||
// upstream `resolveModel` translates routing slugs into raw backend
|
||||
// model IDs (e.g. `us.anthropic.claude-opus-4-6-v1`), which have no `/`
|
||||
// and so isn't parseable as `provider/model`. these IDs only reach this
|
||||
// function via routing aliases, so re-run the matching setup check rather
|
||||
// than falling through to `getModelEnvVars` (which would throw inside
|
||||
// parseModel). resolveModel itself already enforced the model-id env var,
|
||||
// but auth + location/region are still validated here.
|
||||
if (!params.model.includes("/")) {
|
||||
if (process.env[VERTEX_MODEL_ID_ENV]?.trim() === params.model) {
|
||||
validateVertexSetup({ owner: params.owner, name: params.name });
|
||||
return;
|
||||
}
|
||||
validateBedrockSetup({ owner: params.owner, name: params.name });
|
||||
return;
|
||||
}
|
||||
|
||||
const requiredVars = getModelEnvVars(params.model);
|
||||
// free models have no required env vars — skip validation entirely
|
||||
if (requiredVars.length === 0) return;
|
||||
@@ -59,3 +164,52 @@ export function validateAgentApiKey(params: {
|
||||
throw new Error(buildMissingApiKeyError({ owner: params.owner, name: params.name }));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect agent-runtime auth failures that should be reformatted as an actionable
|
||||
* key-fix CTA before being shown to the user. Covers the shapes we see:
|
||||
* - missing key (validateAgentApiKey throw): contains MISSING_KEY_MARKER
|
||||
* - revoked / invalid key (Claude CLI 401 surfaced via api_error_status):
|
||||
* "Invalid API key · Fix external API key" + similar provider variants
|
||||
* - direct-Anthropic 401 (`Failed to authenticate. API Error: 401 ...
|
||||
* {"type":"error","error":{"type":"authentication_error", ...
|
||||
* "Invalid bearer token"}}`) emitted by the Claude CLI for revoked /
|
||||
* mistyped / rotated `ANTHROPIC_API_KEY`. see #782.
|
||||
*/
|
||||
export function isApiKeyAuthError(text: string): boolean {
|
||||
if (!text) return false;
|
||||
return (
|
||||
text.includes(MISSING_KEY_MARKER) ||
|
||||
/Invalid API key/i.test(text) ||
|
||||
/\bUser not found\b/i.test(text) ||
|
||||
/\bInvalid authentication\b/i.test(text) ||
|
||||
/authentication_error/i.test(text) ||
|
||||
/Invalid bearer token/i.test(text) ||
|
||||
/api_error_status\s*=\s*401/i.test(text) ||
|
||||
/API Error:\s*401/i.test(text)
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Friendly Markdown summary for both the missing-key and invalid-key cases.
|
||||
* Used in the catch / result-failure paths in `main.ts` to overwrite the raw
|
||||
* agent error before it's posted to the PR progress comment.
|
||||
*/
|
||||
export function formatApiKeyErrorSummary(params: {
|
||||
owner: string;
|
||||
name: string;
|
||||
raw: string;
|
||||
}): string {
|
||||
if (params.raw.includes(MISSING_KEY_MARKER)) {
|
||||
return buildMissingApiKeyError({ owner: params.owner, name: params.name });
|
||||
}
|
||||
|
||||
const githubSecretsUrl = `https://github.com/${params.owner}/${params.name}/settings/secrets/actions`;
|
||||
const settingsUrl = `${getApiUrl()}/console/${params.owner}/${params.name}`;
|
||||
|
||||
return [
|
||||
`**Your LLM provider API key was rejected (401).** Rotate the key in your provider dashboard, then update the matching GitHub Actions secret.`,
|
||||
"",
|
||||
`[Update repo secret →](${githubSecretsUrl}) · [Model settings →](${settingsUrl}) · [Setup docs →](https://docs.pullfrog.com/keys) · [Ask in Discord →](https://discord.gg/8y96raFg8e)`,
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
@@ -25,3 +25,18 @@ export function getApiUrl(): string {
|
||||
log.debug(`resolved API_URL: ${raw}`);
|
||||
return raw;
|
||||
}
|
||||
|
||||
/**
|
||||
* true when the action is configured to talk to a localhost API server (i.e.
|
||||
* `pnpm dev` running on the developer's box). signals we can use dev-only
|
||||
* affordances like the `x-dev-repo` proxy-token bypass — the corresponding
|
||||
* server-side dev gates (`NODE_ENV === "development"`) ensure these paths
|
||||
* never activate against prod regardless of what the action does.
|
||||
*/
|
||||
export function isLocalApiUrl(): boolean {
|
||||
try {
|
||||
return isLocalUrl(new URL(getApiUrl()));
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,189 @@
|
||||
/**
|
||||
* Billing-error classification + user-facing copy for `/api/proxy-token`
|
||||
* failures and OpenRouter mid-run exhaustion. Two error classes (Billing vs.
|
||||
* Transient) keep the framing honest: a card decline is *not* the same UX as
|
||||
* a 503 from the proxy service. Both originate in `utils/proxy.ts` (mint
|
||||
* failures) and `utils/runErrorRenderer.ts` (mid-run keylimit reclassify).
|
||||
*
|
||||
* Renderers return markdown bodies that are written into both the GitHub
|
||||
* Actions job summary and the PR progress comment.
|
||||
*
|
||||
* Lives outside `main.ts` so adding a new error `code` branch is a one-file
|
||||
* edit that does not retrigger the full LLM CI matrix (`action/main.ts` is
|
||||
* in `action/test/coverage.ts::ALWAYS_RUN_ALL`).
|
||||
*/
|
||||
|
||||
/**
|
||||
* Billing-layer error surfaced from `/api/proxy-token` as a 402. User-actionable
|
||||
* — distinct from TransientError (503 / transient sync issue) so the job
|
||||
* summary + PR comment can use affirmative "you need to do X" copy rather than
|
||||
* the ambiguous "billing error" label that makes transient outages look like
|
||||
* the user's fault.
|
||||
*
|
||||
* `code` is a server-side discriminator: `router_requires_card` (no card + no
|
||||
* wallet balance on Router), or null for unclassified. `declineCode` is
|
||||
* Stripe's more specific sub-reason on `card_declined` (e.g.
|
||||
* `insufficient_funds`, `lost_card`). `needsReauthentication` is the 3DS case
|
||||
* broken out for convenience.
|
||||
*/
|
||||
export class BillingError extends Error {
|
||||
code: string | null;
|
||||
declineCode: string | null;
|
||||
needsReauthentication: boolean;
|
||||
|
||||
constructor(
|
||||
message: string,
|
||||
opts: {
|
||||
code?: string | null;
|
||||
declineCode?: string | null;
|
||||
needsReauthentication?: boolean;
|
||||
} = {}
|
||||
) {
|
||||
super(message);
|
||||
this.name = "BillingError";
|
||||
this.code = opts.code ?? null;
|
||||
this.declineCode = opts.declineCode ?? null;
|
||||
this.needsReauthentication = opts.needsReauthentication ?? false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Transient service failures from `/api/proxy-token` (503: partial OpenRouter
|
||||
* usage sync, DB flake, in-flight payment intent). Not the user's fault — the
|
||||
* summary uses "temporarily unavailable" framing, and the non-zero exit lets
|
||||
* GH Actions apply whatever retry policy the workflow has configured.
|
||||
*/
|
||||
export class TransientError extends Error {
|
||||
constructor(message: string) {
|
||||
super(message);
|
||||
this.name = "TransientError";
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Deep link into the right console section for the failing account. Anchors
|
||||
* are defined in `app/console/[owner]/page.tsx` (`#billing`, `#model-access`).
|
||||
* `owner` is the GitHub login of the repo's account — i.e. the org or user
|
||||
* that pays for this repo's runs, which is the right scope for billing.
|
||||
*/
|
||||
function billingConsoleUrl(owner: string, anchor: "billing" | "model-access"): string {
|
||||
return `https://pullfrog.com/console/${encodeURIComponent(owner)}#${anchor}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Render a BillingError as user-facing markdown (shared between GH job summary
|
||||
* and the PR progress comment). Goals:
|
||||
*
|
||||
* - quiet, not alarmist — bold first line instead of an `### ❌` H3, since
|
||||
* the comment already has Pullfrog branding in the footer
|
||||
* - actionable — every branch ends in a single CTA deep-linked to the
|
||||
* correct section of the owner's console
|
||||
* - honest — say what actually went wrong (card declined vs. balance
|
||||
* empty vs. 3DS required), don't lump them under "billing error"
|
||||
*
|
||||
* Branches:
|
||||
* - `router_requires_card`: user is on Router mode with no card AND no
|
||||
* wallet balance (signup credit exhausted or not granted). Frame as
|
||||
* "add a card to continue", link to `#model-access` where the Add
|
||||
* Card flow lives.
|
||||
* - `router_balance_exhausted`: user has a card on file but auto-reload is
|
||||
* disabled and they've spent past their $5 overdraft buffer. Frame as
|
||||
* "balance ran out" and surface both remediation paths (top up, or flip
|
||||
* on auto-reload).
|
||||
* - `router_keylimit_exhausted`: OpenRouter rejected mid-run because the
|
||||
* per-run key budget was exhausted while the agent was working. The
|
||||
* wallet is now negative; same remediation as `router_balance_exhausted`
|
||||
* but framed for the after-the-fact case ("this run was cut short").
|
||||
* - `needsReauthentication`: issuer requires 3DS on every off-session
|
||||
* charge. Re-adding the card won't help — the only escape is a manual
|
||||
* top-up where 3DS runs interactively in Stripe Checkout.
|
||||
* - `declineCode` set: Stripe declined a real charge. Show the sub-code
|
||||
* so support can act on it; tell the user we'll retry on next dispatch.
|
||||
* - default: balance hit zero with no in-flight charge (auto-reload off
|
||||
* or amount below threshold). Direct them to top up or enable auto-reload.
|
||||
*/
|
||||
export function formatBillingErrorSummary(error: BillingError, owner: string): string {
|
||||
if (error.code === "router_requires_card") {
|
||||
return [
|
||||
"**Add a card to start using Pullfrog Router.**",
|
||||
"",
|
||||
"Router proxies OpenRouter at raw cost — no platform markup. Add a card and we'll auto-reload your wallet so runs keep flowing.",
|
||||
"",
|
||||
`[Add a card →](${billingConsoleUrl(owner, "model-access")})`,
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
if (error.code === "router_balance_exhausted") {
|
||||
return [
|
||||
"**Your Pullfrog Router balance is exhausted.**",
|
||||
"",
|
||||
"You have a card on file but auto-reload is disabled, so runs paused once your balance went past the overdraft buffer.",
|
||||
"",
|
||||
`[Top up balance →](${billingConsoleUrl(owner, "billing")}) · [Enable auto-reload →](${billingConsoleUrl(owner, "model-access")})`,
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
if (error.code === "router_keylimit_exhausted") {
|
||||
return [
|
||||
"**This run was cut short — your Pullfrog Router balance ran out mid-run.**",
|
||||
"",
|
||||
"OpenRouter stopped the agent because the per-run budget was exhausted. Your wallet is now negative; top up or enable auto-reload to keep runs flowing.",
|
||||
"",
|
||||
`[Top up balance →](${billingConsoleUrl(owner, "billing")}) · [Enable auto-reload →](${billingConsoleUrl(owner, "model-access")})`,
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
if (error.code === "router_monthly_limit") {
|
||||
return [
|
||||
"**Pullfrog Router hit its monthly spend limit.**",
|
||||
"",
|
||||
"Auto-reloads are paused for the rest of this UTC month. Ask your admin to raise the cap, or wait for it to reset at 00:00 UTC on the 1st.",
|
||||
"",
|
||||
`[Adjust limit →](${billingConsoleUrl(owner, "model-access")})`,
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
if (error.needsReauthentication) {
|
||||
const code = error.declineCode ?? "authentication_required";
|
||||
return [
|
||||
`**Your card issuer requires 3D Secure on every charge** (\`${code}\`).`,
|
||||
"",
|
||||
"Pullfrog can't complete a 3DS challenge from inside a workflow. Top up your Router balance once in Stripe Checkout — subsequent runs draw from the prepaid balance without re-triggering 3DS.",
|
||||
"",
|
||||
`[Top up balance →](${billingConsoleUrl(owner, "billing")})`,
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
if (error.declineCode) {
|
||||
return [
|
||||
`**Your card was declined** (\`${error.declineCode}\`).`,
|
||||
"",
|
||||
"Update your payment method and Pullfrog will retry on the next run.",
|
||||
"",
|
||||
`[Update payment method →](${billingConsoleUrl(owner, "billing")})`,
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
return [
|
||||
"**Your Pullfrog balance is empty.**",
|
||||
"",
|
||||
"Top up your balance or enable auto-reload to keep runs flowing.",
|
||||
"",
|
||||
`[Manage billing →](${billingConsoleUrl(owner, "billing")})`,
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
/**
|
||||
* Render a TransientError as user-facing markdown. Distinct framing from
|
||||
* BillingError so the user doesn't read an alarm and assume their card
|
||||
* failed — this branch is "our fault, retry shortly", not theirs.
|
||||
*/
|
||||
export function formatTransientErrorSummary(error: TransientError, owner: string): string {
|
||||
return [
|
||||
"**Pullfrog billing is temporarily unavailable.**",
|
||||
"",
|
||||
error.message,
|
||||
"",
|
||||
`Usually transient — the next dispatch should succeed. If it persists, check [status.pullfrog.com](https://status.pullfrog.com) or [your console](${billingConsoleUrl(owner, "billing")}).`,
|
||||
].join("\n");
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user