b8ac42e875
* mcp: embed example calls in top-level tool descriptions
agents (esp. claude sonnet) hallucinate param names from training-data
priors — `pr_number` instead of `pull_number`, `summary` instead of
`body`, full subcommand strings jammed into `git({command})` like it
were `shell({command})`. each error burns a tool round-trip plus a
follow-up ToolSearch, ~40+ events / 24h, no observable recovery cost
to us but visible to users in agent logs.
cheapest fix: add a sample formatted function call to every affected
tool's top-level description. example anchors are more reliable than
schema descriptions alone because the model treats descriptions as
narrative but call examples as canonical structure. for `git` and
`shell` (whose `command` fields collide), include explicit
counter-examples disambiguating which tool owns which shape.
no schema aliases / coercion yet — try the cheap thing first; if the
next audit window still shows the same hallucination rate, layer
aliases on top per #585's recommendation.
closes #585, closes #701
* mcp: drop negative anchors from tool descriptions
negation is a footgun in tool descriptions — telling the model "NOT
pr_number" makes pr_number more salient, not less. let the positive
example carry the schema and trust the model to read it.
removes:
- "the parameter is pull_number (a number), NOT pr_number" and
similar across checkout_pr, get_pull_request, list_pull_request_reviews,
get_review_comments, create_pull_request_review
- "NOT summary, message, or content" on report_progress
- "WRONG: git({ command: 'log --oneline' })" counter-example on git
- redundant param-type restatements after the example (e.g. "depth is a
number, not a string" on git_fetch, "description is required" on shell)
keeps a single positive example per tool. for tools with multiple call
shapes (git, git_fetch, push_branch), two positive examples instead of
one + a counter-example.
182 lines
8.9 KiB
TypeScript
182 lines
8.9 KiB
TypeScript
import { type } from "arktype";
|
|
import { formatMcpToolRef } from "../external.ts";
|
|
import type { Mode } from "../modes.ts";
|
|
import { apiFetch } from "../utils/apiFetch.ts";
|
|
import type { ToolContext } from "./server.ts";
|
|
import { execute, tool } from "./shared.ts";
|
|
|
|
export const SelectModeParams = type({
|
|
mode: type.string.describe(
|
|
"the name of the mode to select (e.g., 'Build', 'Plan', 'Review', 'IncrementalReview', 'Fix', 'AddressReviews', 'Task', 'ResolveConflicts')"
|
|
),
|
|
"issue_number?": type("number").describe(
|
|
"optional issue number; when provided with Plan mode, used to look up an existing plan comment for this issue (edit vs create)"
|
|
),
|
|
});
|
|
|
|
function resolveMode(modes: Mode[], modeName: string): Mode | null {
|
|
return modes.find((m) => m.name.toLowerCase() === modeName.toLowerCase()) ?? null;
|
|
}
|
|
|
|
function buildModeOverrides(t: (name: string) => string): Record<string, string> {
|
|
return {
|
|
PlanEdit: `### Checklist (editing existing plan)
|
|
|
|
An existing plan comment was found for this issue. Update that comment with the revised plan — do not create a new plan comment.
|
|
|
|
1. **task list**: create your task list for this run as your first action.
|
|
2. Use \`previousPlanBody\` from this response as the plan to revise; do not call \`get_issue\` or \`get_issue_comments\`.
|
|
3. Revise the plan based on the user's request:
|
|
- incorporate the current plan (\`previousPlanBody\`) and the user's revision request
|
|
- gather relevant codebase context (file paths, architecture notes from AGENTS.md)
|
|
- produce a structured plan with clear milestones
|
|
4. Call \`${t("report_progress")}\` with the full revised plan text and \`{ target_plan_comment: true }\` so it updates the existing plan comment (not the progress comment).
|
|
5. Then post a short note to the progress comment (e.g. "Plan has been updated in the comment above.") via \`${t("report_progress")}\` so it is not left as "Leaping...".`,
|
|
};
|
|
}
|
|
|
|
type OrchestratorGuidance = {
|
|
modeName: string;
|
|
description: string;
|
|
orchestratorGuidance: string;
|
|
};
|
|
|
|
// IncrementalReview inherits Review's user instructions, Fix inherits Build's
|
|
const modeInstructionParent: Record<string, string> = {
|
|
IncrementalReview: "Review",
|
|
Fix: "Build",
|
|
};
|
|
|
|
function buildOrchestratorGuidance(
|
|
ctx: ToolContext,
|
|
mode: Mode,
|
|
overrideGuidance?: string
|
|
): OrchestratorGuidance {
|
|
const hardcoded = overrideGuidance ?? mode.prompt ?? "";
|
|
const lookupKey = modeInstructionParent[mode.name] ?? mode.name;
|
|
const userInstructions = ctx.modeInstructions[lookupKey] ?? "";
|
|
const guidance = [hardcoded, userInstructions].filter(Boolean).join("\n\n");
|
|
return {
|
|
modeName: mode.name,
|
|
description: mode.description,
|
|
orchestratorGuidance: guidance,
|
|
};
|
|
}
|
|
|
|
// matches the API response for /repo/[owner]/[repo]/issue/[issueNumber]/plan-comment
|
|
export type PlanCommentResponsePayload = { error: string } | { commentId: number; body: string };
|
|
|
|
// IMPORTANT: this route authenticates via GitHub installation token (getEnrichedRepo),
|
|
// NOT the Pullfrog API JWT (ctx.apiToken). use ctx.githubInstallationToken here.
|
|
// see wiki/api-auth.md for the two auth patterns.
|
|
async function fetchExistingPlanComment(
|
|
ctx: ToolContext,
|
|
issueNumber: number
|
|
): Promise<Extract<PlanCommentResponsePayload, { commentId: number }> | null> {
|
|
if (!ctx.githubInstallationToken) return null;
|
|
try {
|
|
const response = await apiFetch({
|
|
path: `/api/repo/${ctx.repo.owner}/${ctx.repo.name}/issue/${issueNumber}/plan-comment`,
|
|
method: "GET",
|
|
headers: { authorization: `Bearer ${ctx.githubInstallationToken}` },
|
|
signal: AbortSignal.timeout(10_000),
|
|
});
|
|
const data = (await response.json()) as PlanCommentResponsePayload;
|
|
return response.ok && "commentId" in data ? data : null;
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
const SUMMARY_MODES = new Set(["Review", "IncrementalReview", "Task"]);
|
|
|
|
/** modes that gain the PR summary edit step when toolState.summaryFilePath is set.
|
|
*
|
|
* NOTE: this snapshot is an internal artifact consumed by future agent runs. it is
|
|
* deliberately NOT shaped by user-supplied summary instructions — those would warp
|
|
* the durable agent context. user-facing summarization (e.g. the review body's
|
|
* "Reviewed changes" section) is governed by review-mode prompts and review
|
|
* instructions, separately from this snapshot. */
|
|
function buildSummaryAddendum(t: (name: string) => string, ctx: ToolContext): string {
|
|
const filePath = ctx.toolState.summaryFilePath;
|
|
if (!filePath) return "";
|
|
return `### PR summary snapshot — required step
|
|
|
|
A rolling PR summary lives at \`${filePath}\`. It is your durable cross-run agent context — a functional summary of what this PR does, the subsystems and files it touches, the material behavior of its changes, and any risks or open questions worth carrying forward. It is NOT a chronological log of past review runs; commit-level history can already be reconstructed from \`${t("list_pull_request_reviews")}\`.
|
|
|
|
How to use it:
|
|
|
|
- read \`${filePath}\` at the START of the run, alongside the diff. it represents what previous agent runs already understood about this PR — absorb it before picking lenses or crafting subagent dispatch prompts. if it's a fresh seed (file is one or two lines), this is a first review and you'll be filling it in from the diff.
|
|
- let the snapshot inform triage and dispatch. when it already tracks a risk, your lens prompts to subagents are stronger when they reference that context (e.g. "the JSDoc explicitly scopes to code points — do not flag grapheme-cluster issues" if the snapshot already documents that contract). when something the snapshot tracks is now resolved by new commits, note that. when new commits introduce something the snapshot doesn't yet describe, that's exactly where your fan-out should focus.
|
|
- update the file in place to reflect the PR's CURRENT state. revise stale claims, drop resolved risks, add new behavior or risks. accuracy over breadth — every claim must be grounded in the diff. write for the next agent run, not for a human.
|
|
- structure however serves THIS PR. there is no required section template. a refactor might organize by renamed export and call-site impact; a feature by capability; a billing change by money path. a compact note of which commit ranges have been reviewed should always be present so future runs scope correctly, but the rest is your call. when the structure works across runs, keep it stable so range-diffs are clean; when the PR's character changes (e.g. scope expands), reshape.
|
|
|
|
Do NOT call \`${t("create_issue_comment")}\` for the summary — the server reads this file at end-of-run and persists it. The file edit is mandatory regardless of whether a review is submitted; the snapshot feeds the next run.`;
|
|
}
|
|
|
|
export function SelectModeTool(ctx: ToolContext) {
|
|
const t = (name: string) => formatMcpToolRef(ctx.agentId, name);
|
|
const overrides = buildModeOverrides(t);
|
|
|
|
return tool({
|
|
name: "select_mode",
|
|
description:
|
|
"Select a mode and receive step-by-step guidance on how to handle the task. Call this to understand the best workflow for the current mode. " +
|
|
'Example: `select_mode({ mode: "Review" })` or `select_mode({ mode: "Plan", issue_number: 1234 })`.',
|
|
parameters: SelectModeParams,
|
|
execute: execute(async (params) => {
|
|
if (ctx.toolState.selectedMode) {
|
|
return {
|
|
error: `mode already selected: "${ctx.toolState.selectedMode}". mode selection is final and cannot be changed. complete your current workflow within this mode.`,
|
|
};
|
|
}
|
|
|
|
const modeName = params.mode;
|
|
|
|
const selectedMode = resolveMode(ctx.modes, modeName);
|
|
|
|
if (!selectedMode) {
|
|
const availableModes = ctx.modes.map((m) => m.name).join(", ");
|
|
return {
|
|
error: `mode "${modeName}" not found. available modes: ${availableModes}`,
|
|
availableModes: ctx.modes.map((m) => ({
|
|
name: m.name,
|
|
description: m.description,
|
|
})),
|
|
};
|
|
}
|
|
|
|
ctx.toolState.selectedMode = selectedMode.name;
|
|
|
|
if (selectedMode.name === "Plan") {
|
|
const issueNumber = params.issue_number ?? ctx.payload.event.issue_number;
|
|
if (issueNumber !== undefined) {
|
|
const existing = await fetchExistingPlanComment(ctx, issueNumber);
|
|
if (existing !== null) {
|
|
ctx.toolState.existingPlanCommentId = existing.commentId;
|
|
ctx.toolState.previousPlanBody = existing.body;
|
|
return {
|
|
...buildOrchestratorGuidance(ctx, selectedMode, overrides.PlanEdit),
|
|
previousPlanBody: existing.body,
|
|
};
|
|
}
|
|
}
|
|
}
|
|
|
|
const summaryAddendum = SUMMARY_MODES.has(selectedMode.name)
|
|
? buildSummaryAddendum(t, ctx)
|
|
: "";
|
|
|
|
const base = buildOrchestratorGuidance(ctx, selectedMode);
|
|
if (summaryAddendum.length > 0) {
|
|
return {
|
|
...base,
|
|
orchestratorGuidance: `${base.orchestratorGuidance}\n\n${summaryAddendum}`,
|
|
summaryFilePath: ctx.toolState.summaryFilePath,
|
|
};
|
|
}
|
|
return base;
|
|
}),
|
|
});
|
|
}
|