b8ac42e875
* mcp: embed example calls in top-level tool descriptions
agents (esp. claude sonnet) hallucinate param names from training-data
priors — `pr_number` instead of `pull_number`, `summary` instead of
`body`, full subcommand strings jammed into `git({command})` like it
were `shell({command})`. each error burns a tool round-trip plus a
follow-up ToolSearch, ~40+ events / 24h, no observable recovery cost
to us but visible to users in agent logs.
cheapest fix: add a sample formatted function call to every affected
tool's top-level description. example anchors are more reliable than
schema descriptions alone because the model treats descriptions as
narrative but call examples as canonical structure. for `git` and
`shell` (whose `command` fields collide), include explicit
counter-examples disambiguating which tool owns which shape.
no schema aliases / coercion yet — try the cheap thing first; if the
next audit window still shows the same hallucination rate, layer
aliases on top per #585's recommendation.
closes #585, closes #701
* mcp: drop negative anchors from tool descriptions
negation is a footgun in tool descriptions — telling the model "NOT
pr_number" makes pr_number more salient, not less. let the positive
example carry the schema and trust the model to read it.
removes:
- "the parameter is pull_number (a number), NOT pr_number" and
similar across checkout_pr, get_pull_request, list_pull_request_reviews,
get_review_comments, create_pull_request_review
- "NOT summary, message, or content" on report_progress
- "WRONG: git({ command: 'log --oneline' })" counter-example on git
- redundant param-type restatements after the example (e.g. "depth is a
number, not a string" on git_fetch, "description is required" on shell)
keeps a single positive example per tool. for tools with multiple call
shapes (git, git_fetch, push_branch), two positive examples instead of
one + a counter-example.
507 lines
18 KiB
TypeScript
507 lines
18 KiB
TypeScript
import { type } from "arktype";
|
|
import { getApiUrl } from "../utils/apiUrl.ts";
|
|
import { buildPullfrogFooter, stripExistingFooter } from "../utils/buildPullfrogFooter.ts";
|
|
import { log } from "../utils/cli.ts";
|
|
import { fixDoubleEscapedString } from "../utils/fixDoubleEscapedString.ts";
|
|
import { patchWorkflowRunFields } from "../utils/patchWorkflowRunFields.ts";
|
|
import {
|
|
createLeapingProgressComment,
|
|
deleteProgressCommentApi,
|
|
updateProgressComment,
|
|
} from "../utils/progressComment.ts";
|
|
import type { ToolContext } from "./server.ts";
|
|
import { execute, tool } from "./shared.ts";
|
|
|
|
// re-export for backward compat with anything importing the leaping helpers from mcp/comment
|
|
export {
|
|
isLeapingIntoActionCommentBody,
|
|
LEAPING_INTO_ACTION_PREFIX,
|
|
} from "../utils/leapingComment.ts";
|
|
|
|
function buildCommentFooter(ctx: ToolContext, customParts?: string[]): string {
|
|
const runId = ctx.runId;
|
|
return buildPullfrogFooter({
|
|
triggeredBy: true,
|
|
workflowRun:
|
|
runId !== undefined
|
|
? {
|
|
owner: ctx.repo.owner,
|
|
repo: ctx.repo.name,
|
|
runId,
|
|
jobId: ctx.jobId,
|
|
}
|
|
: undefined,
|
|
customParts,
|
|
model: ctx.toolState.model,
|
|
});
|
|
}
|
|
|
|
function buildImplementPlanLink(ctx: ToolContext, issueNumber: number, commentId: number): string {
|
|
const apiUrl = getApiUrl();
|
|
return `[Implement plan ➔](${apiUrl}/trigger/${ctx.repo.owner}/${ctx.repo.name}/${issueNumber}?action=implement&comment_id=${commentId})`;
|
|
}
|
|
|
|
export function addFooter(ctx: ToolContext, body: string): string {
|
|
if (/<br\s*\/?>[ \t]*\n(?!\s*\n)/i.test(body)) {
|
|
throw new Error(
|
|
"body contains <br/> followed by a non-blank line, which breaks GitHub markdown rendering. always add a blank line after <br/> tags."
|
|
);
|
|
}
|
|
const bodyWithoutFooter = stripExistingFooter(fixDoubleEscapedString(body));
|
|
const footer = buildCommentFooter(ctx);
|
|
return `${bodyWithoutFooter}${footer}`;
|
|
}
|
|
|
|
export const Comment = type({
|
|
issueNumber: type.number.describe("the issue number to comment on"),
|
|
body: type.string.describe("the comment body content"),
|
|
type: type
|
|
.enumerated("Plan", "Comment")
|
|
.describe("Plan: record as the plan for this run. Comment: regular comment (default).")
|
|
.optional(),
|
|
});
|
|
|
|
export function CreateCommentTool(ctx: ToolContext) {
|
|
return tool({
|
|
name: "create_issue_comment",
|
|
description:
|
|
"Create a comment on a GitHub issue or PR. " +
|
|
'Example: `create_issue_comment({ issueNumber: 1234, body: "Thanks for the report." })`. ' +
|
|
"For progress/plan updates on the current run use report_progress instead. Use type: 'Plan' for plan comments.",
|
|
parameters: Comment,
|
|
execute: execute(async ({ issueNumber, body, type: commentType }) => {
|
|
const bodyWithFooter = addFooter(ctx, body);
|
|
|
|
const result = await ctx.octokit.rest.issues.createComment({
|
|
owner: ctx.repo.owner,
|
|
repo: ctx.repo.name,
|
|
issue_number: issueNumber,
|
|
body: bodyWithFooter,
|
|
});
|
|
|
|
ctx.toolState.wasUpdated = true;
|
|
log.info(`» created comment ${result.data.id}`);
|
|
|
|
if (commentType === "Plan") {
|
|
if (result.data.node_id) {
|
|
await patchWorkflowRunFields(ctx, { planCommentNodeId: result.data.node_id });
|
|
}
|
|
// add "Implement plan" link (needs comment ID, so create-then-update)
|
|
const customParts = [buildImplementPlanLink(ctx, issueNumber, result.data.id)];
|
|
const footer = buildCommentFooter(ctx, customParts);
|
|
const bodyWithPlanLink = `${stripExistingFooter(body)}${footer}`;
|
|
|
|
const updateResult = await ctx.octokit.rest.issues.updateComment({
|
|
owner: ctx.repo.owner,
|
|
repo: ctx.repo.name,
|
|
comment_id: result.data.id,
|
|
body: bodyWithPlanLink,
|
|
});
|
|
log.info(`» updated comment ${updateResult.data.id}`);
|
|
|
|
return {
|
|
success: true,
|
|
commentId: updateResult.data.id,
|
|
url: updateResult.data.html_url,
|
|
body: updateResult.data.body,
|
|
};
|
|
}
|
|
|
|
return {
|
|
success: true,
|
|
commentId: result.data.id,
|
|
url: result.data.html_url,
|
|
body: result.data.body,
|
|
};
|
|
}),
|
|
});
|
|
}
|
|
|
|
export const EditComment = type({
|
|
commentId: type.number.describe("the ID of the comment to edit"),
|
|
body: type.string.describe("the new comment body content"),
|
|
});
|
|
|
|
export function EditCommentTool(ctx: ToolContext) {
|
|
return tool({
|
|
name: "edit_issue_comment",
|
|
description: "Edit a GitHub issue comment by its ID",
|
|
parameters: EditComment,
|
|
execute: execute(async ({ commentId, body }) => {
|
|
const bodyWithFooter = addFooter(ctx, body);
|
|
|
|
const result = await ctx.octokit.rest.issues.updateComment({
|
|
owner: ctx.repo.owner,
|
|
repo: ctx.repo.name,
|
|
comment_id: commentId,
|
|
body: bodyWithFooter,
|
|
});
|
|
log.info(`» updated comment ${result.data.id}`);
|
|
|
|
return {
|
|
success: true,
|
|
commentId: result.data.id,
|
|
url: result.data.html_url,
|
|
body: result.data.body,
|
|
updatedAt: result.data.updated_at,
|
|
};
|
|
}),
|
|
});
|
|
}
|
|
|
|
export const ReportProgress = type({
|
|
body: type.string.describe("the progress update content to share"),
|
|
"target_plan_comment?": type("boolean").describe(
|
|
"when true, update the existing plan comment (from select_mode lookup) instead of the progress comment; use when editing an existing plan"
|
|
),
|
|
});
|
|
|
|
/**
|
|
* Report progress to a GitHub comment.
|
|
*
|
|
* progressComment has three states:
|
|
* - undefined: no comment yet — will create one if an issue/PR target exists
|
|
* - object: active comment — will update it in place via the right REST endpoint for its type
|
|
* - null: deliberately deleted (e.g. after submitting a PR review) — skips silently
|
|
*
|
|
* The body is always tracked in lastProgressBody for the job summary regardless of comment state.
|
|
*
|
|
* The "existing plan comment" path always targets a top-level issue comment (plan comments are
|
|
* created by create_issue_comment with type:"Plan", never as review-thread replies).
|
|
*/
|
|
export async function reportProgress(
|
|
ctx: ToolContext,
|
|
params: { body: string; target_plan_comment?: boolean }
|
|
): Promise<{
|
|
commentId?: number;
|
|
url?: string;
|
|
body: string;
|
|
action: "created" | "updated" | "skipped";
|
|
}> {
|
|
const { body, target_plan_comment } = params;
|
|
// always track the body for job summary
|
|
ctx.toolState.lastProgressBody = body;
|
|
|
|
// silent events (e.g., auto-label, pr-summary Task) should never create or update progress comments.
|
|
// the body is still tracked above for the GitHub Actions job summary.
|
|
if (ctx.payload.event.silent) {
|
|
return { body, action: "skipped" };
|
|
}
|
|
|
|
const issueNumber = ctx.payload.event.issue_number ?? ctx.toolState.issueNumber;
|
|
const isPlanMode = ctx.toolState.selectedMode === "Plan";
|
|
const apiCtx = { octokit: ctx.octokit, owner: ctx.repo.owner, repo: ctx.repo.name };
|
|
|
|
// when editing existing plan: update the plan comment from tool state (set by select_mode)
|
|
if (target_plan_comment === true && ctx.toolState.existingPlanCommentId === undefined) {
|
|
log.warning("target_plan_comment requested but no existingPlanCommentId in tool state");
|
|
}
|
|
if (target_plan_comment === true && ctx.toolState.existingPlanCommentId !== undefined) {
|
|
const commentId = ctx.toolState.existingPlanCommentId;
|
|
const customParts =
|
|
issueNumber !== undefined ? [buildImplementPlanLink(ctx, issueNumber, commentId)] : undefined;
|
|
const bodyWithoutFooter = stripExistingFooter(body);
|
|
const footer = buildCommentFooter(ctx, customParts);
|
|
const bodyWithFooter = `${bodyWithoutFooter}${footer}`;
|
|
|
|
const result = await updateProgressComment(
|
|
apiCtx,
|
|
{ id: commentId, type: "issue" },
|
|
bodyWithFooter
|
|
);
|
|
|
|
ctx.toolState.wasUpdated = true;
|
|
|
|
if (isPlanMode && result.node_id) {
|
|
await patchWorkflowRunFields(ctx, { planCommentNodeId: result.node_id });
|
|
}
|
|
|
|
return {
|
|
commentId: result.id,
|
|
url: result.html_url,
|
|
body: result.body || "",
|
|
action: "updated",
|
|
};
|
|
}
|
|
|
|
const existingComment = ctx.toolState.progressComment;
|
|
|
|
// if we already have a progress comment, update it
|
|
if (existingComment) {
|
|
const customParts =
|
|
isPlanMode && issueNumber !== undefined
|
|
? [buildImplementPlanLink(ctx, issueNumber, existingComment.id)]
|
|
: undefined;
|
|
|
|
const bodyWithoutFooter = stripExistingFooter(body);
|
|
const footer = buildCommentFooter(ctx, customParts);
|
|
const bodyWithFooter = `${bodyWithoutFooter}${footer}`;
|
|
|
|
const result = await updateProgressComment(apiCtx, existingComment, bodyWithFooter);
|
|
|
|
ctx.toolState.wasUpdated = true;
|
|
|
|
if (isPlanMode && result.node_id) {
|
|
await patchWorkflowRunFields(ctx, { planCommentNodeId: result.node_id });
|
|
}
|
|
|
|
return {
|
|
commentId: result.id,
|
|
url: result.html_url,
|
|
body: result.body || "",
|
|
action: "updated",
|
|
};
|
|
}
|
|
|
|
// null = progress comment was deleted by stranded-comment cleanup in main.ts
|
|
if (existingComment === null) {
|
|
return { body, action: "skipped" };
|
|
}
|
|
|
|
// no existing comment - need an issue/PR to create one on
|
|
// use fallback chain: dynamically set context > event payload
|
|
if (issueNumber === undefined) {
|
|
// no-op: no comment target (e.g., workflow_dispatch events)
|
|
// body is already tracked for job summary
|
|
return { body, action: "skipped" };
|
|
}
|
|
|
|
// for new comments, we need to create first, then update with Plan link if in Plan mode
|
|
// self-created progress comments are always top-level issue comments — review-reply
|
|
// progress comments only originate from the dispatch path and arrive pre-created.
|
|
const initialBody = addFooter(ctx, body);
|
|
const created = await createLeapingProgressComment(
|
|
apiCtx,
|
|
{ kind: "issue", issueNumber },
|
|
initialBody
|
|
);
|
|
|
|
ctx.toolState.progressComment = created.comment;
|
|
ctx.toolState.wasUpdated = true;
|
|
|
|
// if Plan mode, update the comment to add the "Implement plan" link
|
|
if (isPlanMode) {
|
|
const customParts = [buildImplementPlanLink(ctx, issueNumber, created.comment.id)];
|
|
const bodyWithoutFooter = stripExistingFooter(body);
|
|
const footer = buildCommentFooter(ctx, customParts);
|
|
const bodyWithPlanLink = `${bodyWithoutFooter}${footer}`;
|
|
|
|
const updateResult = await updateProgressComment(apiCtx, created.comment, bodyWithPlanLink);
|
|
|
|
if (updateResult.node_id) {
|
|
await patchWorkflowRunFields(ctx, { planCommentNodeId: updateResult.node_id });
|
|
}
|
|
|
|
return {
|
|
commentId: updateResult.id,
|
|
url: updateResult.html_url,
|
|
body: updateResult.body || "",
|
|
action: "created",
|
|
};
|
|
}
|
|
|
|
return {
|
|
commentId: created.comment.id,
|
|
url: created.html_url,
|
|
body: created.body || "",
|
|
action: "created",
|
|
};
|
|
}
|
|
|
|
export function ReportProgressTool(ctx: ToolContext) {
|
|
return tool({
|
|
name: "report_progress",
|
|
description:
|
|
"Share progress on the associated GitHub issue/PR. The first call creates a comment; subsequent calls update it in place. " +
|
|
'Example: `report_progress({ body: "Implemented the auth check and added tests." })`. ' +
|
|
"Call this at the end of every run with a brief final summary (1-3 sentences) unless the mode guidance instructs otherwise. The current task list is automatically appended in a collapsible section — do not restate individual steps.",
|
|
parameters: ReportProgress,
|
|
execute: execute(async (params) => {
|
|
let body = params.body;
|
|
|
|
// for non-plan calls: stop auto-updates, wait for in-flight writes to settle,
|
|
// then append completed task list collapsible
|
|
if (!params.target_plan_comment && ctx.toolState.todoTracker) {
|
|
ctx.toolState.todoTracker.cancel();
|
|
await ctx.toolState.todoTracker.settled();
|
|
const collapsible = ctx.toolState.todoTracker.renderCollapsible({
|
|
completeInProgress: true,
|
|
});
|
|
if (collapsible) {
|
|
body = `${body}\n\n${collapsible}`;
|
|
}
|
|
}
|
|
|
|
const reportParams: { body: string; target_plan_comment?: boolean } = { body };
|
|
if (params.target_plan_comment !== undefined) {
|
|
reportParams.target_plan_comment = params.target_plan_comment;
|
|
}
|
|
const result = await reportProgress(ctx, reportParams);
|
|
|
|
if (result.action === "skipped") {
|
|
return {
|
|
success: true,
|
|
message:
|
|
"progress recorded (no GitHub comment created - this may occur for workflow_dispatch events or when there is no associated issue/PR)",
|
|
};
|
|
}
|
|
|
|
if (result.commentId !== undefined) {
|
|
log.info(`» ${result.action} comment ${result.commentId}`);
|
|
}
|
|
|
|
if (!params.target_plan_comment) {
|
|
ctx.toolState.finalSummaryWritten = true;
|
|
}
|
|
|
|
return {
|
|
success: true,
|
|
...result,
|
|
};
|
|
}),
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Delete the progress comment if it exists.
|
|
* Used by main.ts for stranded-comment cleanup (orphaned "Leaping into action" or
|
|
* checklist left by the todo tracker when the agent didn't call report_progress).
|
|
* Sets progressComment to null so subsequent report_progress calls are no-ops.
|
|
*/
|
|
export async function deleteProgressComment(ctx: ToolContext): Promise<boolean> {
|
|
const existing = ctx.toolState.progressComment;
|
|
if (!existing) {
|
|
return false;
|
|
}
|
|
|
|
try {
|
|
await deleteProgressCommentApi(
|
|
{ octokit: ctx.octokit, owner: ctx.repo.owner, repo: ctx.repo.name },
|
|
existing
|
|
);
|
|
} catch (error) {
|
|
// ignore 404 - comment already deleted
|
|
if (error instanceof Error && error.message.includes("Not Found")) {
|
|
// comment already deleted, continue
|
|
} else {
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
// set to null (not undefined) so report_progress skips instead of creating a new comment
|
|
ctx.toolState.progressComment = null;
|
|
|
|
return true;
|
|
}
|
|
|
|
export const ReplyToReviewComment = type({
|
|
pull_number: type.number.describe("the pull request number"),
|
|
comment_id: type.number.describe("the ID of the review comment to reply to"),
|
|
body: type.string.describe(
|
|
"extremely brief reply (1 sentence max) explaining what was fixed, e.g. 'Fixed by renaming to X' or 'Added null check'"
|
|
),
|
|
});
|
|
|
|
/**
|
|
* decision returned by `duplicateReplyDecision` when a session has already
|
|
* posted an identical reply to the same parent review comment.
|
|
*/
|
|
export interface DuplicateReplyDecision {
|
|
kind: "already-replied";
|
|
commentId: number;
|
|
url: string | undefined;
|
|
reason: string;
|
|
}
|
|
|
|
/**
|
|
* decide whether a second reply_to_review_comment call in the same session
|
|
* is a duplicate of an earlier reply to the same parent comment.
|
|
*
|
|
* the agent is instructed to call reply_to_review_comment exactly once per
|
|
* parent comment per AddressReviews session, but in practice it sometimes
|
|
* emits the same call twice. PR #610 reproduced this with Kimi K2:
|
|
* identical body posted 3 seconds apart, only one tool_use event in the
|
|
* agent log. the second post is always redundant and clutters the PR thread.
|
|
*
|
|
* we key on (comment_id, bodyWithFooter) so a legitimate follow-up reply
|
|
* with different content still goes through. within a single run the
|
|
* footer is constant (workflow run + model + jobId), so byte-equal bodies
|
|
* catch the stutter without blocking real follow-ups.
|
|
*
|
|
* mirrors the shape of `duplicateReviewDecision` in mcp/review.ts.
|
|
*/
|
|
export function duplicateReplyDecision(params: {
|
|
existing: { commentId: number; url: string | undefined; bodyWithFooter: string } | undefined;
|
|
bodyWithFooter: string;
|
|
}): DuplicateReplyDecision | null {
|
|
const existing = params.existing;
|
|
if (!existing) return null;
|
|
if (existing.bodyWithFooter !== params.bodyWithFooter) return null;
|
|
return {
|
|
kind: "already-replied",
|
|
commentId: existing.commentId,
|
|
url: existing.url,
|
|
reason: `reply ${existing.commentId} with identical body was already posted in this session; ignoring duplicate call`,
|
|
};
|
|
}
|
|
|
|
export function ReplyToReviewCommentTool(ctx: ToolContext) {
|
|
return tool({
|
|
name: "reply_to_review_comment",
|
|
description:
|
|
"Reply to a PR review comment thread (NOT issue comments — this only works for inline review comments on PR diffs). " +
|
|
'Example: `reply_to_review_comment({ pull_number: 1234, comment_id: 567890, body: "Fixed by adding a null check." })`. ' +
|
|
"Call exactly ONCE per parent comment you address in AddressReviews mode — duplicate calls with the same body are a no-op. Keep replies extremely brief (1 sentence max).",
|
|
parameters: ReplyToReviewComment,
|
|
execute: execute(async ({ pull_number, comment_id, body }) => {
|
|
const bodyWithFooter = addFooter(ctx, body);
|
|
|
|
// guard against duplicate reply submissions in the same session.
|
|
// see duplicateReplyDecision for the rationale.
|
|
const dup = duplicateReplyDecision({
|
|
existing: ctx.toolState.reviewReplies?.get(comment_id),
|
|
bodyWithFooter,
|
|
});
|
|
if (dup) {
|
|
log.info(`skipping duplicate review reply: ${dup.reason}`);
|
|
return {
|
|
success: true,
|
|
skipped: true,
|
|
reason: dup.reason,
|
|
commentId: dup.commentId,
|
|
url: dup.url,
|
|
};
|
|
}
|
|
|
|
const result = await ctx.octokit.rest.pulls.createReplyForReviewComment({
|
|
owner: ctx.repo.owner,
|
|
repo: ctx.repo.name,
|
|
pull_number,
|
|
comment_id,
|
|
body: bodyWithFooter,
|
|
});
|
|
log.info(`» created review comment ${result.data.id} (in reply to ${comment_id})`);
|
|
|
|
// mark progress as updated so error reporting + run-result handling know
|
|
// a substantive write happened (used by reportErrorToComment / handleAgentResult)
|
|
ctx.toolState.wasUpdated = true;
|
|
|
|
// record this reply for in-session dedupe of subsequent identical calls.
|
|
ctx.toolState.reviewReplies ??= new Map();
|
|
ctx.toolState.reviewReplies.set(comment_id, {
|
|
commentId: result.data.id,
|
|
url: result.data.html_url,
|
|
bodyWithFooter,
|
|
});
|
|
|
|
return {
|
|
success: true,
|
|
commentId: result.data.id,
|
|
url: result.data.html_url,
|
|
body: result.data.body,
|
|
in_reply_to_id: result.data.in_reply_to_id,
|
|
};
|
|
}, "reply_to_review_comment"),
|
|
});
|
|
}
|