From b8ac42e875c456e8374f466ca02457bc65a8546f Mon Sep 17 00:00:00 2001 From: Colin McDonnell Date: Wed, 13 May 2026 22:45:08 +0000 Subject: [PATCH] mcp: embed example calls in top-level tool descriptions (#723) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * mcp: embed example calls in top-level tool descriptions agents (esp. claude sonnet) hallucinate param names from training-data priors — `pr_number` instead of `pull_number`, `summary` instead of `body`, full subcommand strings jammed into `git({command})` like it were `shell({command})`. each error burns a tool round-trip plus a follow-up ToolSearch, ~40+ events / 24h, no observable recovery cost to us but visible to users in agent logs. cheapest fix: add a sample formatted function call to every affected tool's top-level description. example anchors are more reliable than schema descriptions alone because the model treats descriptions as narrative but call examples as canonical structure. for `git` and `shell` (whose `command` fields collide), include explicit counter-examples disambiguating which tool owns which shape. no schema aliases / coercion yet — try the cheap thing first; if the next audit window still shows the same hallucination rate, layer aliases on top per #585's recommendation. closes #585, closes #701 * mcp: drop negative anchors from tool descriptions negation is a footgun in tool descriptions — telling the model "NOT pr_number" makes pr_number more salient, not less. let the positive example carry the schema and trust the model to read it. removes: - "the parameter is pull_number (a number), NOT pr_number" and similar across checkout_pr, get_pull_request, list_pull_request_reviews, get_review_comments, create_pull_request_review - "NOT summary, message, or content" on report_progress - "WRONG: git({ command: 'log --oneline' })" counter-example on git - redundant param-type restatements after the example (e.g. "depth is a number, not a string" on git_fetch, "description is required" on shell) keeps a single positive example per tool. for tools with multiple call shapes (git, git_fetch, push_branch), two positive examples instead of one + a counter-example. --- mcp/checkout.ts | 1 + mcp/comment.ts | 12 +++++++++--- mcp/commitInfo.ts | 3 ++- mcp/git.ts | 20 ++++++++++---------- mcp/issueComments.ts | 3 ++- mcp/issueInfo.ts | 4 +++- mcp/prInfo.ts | 4 +++- mcp/review.ts | 9 ++++----- mcp/reviewComments.ts | 4 +++- mcp/selectMode.ts | 3 ++- mcp/shell.ts | 2 ++ 11 files changed, 41 insertions(+), 24 deletions(-) diff --git a/mcp/checkout.ts b/mcp/checkout.ts index bbf948a..ae47dc6 100644 --- a/mcp/checkout.ts +++ b/mcp/checkout.ts @@ -594,6 +594,7 @@ export function CheckoutPrTool(ctx: ToolContext) { description: "Checkout a pull request branch locally. This fetches the PR branch and sets up push configuration for fork PRs. " + "Returns diffPath pointing to the formatted diff file. " + + "Example: `checkout_pr({ pull_number: 1234 })`. " + "Transient fetch timeouts are common — retry the same call up to a few times before treating the failure as terminal. " + "If the error mentions `.git/shallow.lock: File exists` or `.git/index.lock: File exists`, that's a stale lock from a prior timed-out fetch — remove it via the shell tool (`rm -f .git/shallow.lock .git/index.lock`) and retry.", parameters: CheckoutPr, diff --git a/mcp/comment.ts b/mcp/comment.ts index 12c118a..ad717d8 100644 --- a/mcp/comment.ts +++ b/mcp/comment.ts @@ -65,7 +65,9 @@ export function CreateCommentTool(ctx: ToolContext) { return tool({ name: "create_issue_comment", description: - "Create a comment on a GitHub issue or PR. For progress/plan updates on the current run use report_progress instead. Use type: 'Plan' for plan comments.", + "Create a comment on a GitHub issue or PR. " + + 'Example: `create_issue_comment({ issueNumber: 1234, body: "Thanks for the report." })`. ' + + "For progress/plan updates on the current run use report_progress instead. Use type: 'Plan' for plan comments.", parameters: Comment, execute: execute(async ({ issueNumber, body, type: commentType }) => { const bodyWithFooter = addFooter(ctx, body); @@ -310,7 +312,9 @@ export function ReportProgressTool(ctx: ToolContext) { return tool({ name: "report_progress", description: - "Share progress on the associated GitHub issue/PR. The first call creates a comment; subsequent calls update it in place. Call this at the end of every run with a brief final summary (1-3 sentences) unless the mode guidance instructs otherwise. The current task list is automatically appended in a collapsible section — do not restate individual steps.", + "Share progress on the associated GitHub issue/PR. The first call creates a comment; subsequent calls update it in place. " + + 'Example: `report_progress({ body: "Implemented the auth check and added tests." })`. ' + + "Call this at the end of every run with a brief final summary (1-3 sentences) unless the mode guidance instructs otherwise. The current task list is automatically appended in a collapsible section — do not restate individual steps.", parameters: ReportProgress, execute: execute(async (params) => { let body = params.body; @@ -445,7 +449,9 @@ export function ReplyToReviewCommentTool(ctx: ToolContext) { return tool({ name: "reply_to_review_comment", description: - "Reply to a PR review comment thread (NOT issue comments — this only works for inline review comments on PR diffs). Call exactly ONCE per parent comment you address in AddressReviews mode — duplicate calls with the same body are a no-op. Keep replies extremely brief (1 sentence max).", + "Reply to a PR review comment thread (NOT issue comments — this only works for inline review comments on PR diffs). " + + 'Example: `reply_to_review_comment({ pull_number: 1234, comment_id: 567890, body: "Fixed by adding a null check." })`. ' + + "Call exactly ONCE per parent comment you address in AddressReviews mode — duplicate calls with the same body are a no-op. Keep replies extremely brief (1 sentence max).", parameters: ReplyToReviewComment, execute: execute(async ({ pull_number, comment_id, body }) => { const bodyWithFooter = addFooter(ctx, body); diff --git a/mcp/commitInfo.ts b/mcp/commitInfo.ts index 66fc35a..44c86b6 100644 --- a/mcp/commitInfo.ts +++ b/mcp/commitInfo.ts @@ -15,7 +15,8 @@ export function CommitInfoTool(ctx: ToolContext) { name: "get_commit_info", description: "Retrieve commit metadata and diff via GitHub API. Use this instead of git show for reviewing commits - " + - "it works with shallow clones and shows the actual changes in the commit. Returns diffPath pointing to formatted diff file.", + "it works with shallow clones and shows the actual changes in the commit. Returns diffPath pointing to formatted diff file. " + + 'Example: `get_commit_info({ sha: "2a6ab5d06ba64dd987bd88c4403287" })`.', parameters: CommitInfo, execute: execute(async ({ sha }) => { const response = await ctx.octokit.rest.repos.getCommit({ diff --git a/mcp/git.ts b/mcp/git.ts index fe9ea99..d882787 100644 --- a/mcp/git.ts +++ b/mcp/git.ts @@ -218,11 +218,11 @@ export function PushBranchTool(ctx: ToolContext) { name: "push_branch", description: "Push the current branch to the remote repository. Omit branchName to push the current branch (recommended). " + + 'Example: `push_branch({})` to push the current branch. Example: `push_branch({ branchName: "pr-1" })` to push a specific local branch. ' + "If specifying branchName, use the LOCAL branch name (e.g., 'pr-1'), not the remote branch name. " + "The correct remote and remote branch are determined automatically from branch config set by checkout_pr. " + "Requires a clean working tree. Runs the repository prepush hook (if configured) before the network push — hook failure means tests/lint or similar in that script failed, not necessarily a Pullfrog timeout. " + - "Never force push unless explicitly requested. Pushes to the default branch are blocked in restricted mode. " + - "If the response reports a timeout, the underlying push may have actually succeeded — verify with `git log origin/` (or this tool with command 'log') before retrying, otherwise you'll push a duplicate.", + "Never force push unless explicitly requested. Pushes to the default branch are blocked in restricted mode.", parameters: PushBranch, execute: execute(async ({ branchName, force }) => { // permission check @@ -445,19 +445,17 @@ const subcommandPattern = regex("^[a-z][a-z0-9-]*$"); const Git = type({ command: type(subcommandPattern).describe("Git command (e.g., 'status', 'log', 'diff')"), - args: type.string - .array() - .describe( - 'Additional arguments for the git command, as a JSON array of strings (NOT a single string). e.g. args: ["HEAD"], args: ["--oneline", "-20"]. Passing a single string fails validation.' - ) - .optional(), + args: type.string.array().describe("Additional arguments for the git command").optional(), }); export function GitTool(ctx: ToolContext) { return tool({ name: "git", description: - "Run git commands. For push/fetch, use the dedicated MCP tools (push_branch, git_fetch). " + + "Run a git subcommand. `command` is a single subcommand; flags and positional args go in `args`. " + + 'Example: `git({ command: "log", args: ["--oneline", "-n", "20"] })`. ' + + 'Example: `git({ command: "diff", args: ["origin/main..HEAD"] })`. ' + + "For push/fetch, use the dedicated MCP tools (push_branch, git_fetch). " + "git pull is not available — use git_fetch then this tool with command 'merge'.", parameters: Git, execute: execute(async (params) => { @@ -529,7 +527,9 @@ const DEEPEN_RETRY_DEPTH = 1000; export function GitFetchTool(ctx: ToolContext) { return tool({ name: "git_fetch", - description: "Fetch refs from remote repository. Use this instead of git fetch directly.", + description: + "Fetch refs from remote repository. Use this instead of git fetch directly. " + + 'Example: `git_fetch({ ref: "main" })`. With depth: `git_fetch({ ref: "pull/1234/head", depth: 1 })`.', parameters: GitFetch, execute: execute(async (params) => { rejectIfLeadingDash(params.ref, "ref"); diff --git a/mcp/issueComments.ts b/mcp/issueComments.ts index 9d2c8b1..fd1d4db 100644 --- a/mcp/issueComments.ts +++ b/mcp/issueComments.ts @@ -10,7 +10,8 @@ export function GetIssueCommentsTool(ctx: ToolContext) { return tool({ name: "get_issue_comments", description: - "Get all comments for a GitHub issue. Returns all comments including the issue body and all subsequent discussion comments.", + "Get all comments for a GitHub issue. Returns all comments including the issue body and all subsequent discussion comments. " + + "Example: `get_issue_comments({ issue_number: 1234 })`.", parameters: GetIssueComments, execute: execute(async ({ issue_number }) => { // set issue context diff --git a/mcp/issueInfo.ts b/mcp/issueInfo.ts index d74c170..6b222d2 100644 --- a/mcp/issueInfo.ts +++ b/mcp/issueInfo.ts @@ -9,7 +9,9 @@ export const IssueInfo = type({ export function IssueInfoTool(ctx: ToolContext) { return tool({ name: "get_issue", - description: "Retrieve GitHub issue information by issue number", + description: + "Retrieve GitHub issue information by issue number. " + + "Example: `get_issue({ issue_number: 1234 })`.", parameters: IssueInfo, execute: execute(async ({ issue_number }) => { const issue = await ctx.octokit.rest.issues.get({ diff --git a/mcp/prInfo.ts b/mcp/prInfo.ts index 7206a6a..39b83cd 100644 --- a/mcp/prInfo.ts +++ b/mcp/prInfo.ts @@ -30,7 +30,9 @@ export function PullRequestInfoTool(ctx: ToolContext) { return tool({ name: "get_pull_request", description: - "Retrieve PR metadata (title, body, state, branches, author, labels, linked issues). To checkout a PR branch locally, use checkout_pr instead.", + "Retrieve PR metadata (title, body, state, branches, author, labels, linked issues). " + + "Example: `get_pull_request({ pull_number: 1234 })`. " + + "To checkout a PR branch locally, use checkout_pr instead.", parameters: PullRequestInfo, execute: execute(async ({ pull_number }) => { // fetch REST and GraphQL in parallel diff --git a/mcp/review.ts b/mcp/review.ts index 658313b..d03c01c 100644 --- a/mcp/review.ts +++ b/mcp/review.ts @@ -320,16 +320,14 @@ export const CreatePullRequestReview = type({ ) .optional(), commit_id: type.string - .describe( - "Optional SHA of the commit being reviewed. Defaults to latest. Must be the FULL 40-character SHA — abbreviated SHAs are rejected by GitHub with `422 Unprocessable Entity`. The PR-synchronize event payload's `head_sha` is already full-length." - ) + .describe("Optional SHA of the commit being reviewed. Defaults to latest.") .optional(), comments: type({ path: type.string.describe( "The file path to comment on (relative to repo root). Must be a file that appears in the PR diff." ), line: type.number.describe( - "Line number to comment on. For multi-line ranges, this is the end line. Use NEW column from diff format. MUST sit inside a `@@` hunk in the PR diff — anchors on context-only or untouched lines are dropped silently (the rest of the review still posts; dropped entries are reported under `droppedComments` in the response)." + "Line number to comment on. For multi-line ranges, this is the end line. Use NEW column from diff format." ), side: type .enumerated("LEFT", "RIGHT") @@ -347,7 +345,7 @@ export const CreatePullRequestReview = type({ .optional(), start_line: type.number .describe( - "Start line for multi-line comment ranges. Omit for single-line comments. The range [start_line, line] defines which lines a suggestion replaces. BOTH `start_line` AND `line` must sit inside the SAME `@@` hunk — a `start_line` outside the hunk causes the whole comment to be dropped even when `line` is valid. If you need to comment on context just above/below a hunk, shrink the range to a single line that is provably modified." + "Start line for multi-line comment ranges. Omit for single-line comments. The range [start_line, line] defines which lines a suggestion replaces." ) .optional(), }) @@ -363,6 +361,7 @@ export function CreatePullRequestReviewTool(ctx: ToolContext) { name: "create_pull_request_review", description: "Submit a review for an existing pull request. " + + 'Example: `create_pull_request_review({ pull_number: 1234, body: "LGTM", approved: true, comments: [{ path: "src/api.ts", line: 42, body: "nit: rename" }] })`. ' + "Each call creates a permanent, visible review on the PR — NEVER submit test or diagnostic reviews. " + "Reviews with no body AND no comments are silently skipped (nothing to post). " + "IMPORTANT: 95%+ of feedback should be in 'comments' array with file paths and line numbers. " + diff --git a/mcp/reviewComments.ts b/mcp/reviewComments.ts index c642490..7f01067 100644 --- a/mcp/reviewComments.ts +++ b/mcp/reviewComments.ts @@ -602,6 +602,7 @@ export function GetReviewCommentsTool(ctx: ToolContext) { name: "get_review_comments", description: "Get review comments for a pull request review with full thread context. " + + "Example: `get_review_comments({ pull_number: 1234, review_id: 567890 })`. " + "Automatically filters to approved comments when applicable. " + "Returns a TOC and commentsPath pointing to a markdown file with full comment details.", parameters: GetReviewComments, @@ -673,7 +674,8 @@ export function ListPullRequestReviewsTool(ctx: ToolContext) { return tool({ name: "list_pull_request_reviews", description: - "List all reviews for a pull request. Returns all reviews including approvals, request changes, and comments.", + "List all reviews for a pull request. Returns all reviews including approvals, request changes, and comments. " + + "Example: `list_pull_request_reviews({ pull_number: 1234 })`.", parameters: ListPullRequestReviews, execute: execute(async (params) => { const reviews = await ctx.octokit.paginate(ctx.octokit.rest.pulls.listReviews, { diff --git a/mcp/selectMode.ts b/mcp/selectMode.ts index 1c577a9..796f9e0 100644 --- a/mcp/selectMode.ts +++ b/mcp/selectMode.ts @@ -121,7 +121,8 @@ export function SelectModeTool(ctx: ToolContext) { return tool({ name: "select_mode", description: - "Select a mode and receive step-by-step guidance on how to handle the task. Call this to understand the best workflow for the current mode.", + "Select a mode and receive step-by-step guidance on how to handle the task. Call this to understand the best workflow for the current mode. " + + 'Example: `select_mode({ mode: "Review" })` or `select_mode({ mode: "Plan", issue_number: 1234 })`.', parameters: SelectModeParams, execute: execute(async (params) => { if (ctx.toolState.selectedMode) { diff --git a/mcp/shell.ts b/mcp/shell.ts index 2d9f819..56f8ee3 100644 --- a/mcp/shell.ts +++ b/mcp/shell.ts @@ -189,6 +189,8 @@ export function ShellTool(ctx: ToolContext) { name: "shell", description: `Execute shell commands securely. Environment is filtered to remove API keys and secrets. +Example: \`shell({ command: "pnpm test", description: "run the test suite" })\`. + Use this tool to: - Run shell commands (ls, cat, grep, find, etc.) - Execute build tools (npm, pnpm, cargo, make, etc.)