Colin McDonnell
b8ac42e875
* mcp: embed example calls in top-level tool descriptions
agents (esp. claude sonnet) hallucinate param names from training-data
priors — `pr_number` instead of `pull_number`, `summary` instead of
`body`, full subcommand strings jammed into `git({command})` like it
were `shell({command})`. each error burns a tool round-trip plus a
follow-up ToolSearch, ~40+ events / 24h, no observable recovery cost
to us but visible to users in agent logs.
cheapest fix: add a sample formatted function call to every affected
tool's top-level description. example anchors are more reliable than
schema descriptions alone because the model treats descriptions as
narrative but call examples as canonical structure. for `git` and
`shell` (whose `command` fields collide), include explicit
counter-examples disambiguating which tool owns which shape.
no schema aliases / coercion yet — try the cheap thing first; if the
next audit window still shows the same hallucination rate, layer
aliases on top per #585's recommendation.
closes #585, closes #701
* mcp: drop negative anchors from tool descriptions
negation is a footgun in tool descriptions — telling the model "NOT
pr_number" makes pr_number more salient, not less. let the positive
example carry the schema and trust the model to read it.
removes:
- "the parameter is pull_number (a number), NOT pr_number" and
similar across checkout_pr, get_pull_request, list_pull_request_reviews,
get_review_comments, create_pull_request_review
- "NOT summary, message, or content" on report_progress
- "WRONG: git({ command: 'log --oneline' })" counter-example on git
- redundant param-type restatements after the example (e.g. "depth is a
number, not a string" on git_fetch, "description is required" on shell)
keeps a single positive example per tool. for tools with multiple call
shapes (git, git_fetch, push_branch), two positive examples instead of
one + a counter-example.
62 lines
2.2 KiB
TypeScript
62 lines
2.2 KiB
TypeScript
import { writeFileSync } from "node:fs";
|
|
import { join } from "node:path";
|
|
import { type } from "arktype";
|
|
import { log } from "../utils/cli.ts";
|
|
import { formatFilesWithLineNumbers } from "./checkout.ts";
|
|
import type { ToolContext } from "./server.ts";
|
|
import { execute, tool } from "./shared.ts";
|
|
|
|
export const CommitInfo = type({
|
|
sha: type.string.describe("the commit SHA (full or abbreviated) to fetch"),
|
|
});
|
|
|
|
export function CommitInfoTool(ctx: ToolContext) {
|
|
return tool({
|
|
name: "get_commit_info",
|
|
description:
|
|
"Retrieve commit metadata and diff via GitHub API. Use this instead of git show for reviewing commits - " +
|
|
"it works with shallow clones and shows the actual changes in the commit. Returns diffPath pointing to formatted diff file. " +
|
|
'Example: `get_commit_info({ sha: "2a6ab5d06ba64dd987bd88c4403287" })`.',
|
|
parameters: CommitInfo,
|
|
execute: execute(async ({ sha }) => {
|
|
const response = await ctx.octokit.rest.repos.getCommit({
|
|
owner: ctx.repo.owner,
|
|
repo: ctx.repo.name,
|
|
ref: sha,
|
|
});
|
|
|
|
const data = response.data;
|
|
const files = data.files ?? [];
|
|
|
|
// format diff with line numbers and write to file
|
|
const formatResult = formatFilesWithLineNumbers(files);
|
|
const tempDir = process.env.PULLFROG_TEMP_DIR;
|
|
if (!tempDir) {
|
|
throw new Error(
|
|
"PULLFROG_TEMP_DIR not set - get_commit_info must run in pullfrog action context"
|
|
);
|
|
}
|
|
const diffFile = join(tempDir, `commit-${sha.slice(0, 7)}.diff`);
|
|
writeFileSync(diffFile, formatResult.content);
|
|
log.debug(`wrote commit diff to ${diffFile} (${formatResult.content.length} bytes)`);
|
|
|
|
return {
|
|
sha: data.sha,
|
|
message: data.commit.message,
|
|
author: data.author?.login ?? null,
|
|
committer: data.committer?.login ?? null,
|
|
date: data.commit.author?.date ?? data.commit.committer?.date ?? "",
|
|
url: data.html_url,
|
|
parents: data.parents.map((p) => p.sha),
|
|
stats: {
|
|
additions: data.stats?.additions ?? 0,
|
|
deletions: data.stats?.deletions ?? 0,
|
|
total: data.stats?.total ?? 0,
|
|
},
|
|
fileCount: files.length,
|
|
diffFile,
|
|
};
|
|
}),
|
|
});
|
|
}
|