Colin McDonnell
b8ac42e875
* mcp: embed example calls in top-level tool descriptions
agents (esp. claude sonnet) hallucinate param names from training-data
priors — `pr_number` instead of `pull_number`, `summary` instead of
`body`, full subcommand strings jammed into `git({command})` like it
were `shell({command})`. each error burns a tool round-trip plus a
follow-up ToolSearch, ~40+ events / 24h, no observable recovery cost
to us but visible to users in agent logs.
cheapest fix: add a sample formatted function call to every affected
tool's top-level description. example anchors are more reliable than
schema descriptions alone because the model treats descriptions as
narrative but call examples as canonical structure. for `git` and
`shell` (whose `command` fields collide), include explicit
counter-examples disambiguating which tool owns which shape.
no schema aliases / coercion yet — try the cheap thing first; if the
next audit window still shows the same hallucination rate, layer
aliases on top per #585's recommendation.
closes #585, closes #701
* mcp: drop negative anchors from tool descriptions
negation is a footgun in tool descriptions — telling the model "NOT
pr_number" makes pr_number more salient, not less. let the positive
example carry the schema and trust the model to read it.
removes:
- "the parameter is pull_number (a number), NOT pr_number" and
similar across checkout_pr, get_pull_request, list_pull_request_reviews,
get_review_comments, create_pull_request_review
- "NOT summary, message, or content" on report_progress
- "WRONG: git({ command: 'log --oneline' })" counter-example on git
- redundant param-type restatements after the example (e.g. "depth is a
number, not a string" on git_fetch, "description is required" on shell)
keeps a single positive example per tool. for tools with multiple call
shapes (git, git_fetch, push_branch), two positive examples instead of
one + a counter-example.
76 lines
2.3 KiB
TypeScript
76 lines
2.3 KiB
TypeScript
import { type } from "arktype";
|
|
import type { ToolContext } from "./server.ts";
|
|
import { execute, tool } from "./shared.ts";
|
|
|
|
const CLOSING_ISSUES_QUERY = `
|
|
query($owner: String!, $repo: String!, $number: Int!) {
|
|
repository(owner: $owner, name: $repo) {
|
|
pullRequest(number: $number) {
|
|
closingIssuesReferences(first: 10) {
|
|
nodes { number title }
|
|
}
|
|
}
|
|
}
|
|
}
|
|
`;
|
|
|
|
type ClosingIssuesResponse = {
|
|
repository: {
|
|
pullRequest: {
|
|
closingIssuesReferences: { nodes: Array<{ number: number; title: string }> };
|
|
};
|
|
};
|
|
};
|
|
|
|
export const PullRequestInfo = type({
|
|
pull_number: type.number.describe("The pull request number to fetch"),
|
|
});
|
|
|
|
export function PullRequestInfoTool(ctx: ToolContext) {
|
|
return tool({
|
|
name: "get_pull_request",
|
|
description:
|
|
"Retrieve PR metadata (title, body, state, branches, author, labels, linked issues). " +
|
|
"Example: `get_pull_request({ pull_number: 1234 })`. " +
|
|
"To checkout a PR branch locally, use checkout_pr instead.",
|
|
parameters: PullRequestInfo,
|
|
execute: execute(async ({ pull_number }) => {
|
|
// fetch REST and GraphQL in parallel
|
|
const [restResponse, graphqlResponse] = await Promise.all([
|
|
ctx.octokit.rest.pulls.get({
|
|
owner: ctx.repo.owner,
|
|
repo: ctx.repo.name,
|
|
pull_number,
|
|
}),
|
|
ctx.octokit.graphql<ClosingIssuesResponse>(CLOSING_ISSUES_QUERY, {
|
|
owner: ctx.repo.owner,
|
|
repo: ctx.repo.name,
|
|
number: pull_number,
|
|
}),
|
|
]);
|
|
|
|
const data = restResponse.data;
|
|
const isFork = data.head.repo?.full_name !== data.base.repo.full_name;
|
|
const closingIssues = graphqlResponse.repository.pullRequest.closingIssuesReferences.nodes;
|
|
|
|
return {
|
|
number: data.number,
|
|
url: data.html_url,
|
|
title: data.title,
|
|
body: data.body,
|
|
state: data.state,
|
|
draft: data.draft,
|
|
merged: data.merged,
|
|
maintainerCanModify: data.maintainer_can_modify,
|
|
base: data.base.ref,
|
|
head: data.head.ref,
|
|
isFork,
|
|
author: data.user?.login,
|
|
assignees: data.assignees?.map((a) => a.login),
|
|
labels: data.labels.map((l) => l.name),
|
|
closingIssues: closingIssues.map((i) => ({ number: i.number, title: i.title })),
|
|
};
|
|
}),
|
|
});
|
|
}
|