audit: format byok auth errors actionably + tighten audit prompt
- `action/utils/apiKeys.ts`: rewrite the missing-key body as Markdown with linked CTAs (repo secrets / model settings / docs). add `isApiKeyAuthError` + `formatApiKeyErrorSummary` covering both shapes: missing key (#679) and revoked/invalid 401 key (#702). - `action/main.ts`: reclassify in the result-failure branch and the catch block so the PR progress comment surfaces the actionable CTA instead of the raw `Invalid API key · Fix external API key` / numbered-list dump. - `scripts/analyze-logs.ts`: split `failure:user-misconfig` into `:no-key` and `:invalid-key` so both buckets are visible separately and the audit can ignore them as user-correctable. - `.github/workflows/run-audit.yml`: add three explicit prompt rules — cross-customer signal required (≥3 distinct accounts; single-customer concentration is not enough), recovered failures are not actionable, user misconfig is out of scope. closes the loop on #679 / #702 being filed in the first place.
This commit is contained in:
committed by
pullfrog[bot]
parent
b2b1e588e7
commit
868576a474
+52
-18
@@ -3,26 +3,19 @@ import { getApiUrl } from "./apiUrl.ts";
|
||||
|
||||
const knownApiKeys: Set<string> = new Set(Object.values(providers).flatMap((p) => [...p.envVars]));
|
||||
|
||||
/** marker prefix on the throw message for the catch-side reclassification path */
|
||||
const MISSING_KEY_MARKER = "no API key found";
|
||||
|
||||
/** Markdown body used for both the thrown error and the formatted PR comment summary. */
|
||||
function buildMissingApiKeyError(params: { owner: string; name: string }): string {
|
||||
const apiUrl = getApiUrl();
|
||||
const settingsUrl = `${apiUrl}/console/${params.owner}/${params.name}`;
|
||||
const githubSecretsUrl = `https://github.com/${params.owner}/${params.name}/settings/secrets/actions`;
|
||||
const settingsUrl = `${getApiUrl()}/console/${params.owner}/${params.name}`;
|
||||
|
||||
const githubRepoUrl = `https://github.com/${params.owner}/${params.name}`;
|
||||
const githubSecretsUrl = `${githubRepoUrl}/settings/secrets/actions`;
|
||||
|
||||
return `no API key found. Pullfrog requires at least one LLM provider API key.
|
||||
|
||||
to fix this, add the required secret to your GitHub repository:
|
||||
|
||||
1. go to: ${githubSecretsUrl}
|
||||
2. click "New repository secret"
|
||||
3. set the name to your provider's key (e.g., \`ANTHROPIC_API_KEY\`, \`OPENAI_API_KEY\`, \`GEMINI_API_KEY\`)
|
||||
4. set the value to your API key
|
||||
5. click "Add secret"
|
||||
|
||||
configure your model at ${settingsUrl}
|
||||
|
||||
for full setup instructions, see https://docs.pullfrog.com/keys`;
|
||||
return [
|
||||
`**${MISSING_KEY_MARKER}** — Pullfrog needs at least one LLM provider API key (e.g. \`ANTHROPIC_API_KEY\`, \`OPENAI_API_KEY\`, \`GEMINI_API_KEY\`) configured as a GitHub Actions secret.`,
|
||||
"",
|
||||
`[Open repo secrets →](${githubSecretsUrl}) · [Configure model →](${settingsUrl}) · [Setup docs →](https://docs.pullfrog.com/keys)`,
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function hasEnvVar(name: string): boolean {
|
||||
@@ -59,3 +52,44 @@ export function validateAgentApiKey(params: {
|
||||
throw new Error(buildMissingApiKeyError({ owner: params.owner, name: params.name }));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect agent-runtime auth failures that should be reformatted as an actionable
|
||||
* key-fix CTA before being shown to the user. Covers the two shapes we see:
|
||||
* - missing key (validateAgentApiKey throw): contains MISSING_KEY_MARKER
|
||||
* - revoked / invalid key (Claude CLI 401 surfaced via api_error_status):
|
||||
* "Invalid API key · Fix external API key" + similar provider variants
|
||||
*/
|
||||
export function isApiKeyAuthError(text: string): boolean {
|
||||
if (!text) return false;
|
||||
return (
|
||||
text.includes(MISSING_KEY_MARKER) ||
|
||||
/Invalid API key/i.test(text) ||
|
||||
/\bUser not found\b/i.test(text) ||
|
||||
/\bInvalid authentication\b/i.test(text)
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Friendly Markdown summary for both the missing-key and invalid-key cases.
|
||||
* Used in the catch / result-failure paths in `main.ts` to overwrite the raw
|
||||
* agent error before it's posted to the PR progress comment.
|
||||
*/
|
||||
export function formatApiKeyErrorSummary(params: {
|
||||
owner: string;
|
||||
name: string;
|
||||
raw: string;
|
||||
}): string {
|
||||
if (params.raw.includes(MISSING_KEY_MARKER)) {
|
||||
return buildMissingApiKeyError({ owner: params.owner, name: params.name });
|
||||
}
|
||||
|
||||
const githubSecretsUrl = `https://github.com/${params.owner}/${params.name}/settings/secrets/actions`;
|
||||
const settingsUrl = `${getApiUrl()}/console/${params.owner}/${params.name}`;
|
||||
|
||||
return [
|
||||
`**Your LLM provider API key was rejected (401).** Rotate the key in your provider dashboard, then update the matching GitHub Actions secret.`,
|
||||
"",
|
||||
`[Update repo secret →](${githubSecretsUrl}) · [Model settings →](${settingsUrl}) · [Setup docs →](https://docs.pullfrog.com/keys)`,
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user