audit: format byok auth errors actionably + tighten audit prompt

- `action/utils/apiKeys.ts`: rewrite the missing-key body as Markdown with
  linked CTAs (repo secrets / model settings / docs). add
  `isApiKeyAuthError` + `formatApiKeyErrorSummary` covering both shapes:
  missing key (#679) and revoked/invalid 401 key (#702).
- `action/main.ts`: reclassify in the result-failure branch and the catch
  block so the PR progress comment surfaces the actionable CTA instead of
  the raw `Invalid API key · Fix external API key` / numbered-list dump.
- `scripts/analyze-logs.ts`: split `failure:user-misconfig` into
  `:no-key` and `:invalid-key` so both buckets are visible separately
  and the audit can ignore them as user-correctable.
- `.github/workflows/run-audit.yml`: add three explicit prompt rules —
  cross-customer signal required (≥3 distinct accounts; single-customer
  concentration is not enough), recovered failures are not actionable,
  user misconfig is out of scope. closes the loop on #679 / #702 being
  filed in the first place.
This commit is contained in:
Colin McDonnell
2026-05-13 21:59:47 +00:00
committed by pullfrog[bot]
parent b2b1e588e7
commit 868576a474
2 changed files with 77 additions and 25 deletions
+25 -7
View File
@@ -17,7 +17,11 @@ import {
} from "./utils/activity.ts";
import { resolveAgent, resolveModel } from "./utils/agent.ts";
import { apiFetch } from "./utils/apiFetch.ts";
import { validateAgentApiKey } from "./utils/apiKeys.ts";
import {
formatApiKeyErrorSummary,
isApiKeyAuthError,
validateAgentApiKey,
} from "./utils/apiKeys.ts";
import { isLocalApiUrl } from "./utils/apiUrl.ts";
import { resolveBody } from "./utils/body.ts";
import { formatUsageSummary, log, writeSummary } from "./utils/cli.ts";
@@ -1035,10 +1039,15 @@ export async function main(): Promise<MainResult> {
// the comment is still around to update; reportErrorToComment sets
// wasUpdated=true and the !result.success guard skips deletion.
if (!result.success && toolContext && toolState.progressComment) {
await reportErrorToComment({
toolState,
error: result.error || "agent run failed",
}).catch((error) => {
const rawError = result.error || "agent run failed";
const errorBody = isApiKeyAuthError(rawError)
? formatApiKeyErrorSummary({
owner: runContext.repo.owner,
name: runContext.repo.name,
raw: rawError,
})
: rawError;
await reportErrorToComment({ toolState, error: errorBody }).catch((error) => {
log.debug(`failure error report failed: ${error}`);
});
}
@@ -1114,11 +1123,20 @@ export async function main(): Promise<MainResult> {
? new BillingError(errorMessage, { code: "router_keylimit_exhausted" })
: null;
const apiKeyErrorSummary =
!billingError && isApiKeyAuthError(errorMessage)
? formatApiKeyErrorSummary({
owner: runContext.repo.owner,
name: runContext.repo.name,
raw: errorMessage,
})
: null;
// best-effort summary — write the error so it's visible in the Actions summary tab
try {
const errorSummary = billingError
? formatBillingErrorSummary(billingError, runContext.repo.owner)
: `### ❌ Pullfrog failed\n\n\`\`\`\n${errorMessage}\n\`\`\``;
: (apiKeyErrorSummary ?? `### ❌ Pullfrog failed\n\n\`\`\`\n${errorMessage}\n\`\`\``);
const usageSummary = formatUsageSummary(toolState.usageEntries);
const parts = [errorSummary, toolState.lastProgressBody, usageSummary].filter(Boolean);
await writeSummary(parts.join("\n\n"));
@@ -1127,7 +1145,7 @@ export async function main(): Promise<MainResult> {
try {
const commentBody = billingError
? formatBillingErrorSummary(billingError, runContext.repo.owner)
: errorMessage;
: (apiKeyErrorSummary ?? errorMessage);
await reportErrorToComment({ toolState, error: commentBody });
} catch {
// error reporting failed, but don't let it mask the original error