Files
shockbot/utils/apiKeys.ts
T
Colin McDonnell 868576a474 audit: format byok auth errors actionably + tighten audit prompt
- `action/utils/apiKeys.ts`: rewrite the missing-key body as Markdown with
  linked CTAs (repo secrets / model settings / docs). add
  `isApiKeyAuthError` + `formatApiKeyErrorSummary` covering both shapes:
  missing key (#679) and revoked/invalid 401 key (#702).
- `action/main.ts`: reclassify in the result-failure branch and the catch
  block so the PR progress comment surfaces the actionable CTA instead of
  the raw `Invalid API key · Fix external API key` / numbered-list dump.
- `scripts/analyze-logs.ts`: split `failure:user-misconfig` into
  `:no-key` and `:invalid-key` so both buckets are visible separately
  and the audit can ignore them as user-correctable.
- `.github/workflows/run-audit.yml`: add three explicit prompt rules —
  cross-customer signal required (≥3 distinct accounts; single-customer
  concentration is not enough), recovered failures are not actionable,
  user misconfig is out of scope. closes the loop on #679 / #702 being
  filed in the first place.
2026-05-13 21:59:47 +00:00

96 lines
3.9 KiB
TypeScript

import { getModelEnvVars, providers } from "../models.ts";
import { getApiUrl } from "./apiUrl.ts";
const knownApiKeys: Set<string> = new Set(Object.values(providers).flatMap((p) => [...p.envVars]));
/** marker prefix on the throw message for the catch-side reclassification path */
const MISSING_KEY_MARKER = "no API key found";
/** Markdown body used for both the thrown error and the formatted PR comment summary. */
function buildMissingApiKeyError(params: { owner: string; name: string }): string {
const githubSecretsUrl = `https://github.com/${params.owner}/${params.name}/settings/secrets/actions`;
const settingsUrl = `${getApiUrl()}/console/${params.owner}/${params.name}`;
return [
`**${MISSING_KEY_MARKER}** — Pullfrog needs at least one LLM provider API key (e.g. \`ANTHROPIC_API_KEY\`, \`OPENAI_API_KEY\`, \`GEMINI_API_KEY\`) configured as a GitHub Actions secret.`,
"",
`[Open repo secrets →](${githubSecretsUrl}) · [Configure model →](${settingsUrl}) · [Setup docs →](https://docs.pullfrog.com/keys)`,
].join("\n");
}
function hasEnvVar(name: string): boolean {
const value = process.env[name];
return typeof value === "string" && value.length > 0;
}
/** check if the user has a BYOK key for the given model's provider (does not throw) */
export function hasProviderKey(model: string): boolean {
const requiredVars = getModelEnvVars(model);
if (requiredVars.length === 0) return true;
return requiredVars.some((v) => hasEnvVar(v));
}
export function validateAgentApiKey(params: {
agent: { name: string };
model: string | undefined;
owner: string;
name: string;
}): void {
// if a specific model is configured, only check that model's required env vars
if (params.model) {
const requiredVars = getModelEnvVars(params.model);
// free models have no required env vars — skip validation entirely
if (requiredVars.length === 0) return;
if (requiredVars.some((v) => hasEnvVar(v))) return;
throw new Error(buildMissingApiKeyError({ owner: params.owner, name: params.name }));
}
// no model configured — auto-select requires at least one known provider key
const hasAnyKey = [...knownApiKeys].some((k) => hasEnvVar(k));
if (!hasAnyKey) {
throw new Error(buildMissingApiKeyError({ owner: params.owner, name: params.name }));
}
}
/**
* Detect agent-runtime auth failures that should be reformatted as an actionable
* key-fix CTA before being shown to the user. Covers the two shapes we see:
* - missing key (validateAgentApiKey throw): contains MISSING_KEY_MARKER
* - revoked / invalid key (Claude CLI 401 surfaced via api_error_status):
* "Invalid API key · Fix external API key" + similar provider variants
*/
export function isApiKeyAuthError(text: string): boolean {
if (!text) return false;
return (
text.includes(MISSING_KEY_MARKER) ||
/Invalid API key/i.test(text) ||
/\bUser not found\b/i.test(text) ||
/\bInvalid authentication\b/i.test(text)
);
}
/**
* Friendly Markdown summary for both the missing-key and invalid-key cases.
* Used in the catch / result-failure paths in `main.ts` to overwrite the raw
* agent error before it's posted to the PR progress comment.
*/
export function formatApiKeyErrorSummary(params: {
owner: string;
name: string;
raw: string;
}): string {
if (params.raw.includes(MISSING_KEY_MARKER)) {
return buildMissingApiKeyError({ owner: params.owner, name: params.name });
}
const githubSecretsUrl = `https://github.com/${params.owner}/${params.name}/settings/secrets/actions`;
const settingsUrl = `${getApiUrl()}/console/${params.owner}/${params.name}`;
return [
`**Your LLM provider API key was rejected (401).** Rotate the key in your provider dashboard, then update the matching GitHub Actions secret.`,
"",
`[Update repo secret →](${githubSecretsUrl}) · [Model settings →](${settingsUrl}) · [Setup docs →](https://docs.pullfrog.com/keys)`,
].join("\n");
}