868576a474
- `action/utils/apiKeys.ts`: rewrite the missing-key body as Markdown with linked CTAs (repo secrets / model settings / docs). add `isApiKeyAuthError` + `formatApiKeyErrorSummary` covering both shapes: missing key (#679) and revoked/invalid 401 key (#702). - `action/main.ts`: reclassify in the result-failure branch and the catch block so the PR progress comment surfaces the actionable CTA instead of the raw `Invalid API key · Fix external API key` / numbered-list dump. - `scripts/analyze-logs.ts`: split `failure:user-misconfig` into `:no-key` and `:invalid-key` so both buckets are visible separately and the audit can ignore them as user-correctable. - `.github/workflows/run-audit.yml`: add three explicit prompt rules — cross-customer signal required (≥3 distinct accounts; single-customer concentration is not enough), recovered failures are not actionable, user misconfig is out of scope. closes the loop on #679 / #702 being filed in the first place.
96 lines
3.9 KiB
TypeScript
96 lines
3.9 KiB
TypeScript
import { getModelEnvVars, providers } from "../models.ts";
|
|
import { getApiUrl } from "./apiUrl.ts";
|
|
|
|
const knownApiKeys: Set<string> = new Set(Object.values(providers).flatMap((p) => [...p.envVars]));
|
|
|
|
/** marker prefix on the throw message for the catch-side reclassification path */
|
|
const MISSING_KEY_MARKER = "no API key found";
|
|
|
|
/** Markdown body used for both the thrown error and the formatted PR comment summary. */
|
|
function buildMissingApiKeyError(params: { owner: string; name: string }): string {
|
|
const githubSecretsUrl = `https://github.com/${params.owner}/${params.name}/settings/secrets/actions`;
|
|
const settingsUrl = `${getApiUrl()}/console/${params.owner}/${params.name}`;
|
|
|
|
return [
|
|
`**${MISSING_KEY_MARKER}** — Pullfrog needs at least one LLM provider API key (e.g. \`ANTHROPIC_API_KEY\`, \`OPENAI_API_KEY\`, \`GEMINI_API_KEY\`) configured as a GitHub Actions secret.`,
|
|
"",
|
|
`[Open repo secrets →](${githubSecretsUrl}) · [Configure model →](${settingsUrl}) · [Setup docs →](https://docs.pullfrog.com/keys)`,
|
|
].join("\n");
|
|
}
|
|
|
|
function hasEnvVar(name: string): boolean {
|
|
const value = process.env[name];
|
|
return typeof value === "string" && value.length > 0;
|
|
}
|
|
|
|
/** check if the user has a BYOK key for the given model's provider (does not throw) */
|
|
export function hasProviderKey(model: string): boolean {
|
|
const requiredVars = getModelEnvVars(model);
|
|
if (requiredVars.length === 0) return true;
|
|
return requiredVars.some((v) => hasEnvVar(v));
|
|
}
|
|
|
|
export function validateAgentApiKey(params: {
|
|
agent: { name: string };
|
|
model: string | undefined;
|
|
owner: string;
|
|
name: string;
|
|
}): void {
|
|
// if a specific model is configured, only check that model's required env vars
|
|
if (params.model) {
|
|
const requiredVars = getModelEnvVars(params.model);
|
|
// free models have no required env vars — skip validation entirely
|
|
if (requiredVars.length === 0) return;
|
|
if (requiredVars.some((v) => hasEnvVar(v))) return;
|
|
|
|
throw new Error(buildMissingApiKeyError({ owner: params.owner, name: params.name }));
|
|
}
|
|
|
|
// no model configured — auto-select requires at least one known provider key
|
|
const hasAnyKey = [...knownApiKeys].some((k) => hasEnvVar(k));
|
|
if (!hasAnyKey) {
|
|
throw new Error(buildMissingApiKeyError({ owner: params.owner, name: params.name }));
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Detect agent-runtime auth failures that should be reformatted as an actionable
|
|
* key-fix CTA before being shown to the user. Covers the two shapes we see:
|
|
* - missing key (validateAgentApiKey throw): contains MISSING_KEY_MARKER
|
|
* - revoked / invalid key (Claude CLI 401 surfaced via api_error_status):
|
|
* "Invalid API key · Fix external API key" + similar provider variants
|
|
*/
|
|
export function isApiKeyAuthError(text: string): boolean {
|
|
if (!text) return false;
|
|
return (
|
|
text.includes(MISSING_KEY_MARKER) ||
|
|
/Invalid API key/i.test(text) ||
|
|
/\bUser not found\b/i.test(text) ||
|
|
/\bInvalid authentication\b/i.test(text)
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Friendly Markdown summary for both the missing-key and invalid-key cases.
|
|
* Used in the catch / result-failure paths in `main.ts` to overwrite the raw
|
|
* agent error before it's posted to the PR progress comment.
|
|
*/
|
|
export function formatApiKeyErrorSummary(params: {
|
|
owner: string;
|
|
name: string;
|
|
raw: string;
|
|
}): string {
|
|
if (params.raw.includes(MISSING_KEY_MARKER)) {
|
|
return buildMissingApiKeyError({ owner: params.owner, name: params.name });
|
|
}
|
|
|
|
const githubSecretsUrl = `https://github.com/${params.owner}/${params.name}/settings/secrets/actions`;
|
|
const settingsUrl = `${getApiUrl()}/console/${params.owner}/${params.name}`;
|
|
|
|
return [
|
|
`**Your LLM provider API key was rejected (401).** Rotate the key in your provider dashboard, then update the matching GitHub Actions secret.`,
|
|
"",
|
|
`[Update repo secret →](${githubSecretsUrl}) · [Model settings →](${settingsUrl}) · [Setup docs →](https://docs.pullfrog.com/keys)`,
|
|
].join("\n");
|
|
}
|