Files
shockbot/utils/subprocess.ts
T
Colin McDonnell 9f566d20e4 fix proxy key usage tracking and add OSS spend reporting (#500)
* fix proxy key usage tracking and add OSS spend reporting

replace ProxyKey.disabled with finalizedAt to fix a race where keys
were disabled before their usage was synced, causing all HWM values to
be zero. retireKey now fetches final usage from OpenRouter and records
it atomically with optimistic concurrency. syncAccountUsage skips keys
that fail to fetch rather than recording false zeros.

other fixes:
- wrap OpenRouter API calls in retry logic (exponential backoff)
- reconcileStaleWorkflowRuns now retires proxy keys for completed runs
- subprocess activity timeout only tracks stdout (stderr retry loops
  no longer prevent hung agent detection)
- add oss-spend script (single bulk fetch from OpenRouter) and
  backfill-proxy-key-usage script

Made-with: Cursor

* address review: move isActiveKey check inside transaction, remove redundant guard

Made-with: Cursor
2026-03-31 02:18:08 +00:00

234 lines
6.8 KiB
TypeScript

import { type ChildProcess, spawn as nodeSpawn } from "node:child_process";
import { performance } from "node:perf_hooks";
import { DEFAULT_ACTIVITY_CHECK_INTERVAL_MS, DEFAULT_ACTIVITY_TIMEOUT_MS } from "./activity.ts";
import { log } from "./cli.ts";
import { onExitSignal } from "./exitHandler.ts";
export type TrackChildOptions = {
child: ChildProcess;
// if true, kill the entire process group (requires detached spawn)
killGroup?: boolean;
};
// track all spawned child processes for cleanup on Ctrl+C
const activeChildren = new Map<ChildProcess, boolean>();
// signal handler override (used by test runner for graceful shutdown)
export type SignalHandler = (signal: NodeJS.Signals) => void;
let externalSignalHandler: SignalHandler | null = null;
// track a child process for cleanup on Ctrl+C
export function trackChild(options: TrackChildOptions): void {
// the signal handler cleans up all tracked children
// so we only have to install it once some child gets tracked
installSignalHandler();
activeChildren.set(options.child, options.killGroup ?? false);
}
// untrack a child process
export function untrackChild(child: ChildProcess): void {
activeChildren.delete(child);
}
// allow callers to override default signal handling
export function setSignalHandler(handler: SignalHandler | null): void {
externalSignalHandler = handler;
}
// kill all tracked children without exiting
export function killTrackedChildren() {
for (const entry of activeChildren) {
const child = entry[0];
const killGroup = entry[1];
if (killGroup && child.pid) {
try {
process.kill(-child.pid, "SIGKILL");
continue;
} catch {
// fall through to direct kill
}
}
child.kill("SIGKILL");
}
}
// install signal handlers once (call early in process lifecycle)
let handlersInstalled = false;
function installSignalHandler(): void {
if (handlersInstalled) return;
handlersInstalled = true;
onExitSignal((signal) => {
if (externalSignalHandler) {
externalSignalHandler(signal);
return;
}
const count = activeChildren.size;
if (count > 0) {
log.info(`» received ${signal}, killing ${count} subprocess(es)...`);
}
killTrackedChildren();
});
}
export interface SpawnOptions {
cmd: string;
args: string[];
env?: NodeJS.ProcessEnv;
input?: string;
timeout?: number;
// activity timeout: kill process if no stdout for this many ms (default: 30s, 0 to disable).
// only stdout resets the timer — stderr (e.g. provider error retries) does not count as progress.
activityTimeout?: number;
cwd?: string;
stdio?: ("pipe" | "ignore" | "inherit")[];
onStdout?: (chunk: string) => void;
onStderr?: (chunk: string) => void;
}
export interface SpawnResult {
stdout: string;
stderr: string;
exitCode: number;
durationMs: number;
}
/**
* Spawn a subprocess with streaming callbacks and buffered results
*/
export async function spawn(options: SpawnOptions): Promise<SpawnResult> {
const activityTimeoutMs = options.activityTimeout ?? DEFAULT_ACTIVITY_TIMEOUT_MS;
installSignalHandler();
const startTime = performance.now();
let stdoutBuffer = "";
let stderrBuffer = "";
return new Promise((resolve, reject) => {
// security: caller must provide complete env object, not merged with process.env
const child = nodeSpawn(options.cmd, options.args, {
env: options.env || {
PATH: process.env.PATH || "",
HOME: process.env.HOME || "",
},
stdio: options.stdio || ["pipe", "pipe", "pipe"],
cwd: options.cwd || process.cwd(),
});
// track child for cleanup on Ctrl+C
trackChild({ child });
let timeoutId: NodeJS.Timeout | undefined;
let activityCheckIntervalId: NodeJS.Timeout | undefined;
let isTimedOut = false;
let isActivityTimedOut = false;
let lastActivityTime = performance.now();
// overall timeout
if (options.timeout) {
timeoutId = setTimeout(() => {
isTimedOut = true;
child.kill("SIGTERM");
setTimeout(() => {
if (!child.killed) {
child.kill("SIGKILL");
}
}, 5000);
}, options.timeout);
}
// activity timeout: kill if no output for too long
if (activityTimeoutMs > 0) {
log.debug(
`spawn activity timer: pid=${child.pid} cmd=${options.cmd} timeout=${activityTimeoutMs}ms`
);
activityCheckIntervalId = setInterval(() => {
const idleMs = performance.now() - lastActivityTime;
log.debug(
`spawn activity check: pid=${child.pid} idle=${Math.round(idleMs)}ms / ${activityTimeoutMs}ms`
);
if (idleMs > activityTimeoutMs) {
isActivityTimedOut = true;
const idleSec = Math.round(idleMs / 1000);
log.info(
`no output for ${idleSec}s from pid=${child.pid} (${options.cmd}), killing process`
);
child.kill("SIGKILL");
clearInterval(activityCheckIntervalId);
}
}, DEFAULT_ACTIVITY_CHECK_INTERVAL_MS);
}
function updateActivity(): void {
lastActivityTime = performance.now();
}
if (child.stdout) {
child.stdout.on("data", (data: Buffer) => {
updateActivity();
const chunk = data.toString();
stdoutBuffer += chunk;
options.onStdout?.(chunk);
});
}
if (child.stderr) {
child.stderr.on("data", (data: Buffer) => {
const chunk = data.toString();
stderrBuffer += chunk;
options.onStderr?.(chunk);
});
}
child.on("close", (exitCode) => {
const durationMs = performance.now() - startTime;
untrackChild(child);
if (timeoutId) clearTimeout(timeoutId);
if (activityCheckIntervalId) clearInterval(activityCheckIntervalId);
if (isTimedOut) {
reject(new Error(`process timed out after ${options.timeout}ms`));
return;
}
if (isActivityTimedOut) {
const idleSec = Math.round((performance.now() - lastActivityTime) / 1000);
reject(new Error(`activity timeout: no output for ${idleSec}s`));
return;
}
resolve({
stdout: stdoutBuffer,
stderr: stderrBuffer,
exitCode: exitCode || 0,
durationMs,
});
});
child.on("error", (error) => {
const durationMs = performance.now() - startTime;
untrackChild(child);
if (timeoutId) clearTimeout(timeoutId);
if (activityCheckIntervalId) clearInterval(activityCheckIntervalId);
// log spawn errors for debugging
console.error(`[spawn] process spawn error: ${error.message}`);
resolve({
stdout: stdoutBuffer,
stderr: stderrBuffer,
exitCode: 1,
durationMs,
});
});
if (options.input && child.stdin && options.stdio?.[0] !== "ignore") {
child.stdin.write(options.input);
child.stdin.end();
}
});
}