import { type ChildProcess, spawn as nodeSpawn } from "node:child_process"; import { performance } from "node:perf_hooks"; import { DEFAULT_ACTIVITY_CHECK_INTERVAL_MS, DEFAULT_ACTIVITY_TIMEOUT_MS } from "./activity.ts"; import { log } from "./cli.ts"; import { onExitSignal } from "./exitHandler.ts"; export type TrackChildOptions = { child: ChildProcess; // if true, kill the entire process group (requires detached spawn) killGroup?: boolean; }; // sentinel codes for timeout rejections — callers (e.g. lifecycle.ts) use // these to distinguish timeouts from other errors without string-matching // on the error message, which is fragile to rewording. export const SPAWN_TIMEOUT_CODE = "E_SPAWN_TIMEOUT"; export const SPAWN_ACTIVITY_TIMEOUT_CODE = "E_SPAWN_ACTIVITY_TIMEOUT"; export class SpawnTimeoutError extends Error { readonly code: typeof SPAWN_TIMEOUT_CODE | typeof SPAWN_ACTIVITY_TIMEOUT_CODE; constructor( message: string, code: typeof SPAWN_TIMEOUT_CODE | typeof SPAWN_ACTIVITY_TIMEOUT_CODE ) { super(message); this.name = "SpawnTimeoutError"; this.code = code; } } // track all spawned child processes for cleanup on Ctrl+C const activeChildren = new Map(); // signal handler override (used by test runner for graceful shutdown) export type SignalHandler = (signal: NodeJS.Signals) => void; let externalSignalHandler: SignalHandler | null = null; // track a child process for cleanup on Ctrl+C export function trackChild(options: TrackChildOptions): void { // the signal handler cleans up all tracked children // so we only have to install it once some child gets tracked installSignalHandler(); activeChildren.set(options.child, options.killGroup ?? false); } // untrack a child process export function untrackChild(child: ChildProcess): void { activeChildren.delete(child); } // allow callers to override default signal handling export function setSignalHandler(handler: SignalHandler | null): void { externalSignalHandler = handler; } // kill all tracked children without exiting export function killTrackedChildren() { for (const entry of activeChildren) { const child = entry[0]; const killGroup = entry[1]; if (killGroup && child.pid) { try { process.kill(-child.pid, "SIGKILL"); continue; } catch { // fall through to direct kill } } child.kill("SIGKILL"); } } // install signal handlers once (call early in process lifecycle) let handlersInstalled = false; function installSignalHandler(): void { if (handlersInstalled) return; handlersInstalled = true; onExitSignal((signal) => { if (externalSignalHandler) { externalSignalHandler(signal); return; } const count = activeChildren.size; if (count > 0) { log.info(`» received ${signal}, killing ${count} subprocess(es)...`); } killTrackedChildren(); }); } /** * Controls what the wrapper retains in memory across the child's lifetime * for the post-hoc `SpawnResult.stdout` / `SpawnResult.stderr` snapshots. * * Streaming callbacks (`onStdout` / `onStderr`) fire regardless — `retain` * only governs the buffered snapshot returned in `SpawnResult`. * * - `"tail"` (default): keep the last `maxRetainedBytes` UTF-16 code units * of each stream. Once the cap is exceeded, oldest bytes are sliced off * and the result is prefixed with a `... [N MiB truncated] ...` sentinel. * Right default for short-lived commands whose failure mode is in their * final output (git errors, install failures, hook scripts). * - `"none"`: skip the buffer entirely. `SpawnResult.stdout` / `.stderr` * are empty strings. Use this for long-lived streaming agents that already * drain via `onStdout` / `onStderr` and never read the buffered snapshot. * * Default cap is 8 MiB — well below V8's ~1 GiB `kMaxLength` so `+= chunk` * can never throw `RangeError: Invalid string length`. */ export type RetainMode = "tail" | "none"; export const DEFAULT_MAX_RETAINED_BYTES = 8 * 1024 * 1024; export interface SpawnOptions { cmd: string; args: string[]; env?: NodeJS.ProcessEnv; input?: string; timeout?: number; // activity timeout: kill process if no stdout for this many ms (default: 30s, 0 to disable). // only stdout resets the timer — stderr (e.g. provider error retries) does not count as progress. activityTimeout?: number; // fired synchronously when the activity timeout kills the process. used by // callers (main.ts) to tear down shared resources like the MCP HTTP server // so that lingering SSE reconnects don't keep the outer activity timer // alive after the subprocess is already dead. onActivityTimeout?: (() => void) | undefined; // optional pause predicate consulted on every activity check. when true, // the spawn watchdog (a) skips its kill decision, (b) advances // `lastActivityTime` so a stale baseline can't fire on resume. used by // agent harnesses (claude.ts / opencode.ts) to suspend the watchdog // across long synchronous MCP `tools/call` round-trips that the child's // stdout pipe can't see (issue #760). bounded externally by // `MAX_TOOL_CALL_SUSPENSION_MS` plus the outer agent timeout. isPausedExternally?: () => boolean; cwd?: string; stdio?: ("pipe" | "ignore" | "inherit")[]; onStdout?: (chunk: string) => void; onStderr?: (chunk: string) => void; // when true, spawn the child detached (its own process group) and route all // kill paths (timeout, activity timeout, ctrl-c) through `process.kill(-pid, ...)` // so signals reach grandchildren too. critical for binaries that fork through // a shim (e.g. node_modules/opencode-ai/bin/opencode is a Node shim that // spawnSync's the native binary; without killGroup, SIGKILL only hits the // shim and the native binary is reparented to PID 1, holds our stdout pipe // open, keeps emitting NDJSON, and `child.on("close")` never fires — // producing zombie runs that hang until the GitHub Actions job timeout). killGroup?: boolean; retain?: RetainMode; maxRetainedBytes?: number; } /** * Bounded string accumulator that keeps the tail of appended chunks. * Once the cap is exceeded, oldest bytes are sliced off and `toString()` * prefixes the survivors with a sentinel describing the elided byte count. * * Exported because long-lived agent runtimes (opencode, claude) also * accumulate per-run narration strings independently of the spawn wrapper * and need the same protection against V8's `kMaxLength`. */ export class TailBuffer { // explicit field declarations rather than constructor parameter properties: // node's strip-only TS loader (used by action/test/run.ts in CI) rejects // `constructor(private readonly cap: number)` with ERR_UNSUPPORTED_TYPESCRIPT_SYNTAX. private readonly cap: number; private buffer = ""; private truncatedBytes = 0; constructor(cap: number) { this.cap = cap; } append(chunk: string): void { if (this.cap <= 0) return; this.buffer += chunk; if (this.buffer.length > this.cap) { const drop = this.buffer.length - this.cap; this.truncatedBytes += drop; this.buffer = this.buffer.slice(drop); } } toString(): string { if (this.truncatedBytes === 0) return this.buffer; const mib = (this.truncatedBytes / 1024 / 1024).toFixed(1); return `... [${mib} MiB truncated by retain:tail cap] ...\n${this.buffer}`; } } export interface SpawnResult { stdout: string; stderr: string; exitCode: number; durationMs: number; } /** * Spawn a subprocess with streaming callbacks and buffered results */ export async function spawn(options: SpawnOptions): Promise { const activityTimeoutMs = options.activityTimeout ?? DEFAULT_ACTIVITY_TIMEOUT_MS; installSignalHandler(); const startTime = performance.now(); // capped accumulators — unbounded `+= chunk` previously crashed the wrapper // with `RangeError: Invalid string length` once V8's ~1 GiB kMaxLength was // breached on long-lived agent subprocesses (e.g. multi-lens opencode // Reviews on large monorepos). retain:"none" skips the buffer entirely // for callers that already drain via onStdout/onStderr. const retain: RetainMode = options.retain ?? "tail"; const cap = options.maxRetainedBytes ?? DEFAULT_MAX_RETAINED_BYTES; const stdoutBuffer = retain === "none" ? null : new TailBuffer(cap); const stderrBuffer = retain === "none" ? null : new TailBuffer(cap); const killGroup = options.killGroup ?? false; return new Promise((resolve, reject) => { // security: caller must provide complete env object, not merged with process.env const child = nodeSpawn(options.cmd, options.args, { env: options.env || { PATH: process.env.PATH || "", HOME: process.env.HOME || "", }, stdio: options.stdio || ["pipe", "pipe", "pipe"], cwd: options.cwd || process.cwd(), detached: killGroup, }); // sends `signal` to the entire process group when killGroup is set, so // grandchildren (e.g. the native opencode binary spawned by the // opencode-ai Node shim) die with the parent. falls back to a direct // child kill if the process-group send fails (common when the child // already exited or was never made a process group leader). const killSelf = (signal: NodeJS.Signals): void => { if (killGroup && child.pid) { try { process.kill(-child.pid, signal); return; } catch { // fall through to direct kill } } child.kill(signal); }; // track child for cleanup on Ctrl+C trackChild({ child, killGroup }); let timeoutId: NodeJS.Timeout | undefined; let sigkillEscalatorId: NodeJS.Timeout | undefined; let activityCheckIntervalId: NodeJS.Timeout | undefined; let isTimedOut = false; let isActivityTimedOut = false; let lastActivityTime = performance.now(); // idle-ms snapshot taken at the moment the activity timer decides to kill. // we reuse it when composing the SpawnTimeoutError so a final stdout chunk // that races with `close` (and resets lastActivityTime via updateActivity) // can't make the error message contradict the "no output for Ns" log line. let killedAtIdleMs: number | undefined; // overall timeout if (options.timeout) { timeoutId = setTimeout(() => { isTimedOut = true; killSelf("SIGTERM"); // track the escalator so a graceful SIGTERM response (close fires // before the 5s elapses) can clear it. without capture, this timer // was orphaned in the event loop and kept node alive for up to 5s // past a timed-out subprocess's clean exit. sigkillEscalatorId = setTimeout(() => { if (!child.killed) { killSelf("SIGKILL"); } }, 5000); }, options.timeout); } // activity timeout: kill if no output for too long if (activityTimeoutMs > 0) { log.debug( `spawn activity timer: pid=${child.pid} cmd=${options.cmd} timeout=${activityTimeoutMs}ms` ); activityCheckIntervalId = setInterval(() => { if (options.isPausedExternally?.()) { // reset the baseline so a clean resume can't immediately fire on // the pre-pause idle window. lastActivityTime = performance.now(); log.debug(`spawn activity check: pid=${child.pid} paused externally`); return; } const idleMs = performance.now() - lastActivityTime; log.debug( `spawn activity check: pid=${child.pid} idle=${Math.round(idleMs)}ms / ${activityTimeoutMs}ms` ); if (idleMs > activityTimeoutMs) { isActivityTimedOut = true; killedAtIdleMs = idleMs; const idleSec = Math.round(idleMs / 1000); log.info( `no output for ${idleSec}s from pid=${child.pid} (${options.cmd}), killing process${killGroup ? " group" : ""}` ); killSelf("SIGKILL"); clearInterval(activityCheckIntervalId); try { options.onActivityTimeout?.(); } catch (err) { log.debug( `spawn onActivityTimeout handler threw: ${err instanceof Error ? err.message : String(err)}` ); } } }, DEFAULT_ACTIVITY_CHECK_INTERVAL_MS); } function updateActivity(): void { lastActivityTime = performance.now(); } // wrap handlers in try/catch as defense in depth for synchronous throws // inside the listener body. the historical `+= chunk` RangeError was such // a throw — synchronous and fatal under node's default uncaught-exception // policy. with the TailBuffer cap in place the wrapper-side `append` can // no longer throw, but the catch keeps protecting against any future // synchronous regression in this path. // // note: this does NOT catch rejections from async user callbacks — // `options.onStdout?.(chunk)` returns a Promise in the agent callers // (claude.ts, opencode.ts) and a throw inside an async callback surfaces // as an unhandled Promise rejection, not a synchronous exception. agent // callers handle their own NDJSON-parse failures internally; the // synchronous protection here is what matters for the RangeError class // of bugs (issue #680). if (child.stdout) { child.stdout.on("data", (data: Buffer) => { try { updateActivity(); const chunk = data.toString(); stdoutBuffer?.append(chunk); options.onStdout?.(chunk); } catch (err) { log.debug( `spawn stdout handler threw: ${err instanceof Error ? err.message : String(err)}` ); } }); } if (child.stderr) { child.stderr.on("data", (data: Buffer) => { try { const chunk = data.toString(); stderrBuffer?.append(chunk); options.onStderr?.(chunk); } catch (err) { log.debug( `spawn stderr handler threw: ${err instanceof Error ? err.message : String(err)}` ); } }); } child.on("close", (exitCode, signal) => { const durationMs = performance.now() - startTime; untrackChild(child); if (timeoutId) clearTimeout(timeoutId); if (sigkillEscalatorId) clearTimeout(sigkillEscalatorId); if (activityCheckIntervalId) clearInterval(activityCheckIntervalId); if (isTimedOut) { reject( new SpawnTimeoutError(`process timed out after ${options.timeout}ms`, SPAWN_TIMEOUT_CODE) ); return; } if (isActivityTimedOut) { // prefer the idle-ms captured when the kill fired (killedAtIdleMs). // recomputing from lastActivityTime here would be wrong if the child // emitted one final stdout chunk between SIGKILL and close — the // chunk's updateActivity() would reset lastActivityTime and the error // would report near-zero idle, contradicting the kill-site log line. const idleMs = killedAtIdleMs ?? performance.now() - lastActivityTime; const idleSec = Math.round(idleMs / 1000); reject( new SpawnTimeoutError( `activity timeout: no output for ${idleSec}s`, SPAWN_ACTIVITY_TIMEOUT_CODE ) ); return; } // when a child is killed by signal (OOM, segfault, external SIGTERM), // node delivers (code=null, signal=). without this branch, // `exitCode || 0` coerced null to 0 and lifecycle hooks silently // appeared to succeed when they'd actually been killed — caller // checked `result.exitCode !== 0` and moved on. let resolvedExitCode = exitCode ?? 0; let resolvedStderr = stderrBuffer?.toString() ?? ""; if (exitCode === null && signal) { const killMsg = `[spawn] ${options.cmd}: killed by signal ${signal}`; resolvedStderr = resolvedStderr ? `${resolvedStderr}\n${killMsg}` : killMsg; resolvedExitCode = 1; } resolve({ stdout: stdoutBuffer?.toString() ?? "", stderr: resolvedStderr, exitCode: resolvedExitCode, durationMs, }); }); child.on("error", (error) => { const durationMs = performance.now() - startTime; untrackChild(child); if (timeoutId) clearTimeout(timeoutId); if (sigkillEscalatorId) clearTimeout(sigkillEscalatorId); if (activityCheckIntervalId) clearInterval(activityCheckIntervalId); // surface the spawn error in stderr so callers (e.g. lifecycle hook // warnings) don't just see "exit code 1, output: (empty)" when the // command was misspelled, missing, or unexecutable. without this a // user with a bad postCheckout script got an opaque failure, retried // per the guidance, and hit the same wall every run. const errMsg = `[spawn] ${options.cmd}: ${error.message}`; console.error(errMsg); const existingStderr = stderrBuffer?.toString() ?? ""; const finalStderr = existingStderr ? `${existingStderr}\n${errMsg}` : errMsg; resolve({ stdout: stdoutBuffer?.toString() ?? "", stderr: finalStderr, exitCode: 1, durationMs, }); }); if (options.input && child.stdin && options.stdio?.[0] !== "ignore") { child.stdin.write(options.input); child.stdin.end(); } }); }