improve runtest, optimize CI batching (#210)

utils/activity.ts

@@ -0,0 +1,99 @@
+export const DEFAULT_ACTIVITY_TIMEOUT_MS = 30_000;
+export const DEFAULT_ACTIVITY_CHECK_INTERVAL_MS = 5_000;
+
+type ActivityTimeoutContext = {
+  timeoutMs: number;
+  checkIntervalMs: number;
+};
+
+export type ActivityTimeout = {
+  promise: Promise<never>;
+  stop: () => void;
+};
+
+type OutputMonitorContext = {
+  timeoutMs: number;
+  checkIntervalMs: number;
+  onTimeout: (idleMs: number) => void;
+};
+
+type OutputMonitor = {
+  stop: () => void;
+};
+
+type WriteCallback = (error?: Error | null) => void;
+type WriteFunction = {
+  (chunk: string | Uint8Array, cb?: WriteCallback): boolean;
+  (chunk: string | Uint8Array, encoding?: BufferEncoding, cb?: WriteCallback): boolean;
+};
+
+function wrapWrite(original: WriteFunction, onActivity: () => void): WriteFunction {
+  const wrapped: WriteFunction = (
+    chunk: string | Uint8Array,
+    encodingOrCb?: BufferEncoding | WriteCallback,
+    cb?: WriteCallback
+  ): boolean => {
+    onActivity();
+    if (typeof encodingOrCb === "function") {
+      return original(chunk, encodingOrCb);
+    }
+    return original(chunk, encodingOrCb, cb);
+  };
+  return wrapped;
+}
+
+function startProcessOutputMonitor(ctx: OutputMonitorContext): OutputMonitor {
+  let lastActivity = Date.now();
+  let timedOut = false;
+
+  const originalStdoutWrite: WriteFunction = process.stdout.write.bind(process.stdout);
+  const originalStderrWrite: WriteFunction = process.stderr.write.bind(process.stderr);
+
+  function markActivity(): void {
+    lastActivity = Date.now();
+  }
+
+  process.stdout.write = wrapWrite(originalStdoutWrite, markActivity);
+  process.stderr.write = wrapWrite(originalStderrWrite, markActivity);
+
+  const intervalId = setInterval(() => {
+    const idleMs = Date.now() - lastActivity;
+    if (timedOut || idleMs <= ctx.timeoutMs) return;
+    timedOut = true;
+    ctx.onTimeout(idleMs);
+  }, ctx.checkIntervalMs);
+
+  function stop(): void {
+    clearInterval(intervalId);
+    process.stdout.write = originalStdoutWrite;
+    process.stderr.write = originalStderrWrite;
+  }
+
+  return { stop };
+}
+
+export function createProcessOutputActivityTimeout(ctx: ActivityTimeoutContext): ActivityTimeout {
+  let rejectFn: ((error: Error) => void) | null = null;
+  const promise = new Promise<never>((_, reject) => {
+    rejectFn = reject;
+  });
+
+  let monitor: OutputMonitor | null = null;
+  monitor = startProcessOutputMonitor({
+    timeoutMs: ctx.timeoutMs,
+    checkIntervalMs: ctx.checkIntervalMs,
+    onTimeout: (idleMs) => {
+      if (!rejectFn) return;
+      const idleSec = Math.round(idleMs / 1000);
+      if (monitor) {
+        monitor.stop();
+      }
+      rejectFn(new Error(`activity timeout: no output for ${idleSec}s`));
+    },
+  });
+
+  return {
+    promise,
+    stop: monitor.stop,
+  };
+}

utils/docker.ts

@@ -0,0 +1,238 @@
+/**
+ * shared docker utilities for running commands in containers.
+ * used by both play.ts (dev) and test/run.ts (CI).
+ */
+
+import { type SpawnSyncReturns, spawnSync } from "node:child_process";
+import { existsSync } from "node:fs";
+import { platform } from "node:os";
+import { join } from "node:path";
+
+export type DockerRunContext = {
+  actionDir: string;
+  args: string[];
+  platformName: NodeJS.Platform;
+  home: string | undefined;
+  env: NodeJS.ProcessEnv;
+  uid: number;
+  gid: number;
+};
+
+export type SshSetup = {
+  sshFlags: string[];
+  sshSetupCmd: string;
+};
+
+export type DockerRunArgsContext = {
+  ctx: DockerRunContext;
+  envFlags: string[];
+  nodeCmd: string;
+  sshSetup: SshSetup;
+  volumeName: string;
+};
+
+export type VolumeInitContext = {
+  actionDir: string;
+  volumeName: string;
+  uid: number;
+  gid: number;
+};
+
+export function buildDockerRunContext(ctx: {
+  actionDir: string;
+  args: string[];
+}): DockerRunContext {
+  return {
+    actionDir: ctx.actionDir,
+    args: ctx.args,
+    platformName: platform(),
+    home: process.env.HOME,
+    env: process.env,
+    uid: process.getuid?.() ?? 1000,
+    gid: process.getgid?.() ?? 1000,
+  };
+}
+
+export function assertDockerSupported(ctx: DockerRunContext): void {
+  if (ctx.platformName === "win32") {
+    throw new Error("docker mode is not supported on native windows. use wsl2.");
+  }
+}
+
+function buildDarwinSshSetup(ctx: DockerRunContext): SshSetup {
+  const sshFlags: string[] = [];
+  const sshSetupCmd = "";
+  if (ctx.home) {
+    const knownHostsPath = join(ctx.home, ".ssh", "known_hosts");
+    if (existsSync(knownHostsPath)) {
+      sshFlags.push("-v", `${knownHostsPath}:/root/.ssh/known_hosts:ro`);
+    }
+  }
+  sshFlags.push(
+    "-v",
+    "/run/host-services/ssh-auth.sock:/run/host-services/ssh-auth.sock",
+    "-e",
+    "SSH_AUTH_SOCK=/run/host-services/ssh-auth.sock"
+  );
+  return { sshFlags, sshSetupCmd };
+}
+
+function buildLinuxSshSetup(ctx: DockerRunContext): SshSetup {
+  const sshFlags: string[] = [];
+  let sshSetupCmd = "";
+  if (ctx.home) {
+    const sshDir = join(ctx.home, ".ssh");
+    if (existsSync(sshDir)) {
+      sshFlags.push("-v", `${sshDir}:/tmp/.ssh-host:ro`);
+      sshSetupCmd =
+        "mkdir -p /tmp/home/.ssh && cp /tmp/.ssh-host/id_* /tmp/home/.ssh/ 2>/dev/null; chmod 600 /tmp/home/.ssh/id_* 2>/dev/null; " +
+        "ssh-keyscan -t ed25519,rsa github.com >> /tmp/home/.ssh/known_hosts 2>/dev/null; chmod 644 /tmp/home/.ssh/known_hosts; " +
+        "export GIT_SSH_COMMAND='ssh -i /tmp/home/.ssh/id_rsa -o UserKnownHostsFile=/tmp/home/.ssh/known_hosts -o StrictHostKeyChecking=no'; ";
+    }
+  }
+  return { sshFlags, sshSetupCmd };
+}
+
+export function buildSshSetup(ctx: DockerRunContext): SshSetup {
+  if (ctx.platformName === "darwin") {
+    return buildDarwinSshSetup(ctx);
+  }
+  return buildLinuxSshSetup(ctx);
+}
+
+// allowlist of env vars to pass through to the container for test isolation
+const testEnvAllowList = new Set([
+  "GITHUB_TOKEN",
+  "GH_TOKEN",
+  "GITHUB_REPOSITORY",
+  "GITHUB_APP_ID",
+  "GITHUB_PRIVATE_KEY",
+  "OPENAI_API_KEY",
+  "ANTHROPIC_API_KEY",
+  "GEMINI_API_KEY",
+  "GOOGLE_GENERATIVE_AI_API_KEY",
+  "CURSOR_API_KEY",
+  "LOG_LEVEL",
+  "DEBUG",
+  "NODE_ENV",
+  "PLAY_LOCAL",
+  "HOME",
+  "USER",
+  "SSH_AUTH_SOCK",
+  "ACTIONS_ID_TOKEN_REQUEST_URL",
+  "ACTIONS_ID_TOKEN_REQUEST_TOKEN",
+  "GITHUB_API_URL",
+  "GITHUB_SERVER_URL",
+  "GITHUB_GRAPHQL_URL",
+]);
+
+export type EnvFilterMode = "allowlist" | "passthrough";
+
+export function buildEnvFlags(ctx: DockerRunContext, mode: EnvFilterMode): string[] {
+  const envFlags: string[] = [];
+  const entries = Object.entries(ctx.env);
+
+  for (const entry of entries) {
+    const key = entry[0];
+    const value = entry[1];
+
+    if (value === undefined) continue;
+
+    if (mode === "passthrough" || testEnvAllowList.has(key)) {
+      envFlags.push("-e", `${key}=${value}`);
+    }
+  }
+  return envFlags;
+}
+
+export function initializeNodeModulesVolume(ctx: VolumeInitContext): void {
+  spawnSync(
+    "docker",
+    [
+      "run",
+      "--rm",
+      "-v",
+      `${ctx.volumeName}:/app/action/node_modules`,
+      "node:24",
+      "chown",
+      "-R",
+      `${ctx.uid}:${ctx.gid}`,
+      "/app/action/node_modules",
+    ],
+    { stdio: "ignore", cwd: ctx.actionDir }
+  );
+}
+
+export function buildDockerRunArgs(config: DockerRunArgsContext): string[] {
+  const args: string[] = [
+    "run",
+    "--rm",
+    "-t",
+    "--user",
+    `${config.ctx.uid}:${config.ctx.gid}`,
+    "-v",
+    `${config.ctx.actionDir}:/app/action:cached`,
+    "-v",
+    `${config.volumeName}:/app/action/node_modules`,
+    "-w",
+    "/app/action",
+  ];
+  args.push(...config.envFlags);
+  args.push(...config.sshSetup.sshFlags);
+  args.push(
+    "-e",
+    "COREPACK_ENABLE_DOWNLOAD_PROMPT=0",
+    "-e",
+    "HOME=/tmp/home",
+    "-e",
+    "TMPDIR=/tmp",
+    "node:24",
+    "bash",
+    "-c",
+    `${config.sshSetup.sshSetupCmd}mkdir -p /tmp/home/.config /tmp/home/.cache && corepack pnpm install --frozen-lockfile --ignore-scripts && ${config.nodeCmd}`
+  );
+  return args;
+}
+
+export type RunInDockerOptions = {
+  actionDir: string;
+  args: string[];
+  nodeCmd: string;
+  volumeName: string;
+  envFilterMode: EnvFilterMode;
+  onStart?: () => void;
+};
+
+export function runInDocker(options: RunInDockerOptions): SpawnSyncReturns<Buffer> {
+  const ctx = buildDockerRunContext({
+    actionDir: options.actionDir,
+    args: options.args,
+  });
+  assertDockerSupported(ctx);
+
+  const sshSetup = buildSshSetup(ctx);
+  const envFlags = buildEnvFlags(ctx, options.envFilterMode);
+
+  initializeNodeModulesVolume({
+    actionDir: ctx.actionDir,
+    volumeName: options.volumeName,
+    uid: ctx.uid,
+    gid: ctx.gid,
+  });
+
+  if (options.onStart) {
+    options.onStart();
+  }
+
+  return spawnSync(
+    "docker",
+    buildDockerRunArgs({
+      ctx,
+      envFlags,
+      nodeCmd: options.nodeCmd,
+      sshSetup,
+      volumeName: options.volumeName,
+    }),
+    { stdio: "inherit", cwd: ctx.actionDir }
+  );
+}

utils/setup.ts

@@ -6,6 +6,7 @@ import type { BashPermission, PayloadEvent } from "../external.ts";
 import { checkoutPrBranch } from "../mcp/checkout.ts";
 import type { ToolState } from "../mcp/server.ts";
 import { log } from "./cli.ts";
+import { isInsideDocker } from "./globals.ts";
 import type { OctokitWithPlugins } from "./github.ts";
 import { $ } from "./shell.ts";
 
@@ -27,15 +28,17 @@ export function createTempDirectory(): string {
  * Setup the test repository for running actions
  */
 export function setupTestRepo(options: SetupOptions): void {
-  const { tempDir } = options;
+  const tempDir = options.tempDir;
   const repo = process.env.GITHUB_REPOSITORY;
   if (!repo) throw new Error("GITHUB_REPOSITORY is required");
   log.info(`» cloning ${repo} into ${tempDir}...`);
 
-  // use HTTPS with token in CI, SSH locally
-  if (process.env.CI) {
-    const token = process.env.GITHUB_TOKEN;
-    if (!token) throw new Error("GITHUB_TOKEN is required in CI");
+  // use https with token in ci or when running inside docker
+  if (process.env.CI || isInsideDocker) {
+    const token = process.env.GITHUB_TOKEN ?? process.env.GH_TOKEN;
+    if (!token) {
+      throw new Error("GITHUB_TOKEN or GH_TOKEN is required for https clone in ci or docker");
+    }
     $("git", ["clone", `https://x-access-token:${token}@github.com/${repo}.git`, tempDir]);
   } else {
     $("git", ["clone", `git@github.com:${repo}.git`, tempDir]);

utils/subprocess.ts

@@ -1,4 +1,81 @@
-import { spawn as nodeSpawn } from "node:child_process";
+import { type ChildProcess, spawn as nodeSpawn } from "node:child_process";
+import { DEFAULT_ACTIVITY_CHECK_INTERVAL_MS, DEFAULT_ACTIVITY_TIMEOUT_MS } from "./activity.ts";
+import { log } from "./cli.ts";
+
+export type TrackChildOptions = {
+  child: ChildProcess;
+  // if true, kill the entire process group (requires detached spawn)
+  killGroup?: boolean;
+};
+
+// track all spawned child processes for cleanup on Ctrl+C
+const activeChildren = new Map<ChildProcess, boolean>();
+
+// signal handler override (used by test runner for graceful shutdown)
+export type SignalHandler = (signal: NodeJS.Signals) => void;
+let externalSignalHandler: SignalHandler | null = null;
+
+// track a child process for cleanup on Ctrl+C
+export function trackChild(options: TrackChildOptions): void {
+  activeChildren.set(options.child, options.killGroup ?? false);
+}
+
+// untrack a child process
+export function untrackChild(child: ChildProcess): void {
+  activeChildren.delete(child);
+}
+
+// allow callers to override default signal handling
+export function setSignalHandler(handler: SignalHandler | null): void {
+  externalSignalHandler = handler;
+}
+
+// kill all tracked children without exiting
+export function killTrackedChildren(): number {
+  const count = activeChildren.size;
+  for (const entry of activeChildren) {
+    const child = entry[0];
+    const killGroup = entry[1];
+    if (killGroup && child.pid) {
+      try {
+        process.kill(-child.pid, "SIGKILL");
+        continue;
+      } catch {
+        // fall through to direct kill
+      }
+    }
+    child.kill("SIGKILL");
+  }
+  return count;
+}
+
+// cleanup handler for SIGINT/SIGTERM - kills all tracked children
+function cleanupAndExit(signal: string): void {
+  const count = killTrackedChildren();
+  if (count > 0) {
+    log.info(`» received ${signal}, killing ${count} subprocess(es)...`);
+  }
+  // force exit after a short delay if process is stuck
+  setTimeout(() => process.exit(1), 500).unref();
+  process.exit(1);
+}
+
+function handleSignal(signal: NodeJS.Signals): void {
+  if (externalSignalHandler) {
+    externalSignalHandler(signal);
+    return;
+  }
+  cleanupAndExit(signal);
+}
+
+// install signal handlers once (call early in process lifecycle)
+let handlersInstalled = false;
+export function installSignalHandlers(): void {
+  if (handlersInstalled) return;
+  handlersInstalled = true;
+  process.on("SIGINT", () => handleSignal("SIGINT"));
+  process.on("SIGTERM", () => handleSignal("SIGTERM"));
+}
 
 export interface SpawnOptions {
   cmd: string;
@@ -6,6 +83,8 @@ export interface SpawnOptions {
   env?: NodeJS.ProcessEnv;
   input?: string;
   timeout?: number;
+  // activity timeout: kill process if no stdout/stderr for this many ms (default: 30s, 0 to disable)
+  activityTimeout?: number;
   cwd?: string;
   stdio?: ("pipe" | "ignore" | "inherit")[];
   onStdout?: (chunk: string) => void;
@@ -24,6 +103,9 @@ export interface SpawnResult {
  */
 export async function spawn(options: SpawnOptions): Promise<SpawnResult> {
   const { cmd, args, env, input, timeout, cwd, stdio, onStdout, onStderr } = options;
+  const activityTimeoutMs = options.activityTimeout ?? DEFAULT_ACTIVITY_TIMEOUT_MS;
+
+  installSignalHandlers();
 
   const startTime = Date.now();
   let stdoutBuffer = "";
@@ -40,9 +122,16 @@ export async function spawn(options: SpawnOptions): Promise<SpawnResult> {
       cwd: cwd || process.cwd(),
     });
 
-    let timeoutId: NodeJS.Timeout | undefined;
-    let isTimedOut = false;
+    // track child for cleanup on Ctrl+C
+    trackChild({ child });
 
+    let timeoutId: NodeJS.Timeout | undefined;
+    let activityCheckIntervalId: NodeJS.Timeout | undefined;
+    let isTimedOut = false;
+    let isActivityTimedOut = false;
+    let lastActivityTime = Date.now();
+
+    // overall timeout
     if (timeout) {
       timeoutId = setTimeout(() => {
         isTimedOut = true;
@@ -56,8 +145,27 @@ export async function spawn(options: SpawnOptions): Promise<SpawnResult> {
       }, timeout);
     }
 
+    // activity timeout: kill if no output for too long
+    if (activityTimeoutMs > 0) {
+      activityCheckIntervalId = setInterval(() => {
+        const idleMs = Date.now() - lastActivityTime;
+        if (idleMs > activityTimeoutMs) {
+          isActivityTimedOut = true;
+          const idleSec = Math.round(idleMs / 1000);
+          log.error(`no output for ${idleSec}s, killing process`);
+          child.kill("SIGKILL");
+          clearInterval(activityCheckIntervalId);
+        }
+      }, DEFAULT_ACTIVITY_CHECK_INTERVAL_MS);
+    }
+
+    function updateActivity(): void {
+      lastActivityTime = Date.now();
+    }
+
     if (child.stdout) {
       child.stdout.on("data", (data: Buffer) => {
+        updateActivity();
         const chunk = data.toString();
         stdoutBuffer += chunk;
         onStdout?.(chunk);
@@ -66,6 +174,7 @@ export async function spawn(options: SpawnOptions): Promise<SpawnResult> {
 
     if (child.stderr) {
       child.stderr.on("data", (data: Buffer) => {
+        updateActivity();
         const chunk = data.toString();
         stderrBuffer += chunk;
         onStderr?.(chunk);
@@ -75,12 +184,18 @@ export async function spawn(options: SpawnOptions): Promise<SpawnResult> {
     child.on("close", (exitCode) => {
       const durationMs = Date.now() - startTime;
 
-      if (timeoutId) {
-        clearTimeout(timeoutId);
-      }
+      untrackChild(child);
+      if (timeoutId) clearTimeout(timeoutId);
+      if (activityCheckIntervalId) clearInterval(activityCheckIntervalId);
 
       if (isTimedOut) {
-        reject(new Error(`Process timed out after ${timeout}ms`));
+        reject(new Error(`process timed out after ${timeout}ms`));
+        return;
+      }
+
+      if (isActivityTimedOut) {
+        const idleSec = Math.round((Date.now() - lastActivityTime) / 1000);
+        reject(new Error(`activity timeout: no output for ${idleSec}s`));
         return;
       }
 
@@ -95,12 +210,12 @@ export async function spawn(options: SpawnOptions): Promise<SpawnResult> {
     child.on("error", (error) => {
       const durationMs = Date.now() - startTime;
 
-      if (timeoutId) {
-        clearTimeout(timeoutId);
-      }
+      untrackChild(child);
+      if (timeoutId) clearTimeout(timeoutId);
+      if (activityCheckIntervalId) clearInterval(activityCheckIntervalId);
 
       // log spawn errors for debugging
-      console.error(`[spawn] Process spawn error: ${error.message}`);
+      console.error(`[spawn] process spawn error: ${error.message}`);
 
       resolve({
         stdout: stdoutBuffer,