add #timeout, macro errors, refactor tests (#191)

action.yml

@@ -9,6 +9,9 @@ inputs:
   effort:
     description: "Effort level: mini (fast), auto (default), max (most capable)"
     required: false
+  timeout:
+    description: "Maximum run duration (e.g., 10m, 1h30m). Default: 1h"
+    required: false
   agent:
     description: "Agent to use: claude, codex, gemini, cursor, opencode"
     required: false

agents/shared.ts

@@ -32,12 +32,17 @@ export const agent = <const input extends AgentInput>(input: input): defineAgent
       const web = ctx.payload.web;
       const search = ctx.payload.search;
       const write = ctx.payload.write;
-      log.info(`» running ${input.name} with effort=${ctx.payload.effort}...`);
-      // build log box content: user prompt first, then event data with eventInstructions as property
-      const eventWithInstructions = ctx.instructions.eventInstructions
-        ? `additionalInstructions: ${ctx.instructions.eventInstructions}\n${ctx.instructions.event}`
-        : ctx.instructions.event;
-      const logParts = [ctx.instructions.user, eventWithInstructions].filter(Boolean);
+      log.info(
+        `» running ${input.name} with effort=${ctx.payload.effort}, timeout=${ctx.payload.timeout}...`
+      );
+      // build log box content: eventInstructions (if any) + user request (if any) + event data
+      const logParts = [
+        ctx.instructions.eventInstructions
+          ? `EVENT-LEVEL INSTRUCTIONS:\n${ctx.instructions.eventInstructions}`
+          : null,
+        ctx.instructions.user ? `USER REQUEST:\n${ctx.instructions.user}` : null,
+        ctx.instructions.event,
+      ].filter(Boolean);
       log.box(logParts.join("\n\n---\n\n"), {
         title: "Instructions",
       });

entry

@@ -136833,7 +136833,7 @@ function GetCheckSuiteLogsTool(ctx) {
 
 // utils/buildPullfrogFooter.ts
 var PULLFROG_DIVIDER = "<!-- PULLFROG_DIVIDER_DO_NOT_REMOVE_PLZ -->";
-var FROG_LOGO = `<a href="https://pullfrog.com"><picture><source media="(prefers-color-scheme: dark)" srcset="https://pullfrog.com/logos/frog-white-full-128px.png"><img src="https://pullfrog.com/logos/frog-green-full-128px.png" width="9px" height="9px" style="vertical-align: middle; " alt="Pullfrog"></picture></a>`;
+var FROG_LOGO = `<a href="https://pullfrog.com"><picture><source media="(prefers-color-scheme: dark)" srcset="https://pullfrog.com/logos/frog-white-full-18px.png"><img src="https://pullfrog.com/logos/frog-green-full-18px.png" width="9px" height="9px" style="vertical-align: middle; " alt="Pullfrog"></picture></a>`;
 function buildPullfrogFooter(params) {
   const parts = [];
   if (params.triggeredBy) {
@@ -160004,9 +160004,7 @@ var package_default = {
     typecheck: "tsc --noEmit",
     build: "node esbuild.config.js",
     play: "node play.ts",
-    smoke: "node test/smoke.ts",
-    nobash: "node test/nobash.ts",
-    restricted: "node test/restricted.ts",
+    runtest: "node test/run.ts",
     scratch: "node scratch.ts",
     upDeps: "pnpm up --latest",
     lock: "pnpm --ignore-workspace install",
@@ -160267,10 +160265,16 @@ var agent = (input) => {
       const web = ctx.payload.web;
       const search2 = ctx.payload.search;
       const write2 = ctx.payload.write;
-      log.info(`\xBB running ${input.name} with effort=${ctx.payload.effort}...`);
-      const eventWithInstructions = ctx.instructions.eventInstructions ? `additionalInstructions: ${ctx.instructions.eventInstructions}
-${ctx.instructions.event}` : ctx.instructions.event;
-      const logParts = [ctx.instructions.user, eventWithInstructions].filter(Boolean);
+      log.info(
+        `\xBB running ${input.name} with effort=${ctx.payload.effort}, timeout=${ctx.payload.timeout}...`
+      );
+      const logParts = [
+        ctx.instructions.eventInstructions ? `EVENT-LEVEL INSTRUCTIONS:
+${ctx.instructions.eventInstructions}` : null,
+        ctx.instructions.user ? `USER REQUEST:
+${ctx.instructions.user}` : null,
+        ctx.instructions.event
+      ].filter(Boolean);
       log.box(logParts.join("\n\n---\n\n"), {
         title: "Instructions"
       });
@@ -162301,7 +162305,8 @@ var JsonPayload = type({
   "eventInstructions?": "string",
   "repoInstructions?": "string",
   "event?": "object",
-  "effort?": Effort.or("undefined")
+  "effort?": Effort.or("undefined"),
+  "timeout?": type.string.or("undefined")
 });
 var COLLABORATOR_PERMISSIONS = ["admin", "maintain", "write"];
 function isCollaborator(event) {
@@ -162311,6 +162316,7 @@ function isCollaborator(event) {
 var Inputs = type({
   prompt: "string",
   "effort?": Effort.or("undefined"),
+  "timeout?": type.string.or("undefined"),
   "agent?": AgentName.or("undefined"),
   "web?": ToolPermissionInput.or("undefined"),
   "search?": ToolPermissionInput.or("undefined"),
@@ -162334,6 +162340,7 @@ function resolvePayload(repoSettings) {
   const inputs = Inputs.assert({
     prompt: core4.getInput("prompt", { required: true }),
     effort: core4.getInput("effort") || void 0,
+    timeout: core4.getInput("timeout") || void 0,
     agent: core4.getInput("agent") || void 0,
     cwd: core4.getInput("cwd") || void 0,
     web: core4.getInput("web") || void 0,
@@ -162381,6 +162388,7 @@ function resolvePayload(repoSettings) {
     repoInstructions: jsonPayload?.repoInstructions,
     event,
     effort: inputs.effort ?? jsonPayload?.effort ?? "auto",
+    timeout: inputs.timeout ?? jsonPayload?.timeout,
     cwd: resolveCwd(inputs.cwd),
     // permissions: inputs > repoSettings > fallbacks
     web: inputs.web ?? repoSettings.web ?? "enabled",
@@ -162553,6 +162561,18 @@ async function setupGit(params) {
   params.toolState.prNumber = prContext.prNumber;
 }
 
+// utils/time.ts
+var TIMEOUT_DISABLED = "none";
+var TIME_STRING_REGEX = /^(?:(\d+)h)?(?:(\d+)m)?(?:(\d+)s)?$/;
+function parseTimeString(input) {
+  const match2 = input.match(TIME_STRING_REGEX);
+  if (!match2 || !match2[1] && !match2[2] && !match2[3]) return null;
+  const hours = parseInt(match2[1] || "0", 10);
+  const minutes = parseInt(match2[2] || "0", 10);
+  const seconds = parseInt(match2[3] || "0", 10);
+  return (hours * 3600 + minutes * 60 + seconds) * 1e3;
+}
+
 // utils/timer.ts
 var Timer = class {
   initialTimestamp;
@@ -162694,12 +162714,36 @@ async function main() {
           repo: runContext.repo,
           modes: modes2
         });
-        const result = await agent2.run({
+        const agentPromise = agent2.run({
           payload,
           mcpServerUrl: mcpHttpServer.url,
           tmpdir: tmpdir3,
           instructions
         });
+        let result;
+        if (payload.timeout === TIMEOUT_DISABLED) {
+          result = await agentPromise;
+        } else {
+          const parsed2 = payload.timeout ? parseTimeString(payload.timeout) : null;
+          if (payload.timeout && parsed2 === null) {
+            log.warning(`Invalid timeout format "${payload.timeout}", using default 1h`);
+          }
+          const timeoutMs = parsed2 ?? 36e5;
+          const actualTimeout = parsed2 !== null ? payload.timeout : "1h";
+          let timeoutId;
+          const timeoutPromise = new Promise((_3, reject) => {
+            timeoutId = setTimeout(() => {
+              reject(new Error(`agent run timed out after ${actualTimeout}`));
+            }, timeoutMs);
+          });
+          timeoutPromise.catch(() => {
+          });
+          try {
+            result = await Promise.race([agentPromise, timeoutPromise]);
+          } finally {
+            clearTimeout(timeoutId);
+          }
+        }
         if (toolState.lastProgressBody) {
           await writeSummary(toolState.lastProgressBody);
         }

external.ts

@@ -255,6 +255,8 @@ export interface WriteablePayload {
   event: PayloadEvent;
   /** effort level for model selection (mini, auto, max) - defaults to "auto" */
   effort?: Effort | undefined;
+  /** timeout for agent run (e.g., "10m", "1h30m") - defaults to "1h" */
+  timeout?: string | undefined;
   /** working directory for the agent */
   cwd?: string | undefined;
 }

main.ts

@@ -14,6 +14,7 @@ import { resolvePayload } from "./utils/payload.ts";
 import { handleAgentResult } from "./utils/run.ts";
 import { resolveRunContextData } from "./utils/runContextData.ts";
 import { createTempDirectory, setupGit } from "./utils/setup.ts";
+import { parseTimeString, TIMEOUT_DISABLED } from "./utils/time.ts";
 import { Timer } from "./utils/timer.ts";
 import { resolveInstallationToken } from "./utils/token.ts";
 import { resolveRun } from "./utils/workflow.ts";
@@ -35,7 +36,6 @@ export async function main(): Promise<MainResult> {
   await using tokenRef = await resolveInstallationToken();
   process.env.GITHUB_TOKEN = tokenRef.token;
 
-
   const octokit = createOctokit(tokenRef.token);
   const runInfo = await resolveRun({ octokit });
   const toolState = initToolState({ runInfo });
@@ -114,13 +114,41 @@ export async function main(): Promise<MainResult> {
       modes,
     });
 
-    const result = await agent.run({
+    // run agent, optionally with timeout enforcement
+    const agentPromise = agent.run({
       payload,
       mcpServerUrl: mcpHttpServer.url,
       tmpdir,
       instructions,
     });
 
+    // timeout enforcement: default is 1 hour, but can be overridden via macros in the prompt:
+    // - #timeout2h (or any duration like "#timeout30m", "#timeout1h30m") to set a custom timeout
+    // - #notimeout to disable timeout entirely
+    let result: Awaited<typeof agentPromise>;
+    if (payload.timeout === TIMEOUT_DISABLED) {
+      result = await agentPromise;
+    } else {
+      const parsed = payload.timeout ? parseTimeString(payload.timeout) : null;
+      if (payload.timeout && parsed === null) {
+        log.warning(`Invalid timeout format "${payload.timeout}", using default 1h`);
+      }
+      const timeoutMs = parsed ?? 3600000;
+      const actualTimeout = parsed !== null ? payload.timeout : "1h";
+      let timeoutId: NodeJS.Timeout | undefined;
+      const timeoutPromise = new Promise<never>((_, reject) => {
+        timeoutId = setTimeout(() => {
+          reject(new Error(`agent run timed out after ${actualTimeout}`));
+        }, timeoutMs);
+      });
+      timeoutPromise.catch(() => {}); // prevent unhandled rejection if agent wins race
+      try {
+        result = await Promise.race([agentPromise, timeoutPromise]);
+      } finally {
+        clearTimeout(timeoutId);
+      }
+    }
+
     // write last progress body to job summary
     if (toolState.lastProgressBody) {
       await writeSummary(toolState.lastProgressBody);

package.json

@@ -17,9 +17,7 @@
     "typecheck": "tsc --noEmit",
     "build": "node esbuild.config.js",
     "play": "node play.ts",
-    "smoke": "node test/smoke.ts",
-    "nobash": "node test/nobash.ts",
-    "restricted": "node test/restricted.ts",
+    "runtest": "node test/run.ts",
     "scratch": "node scratch.ts",
     "upDeps": "pnpm up --latest",
     "lock": "pnpm --ignore-workspace install",

play.ts

@@ -94,17 +94,17 @@ Usage: node play.ts [options]
 Test the Pullfrog action with the inline playFixture.
 
 Options:
-  --raw [prompt]          Use raw string as prompt instead of playFixture
+  --raw [input]           Use raw string as prompt, or JSON object as full fixture
   --local, -l             Run locally (default: runs in Docker)
   -h, --help              Show this help message
 
 Environment:
   PLAY_LOCAL=1            Same as --local
-  PLAY_FIXTURE            JSON fixture passed by test runner (internal)
 
 Examples:
-  node play.ts                       # Run inline playFixture
-  node play.ts --raw "Hello world"   # Use raw string as prompt
+  node play.ts                                              # Run inline playFixture
+  node play.ts --raw "Hello world"                          # Use raw string as prompt
+  node play.ts --raw '{"prompt":"Hello","timeout":"5s"}'    # Use JSON fixture
     `);
     process.exit(0);
   }
@@ -228,15 +228,16 @@ Examples:
     process.exit(result.status ?? 1);
   }
 
-  // check for fixture passed via env var (from test runner)
-  if (process.env.PLAY_FIXTURE) {
-    const fixtureFromEnv = JSON.parse(process.env.PLAY_FIXTURE) as Inputs;
-    const result = await run(fixtureFromEnv);
-    process.exit(result.success ? 0 : 1);
-  }
-
   if (args["--raw"]) {
-    const result = await run(args["--raw"]);
+    const raw = args["--raw"];
+    // try to parse as JSON, otherwise treat as prompt string
+    let input: Inputs | string = raw;
+    try {
+      input = JSON.parse(raw) as Inputs;
+    } catch {
+      // not valid JSON, use as prompt string
+    }
+    const result = await run(input);
     process.exit(result.success ? 0 : 1);
   }
 

test/agnostic/timeout.ts

@@ -0,0 +1,33 @@
+import type { AgentResult, TestRunnerOptions, ValidationCheck } from "../utils.ts";
+import { defineFixture } from "../utils.ts";
+
+/**
+ * timeout test - validates timeout enforcement works correctly.
+ * sets a very short timeout (5s) and gives the agent a task that takes longer.
+ * the run should fail with a timeout error.
+ */
+
+const fixture = defineFixture(
+  {
+    prompt: `Call the select_mode tool with modeName "Build", then analyze the mode instructions.
+Then call select_mode with modeName "Review" and compare the two modes in detail.
+Finally say "TIMEOUT TEST COMPLETED".`,
+    timeout: "5s",
+    effort: "mini",
+  },
+  { localOnly: true }
+);
+
+function validator(result: AgentResult): ValidationCheck[] {
+  // run should have failed due to timeout
+  const timedOut = !result.success && /timed out/i.test(result.output);
+  return [{ name: "timeout_triggered", passed: timedOut }];
+}
+
+export const test: TestRunnerOptions = {
+  name: "timeout tests",
+  fixture,
+  validator,
+  // agnostic tests expect failures, so override the default success check
+  expectFailure: true,
+};

test/crossagent/nobash.ts

@@ -1,5 +1,5 @@
-import type { AgentResult, ValidationCheck } from "./utils.ts";
-import { defineFixture, generateAgentUuids, getAgentOutput, runTests } from "./utils.ts";
+import type { AgentResult, TestRunnerOptions, ValidationCheck } from "../utils.ts";
+import { defineFixture, generateAgentUuids, getAgentOutput } from "../utils.ts";
 
 /**
  * nobash test - validates agents respect bash=disabled setting.
@@ -32,9 +32,9 @@ function validator(result: AgentResult): ValidationCheck[] {
   return [{ name: "no_bash", passed: bashNotExecuted }];
 }
 
-runTests({
+export const test: TestRunnerOptions = {
   name: "nobash tests",
   fixture,
   validator,
   agentEnv,
-});
+};

test/crossagent/restricted.ts

@@ -1,5 +1,5 @@
-import type { AgentResult, ValidationCheck } from "./utils.ts";
-import { defineFixture, generateAgentUuids, getAgentOutput, runTests } from "./utils.ts";
+import type { AgentResult, TestRunnerOptions, ValidationCheck } from "../utils.ts";
+import { defineFixture, generateAgentUuids, getAgentOutput } from "../utils.ts";
 
 /**
  * restricted test - validates bash=restricted environment filtering.
@@ -48,9 +48,9 @@ function validator(result: AgentResult): ValidationCheck[] {
   ];
 }
 
-runTests({
+export const test: TestRunnerOptions = {
   name: "restricted tests",
   fixture,
   validator,
   agentEnv,
-});
+};

test/crossagent/smoke.ts

@@ -1,5 +1,5 @@
-import type { AgentResult, ValidationCheck } from "./utils.ts";
-import { defineFixture, runTests } from "./utils.ts";
+import type { AgentResult, TestRunnerOptions, ValidationCheck } from "../utils.ts";
+import { defineFixture } from "../utils.ts";
 
 /**
  * smoke test - validates agent can connect to API and call MCP tools.
@@ -29,8 +29,8 @@ function validator(result: AgentResult): ValidationCheck[] {
   ];
 }
 
-runTests({
+export const test: TestRunnerOptions = {
   name: "smoke tests",
   fixture,
   validator,
-});
+};

test/run.ts

@@ -0,0 +1,184 @@
+import { existsSync, readdirSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { config } from "dotenv";
+import {
+  agents,
+  printResults,
+  printSingleValidation,
+  runAgentStreaming,
+  runAllAgentsStreaming,
+  type TestRunnerOptions,
+  type ValidateResultOptions,
+  type ValidationResult,
+  validateResult,
+} from "./utils.ts";
+
+/**
+ * unified test runner for all agent tests.
+ *
+ * usage: node test/run.ts <test> [agent]
+ *
+ * tests are organized into two directories:
+ * - crossagent/ - tests that run across all agents (smoke, nobash, restricted)
+ * - agnostic/   - tests that only need to run once with any agent (timeout)
+ *
+ * the runner automatically detects which type of test based on directory.
+ *
+ * examples:
+ *   node test/run.ts smoke claude     # run smoke test for claude only
+ *   node test/run.ts smoke            # run smoke test for all agents
+ *   node test/run.ts timeout claude   # run timeout test with claude
+ *   node test/run.ts timeout          # run timeout test with default agent
+ */
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+export const actionDir = join(__dirname, "..");
+
+// load .env files
+config({ path: join(actionDir, ".env") });
+config({ path: join(actionDir, "..", ".env") });
+
+type TestType = "crossagent" | "agnostic";
+
+type TestInfo = {
+  type: TestType;
+  config: TestRunnerOptions;
+};
+
+async function loadTest(testName: string): Promise<TestInfo | null> {
+  // check crossagent directory first
+  const crossagentPath = join(__dirname, "crossagent", `${testName}.ts`);
+  if (existsSync(crossagentPath)) {
+    const module = await import(crossagentPath);
+    return { type: "crossagent", config: module.test };
+  }
+
+  // check agnostic directory
+  const agnosticPath = join(__dirname, "agnostic", `${testName}.ts`);
+  if (existsSync(agnosticPath)) {
+    const module = await import(agnosticPath);
+    return { type: "agnostic", config: module.test };
+  }
+
+  return null;
+}
+
+function listAvailableTests(): string[] {
+  const tests: string[] = [];
+
+  // list crossagent tests
+  const crossagentDir = join(__dirname, "crossagent");
+  if (existsSync(crossagentDir)) {
+    for (const file of readdirSync(crossagentDir)) {
+      if (file.endsWith(".ts")) {
+        tests.push(file.replace(".ts", ""));
+      }
+    }
+  }
+
+  // list agnostic tests
+  const agnosticDir = join(__dirname, "agnostic");
+  if (existsSync(agnosticDir)) {
+    for (const file of readdirSync(agnosticDir)) {
+      if (file.endsWith(".ts")) {
+        tests.push(file.replace(".ts", ""));
+      }
+    }
+  }
+
+  return tests;
+}
+
+async function runAgnosticTest(
+  config: TestRunnerOptions,
+  agent: string
+): Promise<ValidationResult> {
+  const env = { ...config.env, ...config.agentEnv?.get(agent) };
+  const result = await runAgentStreaming(agent, { fixture: config.fixture, env });
+  const checks = config.validator(result);
+  const allPassed = checks.every((c) => c.passed);
+
+  // for tests with expectFailure: passed = agent failed AND all validation checks pass
+  // for normal tests: passed = agent succeeded AND all validation checks pass
+  const passed = config.expectFailure ? !result.success && allPassed : result.success && allPassed;
+
+  return {
+    agent: result.agent,
+    passed,
+    checks,
+    output: result.output,
+  };
+}
+
+async function main(): Promise<void> {
+  const testName = process.argv[2];
+  const agentArg = process.argv[3];
+
+  if (!testName) {
+    const available = listAvailableTests();
+    console.error(`usage: node test/run.ts <test> [agent]`);
+    console.error(`available tests: ${available.join(", ")}`);
+    process.exit(1);
+  }
+
+  const testInfo = await loadTest(testName);
+  if (!testInfo) {
+    const available = listAvailableTests();
+    console.error(`unknown test: ${testName}`);
+    console.error(`available tests: ${available.join(", ")}`);
+    process.exit(1);
+  }
+
+  if (agentArg && !agents.includes(agentArg as (typeof agents)[number])) {
+    console.error(`unknown agent: ${agentArg}`);
+    console.error(`available agents: ${agents.join(", ")}`);
+    process.exit(1);
+  }
+
+  const config = testInfo.config;
+
+  if (testInfo.type === "crossagent") {
+    // crossagent tests: run for specified agent or all agents
+    const validateOptions: ValidateResultOptions = { expectFailure: config.expectFailure };
+    if (agentArg) {
+      console.log(`running ${config.name} for: ${agentArg}\n`);
+      const env = { ...config.env, ...config.agentEnv?.get(agentArg) };
+      const result = await runAgentStreaming(agentArg, { fixture: config.fixture, env });
+      const validation = validateResult(result, config.validator, validateOptions);
+      console.log();
+      printSingleValidation(validation);
+      printResults([validation]);
+      process.exit(validation.passed ? 0 : 1);
+    } else {
+      // run all agents
+      console.log(`running ${config.name} for all agents...\n`);
+      const results = await runAllAgentsStreaming({
+        fixture: config.fixture,
+        env: config.env,
+        agentEnv: config.agentEnv,
+      });
+      const validations = results.map((r) => validateResult(r, config.validator, validateOptions));
+      console.log();
+      for (const v of validations) {
+        printSingleValidation(v);
+      }
+      printResults(validations);
+      const allPassed = validations.every((v) => v.passed);
+      process.exit(allPassed ? 0 : 1);
+    }
+  } else {
+    // agnostic tests: run once with specified agent or default
+    const agent = agentArg ?? agents[0];
+    console.log(`running ${config.name} for: ${agent}\n`);
+
+    const validation = await runAgnosticTest(config, agent);
+
+    console.log();
+    printSingleValidation(validation);
+    printResults([validation]);
+    process.exit(validation.passed ? 0 : 1);
+  }
+}
+
+main();

test/utils.ts

@@ -2,17 +2,12 @@ import { spawn } from "node:child_process";
 import { randomUUID } from "node:crypto";
 import { dirname, join } from "node:path";
 import { fileURLToPath } from "node:url";
-import { config } from "dotenv";
 import { agentsManifest } from "../external.ts";
 import type { Inputs } from "../main.ts";
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 export const actionDir = join(__dirname, "..");
 
-// load .env files
-config({ path: join(actionDir, ".env") });
-config({ path: join(actionDir, "..", ".env") });
-
 const LOCAL_TEST_WARNING = "This is a local test - do not post any comments to GitHub.";
 
 export type FixtureOptions = {
@@ -116,18 +111,25 @@ export interface RunOptions {
   env?: Record<string, string> | undefined;
 }
 
+const DEFAULT_TEST_TIMEOUT = "10m";
+
 // run agent and stream output with prefix labels
 export async function runAgentStreaming(agent: string, options: RunOptions): Promise<AgentResult> {
   return new Promise((resolve) => {
     const chunks: Buffer[] = [];
     const prefix = getAgentPrefix(agent);
 
-    const child = spawn("node", ["play.ts"], {
+    // apply default timeout if not specified in fixture
+    const fixture: Inputs = {
+      ...options.fixture,
+      timeout: options.fixture.timeout ?? DEFAULT_TEST_TIMEOUT,
+    };
+
+    const child = spawn("node", ["play.ts", "--raw", JSON.stringify(fixture)], {
       cwd: actionDir,
       env: {
         ...process.env,
         AGENT_OVERRIDE: agent,
-        PLAY_FIXTURE: JSON.stringify(options.fixture),
         ...options.env,
       },
       stdio: "pipe",
@@ -174,12 +176,17 @@ export async function runAgent(agent: string, options: RunOptions): Promise<Agen
   return new Promise((resolve) => {
     const chunks: Buffer[] = [];
 
-    const child = spawn("node", ["play.ts"], {
+    // apply default timeout if not specified in fixture
+    const fixture: Inputs = {
+      ...options.fixture,
+      timeout: options.fixture.timeout ?? DEFAULT_TEST_TIMEOUT,
+    };
+
+    const child = spawn("node", ["play.ts", "--raw", JSON.stringify(fixture)], {
       cwd: actionDir,
       env: {
         ...process.env,
         AGENT_OVERRIDE: agent,
-        PLAY_FIXTURE: JSON.stringify(options.fixture),
         ...options.env,
       },
       stdio: "pipe",
@@ -198,13 +205,29 @@ export async function runAgent(agent: string, options: RunOptions): Promise<Agen
   });
 }
 
-export function validateResult(result: AgentResult, validator: ValidatorFn): ValidationResult {
+export type ValidateResultOptions = {
+  // if true, test passes when validation checks pass regardless of agent success
+  // (used for tests like timeout that expect the agent run to fail)
+  expectFailure?: boolean | undefined;
+};
+
+export function validateResult(
+  result: AgentResult,
+  validator: ValidatorFn,
+  options?: ValidateResultOptions
+): ValidationResult {
   const checks = validator(result);
   const allPassed = checks.every((c) => c.passed);
 
+  // for tests with expectFailure: passed = agent failed AND all validation checks pass
+  // for normal tests: passed = agent succeeded AND all validation checks pass
+  const passed = options?.expectFailure
+    ? !result.success && allPassed
+    : result.success && allPassed;
+
   return {
     agent: result.agent,
-    passed: result.success && allPassed,
+    passed,
     checks,
     output: result.output,
   };
@@ -234,43 +257,9 @@ export interface TestRunnerOptions {
   env?: Record<string, string>;
   // per-agent env vars (for unique markers)
   agentEnv?: Map<string, Record<string, string>>;
-}
-
-export async function runTests(options: TestRunnerOptions): Promise<void> {
-  const agentArg = process.argv[2];
-
-  if (agentArg) {
-    // single agent mode
-    if (!agents.includes(agentArg as (typeof agents)[number])) {
-      console.error(`unknown agent: ${agentArg}`);
-      console.error(`available agents: ${agents.join(", ")}`);
-      process.exit(1);
-    }
-    console.log(`running ${options.name} for: ${agentArg}\n`);
-    const env = { ...options.env, ...options.agentEnv?.get(agentArg) };
-    const result = await runAgentStreaming(agentArg, { fixture: options.fixture, env });
-    const validation = validateResult(result, options.validator);
-    console.log();
-    printSingleValidation(validation);
-    process.exit(validation.passed ? 0 : 1);
-  }
-
-  // parallel mode with streaming
-  console.log(`running ${options.name} for: ${agents.join(", ")}\n`);
-
-  const results = await runAllAgentsStreaming({
-    fixture: options.fixture,
-    env: options.env,
-    agentEnv: options.agentEnv,
-  });
-
-  console.log();
-  const validations = results.map((r) => validateResult(r, options.validator));
-
-  printResults(validations);
-
-  const failed = validations.filter((v) => !v.passed);
-  process.exit(failed.length > 0 ? 1 : 0);
+  // if true, test passes when agent fails AND validation checks pass
+  // (used for tests like timeout that expect the agent run to fail)
+  expectFailure?: boolean;
 }
 
 export function printSingleValidation(validation: ValidationResult): void {

utils/buildPullfrogFooter.ts

@@ -1,6 +1,6 @@
 export const PULLFROG_DIVIDER = "<!-- PULLFROG_DIVIDER_DO_NOT_REMOVE_PLZ -->";
 
-const FROG_LOGO = `<a href="https://pullfrog.com"><picture><source media="(prefers-color-scheme: dark)" srcset="https://pullfrog.com/logos/frog-white-full-128px.png"><img src="https://pullfrog.com/logos/frog-green-full-128px.png" width="9px" height="9px" style="vertical-align: middle; " alt="Pullfrog"></picture></a>`;
+const FROG_LOGO = `<a href="https://pullfrog.com"><picture><source media="(prefers-color-scheme: dark)" srcset="https://pullfrog.com/logos/frog-white-full-18px.png"><img src="https://pullfrog.com/logos/frog-green-full-18px.png" width="9px" height="9px" style="vertical-align: middle; " alt="Pullfrog"></picture></a>`;
 
 export interface AgentInfo {
   displayName: string;

utils/payload.test.ts

@@ -24,6 +24,10 @@ describe("Inputs schema", () => {
     ["effort", "mini"],
     ["effort", "auto"],
     ["effort", "max"],
+    ["timeout", "10m"],
+    ["timeout", "1h30m"],
+    ["timeout", "30s"],
+    ["timeout", undefined],
     ["agent", "claude"],
     ["agent", "codex"],
     ["agent", "cursor"],
@@ -63,6 +67,9 @@ describe("JsonPayload schema", () => {
     ["effort", "mini"],
     ["effort", "auto"],
     ["effort", "max"],
+    ["timeout", "10m"],
+    ["timeout", "1h30m"],
+    ["timeout", "30s"],
     ["event", { trigger: "unknown" }],
   ] as const)("should accept optional %s with value %s", (prop, value) => {
     const input = { "~pullfrog": true, version: "1.2.3", [prop]: value };

utils/payload.ts

@@ -22,6 +22,7 @@ export const JsonPayload = type({
   "repoInstructions?": "string",
   "event?": "object",
   "effort?": Effort.or("undefined"),
+  "timeout?": type.string.or("undefined"),
 });
 
 // permission levels that indicate collaborator status (have push access)
@@ -41,6 +42,7 @@ function isCollaborator(event: PayloadEvent): boolean {
 export const Inputs = type({
   prompt: "string",
   "effort?": Effort.or("undefined"),
+  "timeout?": type.string.or("undefined"),
   "agent?": AgentName.or("undefined"),
   "web?": ToolPermissionInput.or("undefined"),
   "search?": ToolPermissionInput.or("undefined"),
@@ -70,6 +72,7 @@ export function resolvePayload(repoSettings: RepoSettings) {
   const inputs = Inputs.assert({
     prompt: core.getInput("prompt", { required: true }),
     effort: core.getInput("effort") || undefined,
+    timeout: core.getInput("timeout") || undefined,
     agent: core.getInput("agent") || undefined,
     cwd: core.getInput("cwd") || undefined,
     web: core.getInput("web") || undefined,
@@ -145,6 +148,7 @@ export function resolvePayload(repoSettings: RepoSettings) {
     repoInstructions: jsonPayload?.repoInstructions,
     event,
     effort: inputs.effort ?? jsonPayload?.effort ?? "auto",
+    timeout: inputs.timeout ?? jsonPayload?.timeout,
     cwd: resolveCwd(inputs.cwd),
 
     // permissions: inputs > repoSettings > fallbacks

utils/time.test.ts

@@ -0,0 +1,47 @@
+import { describe, expect, it } from "vitest";
+import { isValidTimeString, parseTimeString } from "./time.ts";
+
+describe("parseTimeString", () => {
+  it.each([
+    ["10m", 600000], // 10 minutes
+    ["1h", 3600000], // 1 hour
+    ["30s", 30000], // 30 seconds
+    ["1h30m", 5400000], // 1 hour 30 minutes
+    ["10m12s", 612000], // 10 minutes 12 seconds
+    ["1h30m45s", 5445000], // 1 hour 30 minutes 45 seconds
+    ["2h", 7200000], // 2 hours
+    ["90m", 5400000], // 90 minutes
+    ["0m", 0], // 0 minutes (edge case)
+    ["0s", 0], // 0 seconds (edge case)
+  ])("parses '%s' to %d ms", (input, expected) => {
+    expect(parseTimeString(input)).toBe(expected);
+  });
+
+  it.each([
+    [""], // empty string
+    ["abc"], // no numbers
+    ["10"], // no unit
+    ["10x"], // invalid unit
+    ["h10m"], // hours without number
+    ["m10"], // units before number
+    ["10 m"], // space between number and unit
+    ["-10m"], // negative number
+    ["10.5m"], // decimal
+    ["10m 30s"], // space between components
+  ])("returns null for invalid input '%s'", (input) => {
+    expect(parseTimeString(input)).toBeNull();
+  });
+});
+
+describe("isValidTimeString", () => {
+  it.each(["10m", "1h", "30s", "1h30m", "10m12s", "1h30m45s"])(
+    "returns true for valid '%s'",
+    (input) => {
+      expect(isValidTimeString(input)).toBe(true);
+    }
+  );
+
+  it.each(["", "abc", "10", "10x", "-10m", "10.5m"])("returns false for invalid '%s'", (input) => {
+    expect(isValidTimeString(input)).toBe(false);
+  });
+});

utils/time.ts

@@ -0,0 +1,33 @@
+/**
+ * time string parsing utilities for timeout configuration.
+ * supports formats like "10m", "1h30m", "10m12s", "30s".
+ */
+
+// special value indicating timeout is explicitly disabled via #notimeout macro
+export const TIMEOUT_DISABLED = "none";
+
+// time string regex: supports formats like "10m", "1h30m", "10m12s", "30s"
+// at least one component (hours, minutes, or seconds) is required
+const TIME_STRING_REGEX = /^(?:(\d+)h)?(?:(\d+)m)?(?:(\d+)s)?$/;
+
+/**
+ * parse a time string like "10m", "1h30m", "10m12s" into milliseconds.
+ * returns null if the string is not a valid time format.
+ */
+export function parseTimeString(input: string): number | null {
+  const match = input.match(TIME_STRING_REGEX);
+  if (!match || (!match[1] && !match[2] && !match[3])) return null;
+
+  const hours = parseInt(match[1] || "0", 10);
+  const minutes = parseInt(match[2] || "0", 10);
+  const seconds = parseInt(match[3] || "0", 10);
+
+  return (hours * 3600 + minutes * 60 + seconds) * 1000;
+}
+
+/**
+ * check if a string is a valid time format.
+ */
+export function isValidTimeString(input: string): boolean {
+  return parseTimeString(input) !== null;
+}