From 943409c4178feed3abe0db240f4d90f70554b8d1 Mon Sep 17 00:00:00 2001 From: David Blass Date: Wed, 28 Jan 2026 21:06:57 +0000 Subject: [PATCH] add #timeout, macro errors, refactor tests (#191) --- action.yml | 3 + agents/shared.ts | 17 ++- entry | 64 ++++++++-- external.ts | 2 + main.ts | 32 ++++- package.json | 4 +- play.ts | 25 ++-- test/agnostic/timeout.ts | 33 +++++ test/{ => crossagent}/nobash.ts | 8 +- test/{ => crossagent}/restricted.ts | 8 +- test/{ => crossagent}/smoke.ts | 8 +- test/run.ts | 184 ++++++++++++++++++++++++++++ test/utils.ts | 85 ++++++------- utils/buildPullfrogFooter.ts | 2 +- utils/payload.test.ts | 7 ++ utils/payload.ts | 4 + utils/time.test.ts | 47 +++++++ utils/time.ts | 33 +++++ 18 files changed, 472 insertions(+), 94 deletions(-) create mode 100644 test/agnostic/timeout.ts rename test/{ => crossagent}/nobash.ts (84%) rename test/{ => crossagent}/restricted.ts (89%) rename test/{ => crossagent}/smoke.ts (84%) create mode 100644 test/run.ts create mode 100644 utils/time.test.ts create mode 100644 utils/time.ts diff --git a/action.yml b/action.yml index ace8f25..e4fe3b8 100644 --- a/action.yml +++ b/action.yml @@ -9,6 +9,9 @@ inputs: effort: description: "Effort level: mini (fast), auto (default), max (most capable)" required: false + timeout: + description: "Maximum run duration (e.g., 10m, 1h30m). Default: 1h" + required: false agent: description: "Agent to use: claude, codex, gemini, cursor, opencode" required: false diff --git a/agents/shared.ts b/agents/shared.ts index 0c93c69..56dc0ec 100644 --- a/agents/shared.ts +++ b/agents/shared.ts @@ -32,12 +32,17 @@ export const agent = (input: input): defineAgent const web = ctx.payload.web; const search = ctx.payload.search; const write = ctx.payload.write; - log.info(`» running ${input.name} with effort=${ctx.payload.effort}...`); - // build log box content: user prompt first, then event data with eventInstructions as property - const eventWithInstructions = ctx.instructions.eventInstructions - ? `additionalInstructions: ${ctx.instructions.eventInstructions}\n${ctx.instructions.event}` - : ctx.instructions.event; - const logParts = [ctx.instructions.user, eventWithInstructions].filter(Boolean); + log.info( + `» running ${input.name} with effort=${ctx.payload.effort}, timeout=${ctx.payload.timeout}...` + ); + // build log box content: eventInstructions (if any) + user request (if any) + event data + const logParts = [ + ctx.instructions.eventInstructions + ? `EVENT-LEVEL INSTRUCTIONS:\n${ctx.instructions.eventInstructions}` + : null, + ctx.instructions.user ? `USER REQUEST:\n${ctx.instructions.user}` : null, + ctx.instructions.event, + ].filter(Boolean); log.box(logParts.join("\n\n---\n\n"), { title: "Instructions", }); diff --git a/entry b/entry index 1e77fb3..c2af0f9 100755 --- a/entry +++ b/entry @@ -136833,7 +136833,7 @@ function GetCheckSuiteLogsTool(ctx) { // utils/buildPullfrogFooter.ts var PULLFROG_DIVIDER = ""; -var FROG_LOGO = `Pullfrog`; +var FROG_LOGO = `Pullfrog`; function buildPullfrogFooter(params) { const parts = []; if (params.triggeredBy) { @@ -160004,9 +160004,7 @@ var package_default = { typecheck: "tsc --noEmit", build: "node esbuild.config.js", play: "node play.ts", - smoke: "node test/smoke.ts", - nobash: "node test/nobash.ts", - restricted: "node test/restricted.ts", + runtest: "node test/run.ts", scratch: "node scratch.ts", upDeps: "pnpm up --latest", lock: "pnpm --ignore-workspace install", @@ -160267,10 +160265,16 @@ var agent = (input) => { const web = ctx.payload.web; const search2 = ctx.payload.search; const write2 = ctx.payload.write; - log.info(`\xBB running ${input.name} with effort=${ctx.payload.effort}...`); - const eventWithInstructions = ctx.instructions.eventInstructions ? `additionalInstructions: ${ctx.instructions.eventInstructions} -${ctx.instructions.event}` : ctx.instructions.event; - const logParts = [ctx.instructions.user, eventWithInstructions].filter(Boolean); + log.info( + `\xBB running ${input.name} with effort=${ctx.payload.effort}, timeout=${ctx.payload.timeout}...` + ); + const logParts = [ + ctx.instructions.eventInstructions ? `EVENT-LEVEL INSTRUCTIONS: +${ctx.instructions.eventInstructions}` : null, + ctx.instructions.user ? `USER REQUEST: +${ctx.instructions.user}` : null, + ctx.instructions.event + ].filter(Boolean); log.box(logParts.join("\n\n---\n\n"), { title: "Instructions" }); @@ -162301,7 +162305,8 @@ var JsonPayload = type({ "eventInstructions?": "string", "repoInstructions?": "string", "event?": "object", - "effort?": Effort.or("undefined") + "effort?": Effort.or("undefined"), + "timeout?": type.string.or("undefined") }); var COLLABORATOR_PERMISSIONS = ["admin", "maintain", "write"]; function isCollaborator(event) { @@ -162311,6 +162316,7 @@ function isCollaborator(event) { var Inputs = type({ prompt: "string", "effort?": Effort.or("undefined"), + "timeout?": type.string.or("undefined"), "agent?": AgentName.or("undefined"), "web?": ToolPermissionInput.or("undefined"), "search?": ToolPermissionInput.or("undefined"), @@ -162334,6 +162340,7 @@ function resolvePayload(repoSettings) { const inputs = Inputs.assert({ prompt: core4.getInput("prompt", { required: true }), effort: core4.getInput("effort") || void 0, + timeout: core4.getInput("timeout") || void 0, agent: core4.getInput("agent") || void 0, cwd: core4.getInput("cwd") || void 0, web: core4.getInput("web") || void 0, @@ -162381,6 +162388,7 @@ function resolvePayload(repoSettings) { repoInstructions: jsonPayload?.repoInstructions, event, effort: inputs.effort ?? jsonPayload?.effort ?? "auto", + timeout: inputs.timeout ?? jsonPayload?.timeout, cwd: resolveCwd(inputs.cwd), // permissions: inputs > repoSettings > fallbacks web: inputs.web ?? repoSettings.web ?? "enabled", @@ -162553,6 +162561,18 @@ async function setupGit(params) { params.toolState.prNumber = prContext.prNumber; } +// utils/time.ts +var TIMEOUT_DISABLED = "none"; +var TIME_STRING_REGEX = /^(?:(\d+)h)?(?:(\d+)m)?(?:(\d+)s)?$/; +function parseTimeString(input) { + const match2 = input.match(TIME_STRING_REGEX); + if (!match2 || !match2[1] && !match2[2] && !match2[3]) return null; + const hours = parseInt(match2[1] || "0", 10); + const minutes = parseInt(match2[2] || "0", 10); + const seconds = parseInt(match2[3] || "0", 10); + return (hours * 3600 + minutes * 60 + seconds) * 1e3; +} + // utils/timer.ts var Timer = class { initialTimestamp; @@ -162694,12 +162714,36 @@ async function main() { repo: runContext.repo, modes: modes2 }); - const result = await agent2.run({ + const agentPromise = agent2.run({ payload, mcpServerUrl: mcpHttpServer.url, tmpdir: tmpdir3, instructions }); + let result; + if (payload.timeout === TIMEOUT_DISABLED) { + result = await agentPromise; + } else { + const parsed2 = payload.timeout ? parseTimeString(payload.timeout) : null; + if (payload.timeout && parsed2 === null) { + log.warning(`Invalid timeout format "${payload.timeout}", using default 1h`); + } + const timeoutMs = parsed2 ?? 36e5; + const actualTimeout = parsed2 !== null ? payload.timeout : "1h"; + let timeoutId; + const timeoutPromise = new Promise((_3, reject) => { + timeoutId = setTimeout(() => { + reject(new Error(`agent run timed out after ${actualTimeout}`)); + }, timeoutMs); + }); + timeoutPromise.catch(() => { + }); + try { + result = await Promise.race([agentPromise, timeoutPromise]); + } finally { + clearTimeout(timeoutId); + } + } if (toolState.lastProgressBody) { await writeSummary(toolState.lastProgressBody); } diff --git a/external.ts b/external.ts index 03f357f..ebefc02 100644 --- a/external.ts +++ b/external.ts @@ -255,6 +255,8 @@ export interface WriteablePayload { event: PayloadEvent; /** effort level for model selection (mini, auto, max) - defaults to "auto" */ effort?: Effort | undefined; + /** timeout for agent run (e.g., "10m", "1h30m") - defaults to "1h" */ + timeout?: string | undefined; /** working directory for the agent */ cwd?: string | undefined; } diff --git a/main.ts b/main.ts index 8450509..62d7da2 100644 --- a/main.ts +++ b/main.ts @@ -14,6 +14,7 @@ import { resolvePayload } from "./utils/payload.ts"; import { handleAgentResult } from "./utils/run.ts"; import { resolveRunContextData } from "./utils/runContextData.ts"; import { createTempDirectory, setupGit } from "./utils/setup.ts"; +import { parseTimeString, TIMEOUT_DISABLED } from "./utils/time.ts"; import { Timer } from "./utils/timer.ts"; import { resolveInstallationToken } from "./utils/token.ts"; import { resolveRun } from "./utils/workflow.ts"; @@ -35,7 +36,6 @@ export async function main(): Promise { await using tokenRef = await resolveInstallationToken(); process.env.GITHUB_TOKEN = tokenRef.token; - const octokit = createOctokit(tokenRef.token); const runInfo = await resolveRun({ octokit }); const toolState = initToolState({ runInfo }); @@ -114,13 +114,41 @@ export async function main(): Promise { modes, }); - const result = await agent.run({ + // run agent, optionally with timeout enforcement + const agentPromise = agent.run({ payload, mcpServerUrl: mcpHttpServer.url, tmpdir, instructions, }); + // timeout enforcement: default is 1 hour, but can be overridden via macros in the prompt: + // - #timeout2h (or any duration like "#timeout30m", "#timeout1h30m") to set a custom timeout + // - #notimeout to disable timeout entirely + let result: Awaited; + if (payload.timeout === TIMEOUT_DISABLED) { + result = await agentPromise; + } else { + const parsed = payload.timeout ? parseTimeString(payload.timeout) : null; + if (payload.timeout && parsed === null) { + log.warning(`Invalid timeout format "${payload.timeout}", using default 1h`); + } + const timeoutMs = parsed ?? 3600000; + const actualTimeout = parsed !== null ? payload.timeout : "1h"; + let timeoutId: NodeJS.Timeout | undefined; + const timeoutPromise = new Promise((_, reject) => { + timeoutId = setTimeout(() => { + reject(new Error(`agent run timed out after ${actualTimeout}`)); + }, timeoutMs); + }); + timeoutPromise.catch(() => {}); // prevent unhandled rejection if agent wins race + try { + result = await Promise.race([agentPromise, timeoutPromise]); + } finally { + clearTimeout(timeoutId); + } + } + // write last progress body to job summary if (toolState.lastProgressBody) { await writeSummary(toolState.lastProgressBody); diff --git a/package.json b/package.json index 73e069c..1dbab4c 100644 --- a/package.json +++ b/package.json @@ -17,9 +17,7 @@ "typecheck": "tsc --noEmit", "build": "node esbuild.config.js", "play": "node play.ts", - "smoke": "node test/smoke.ts", - "nobash": "node test/nobash.ts", - "restricted": "node test/restricted.ts", + "runtest": "node test/run.ts", "scratch": "node scratch.ts", "upDeps": "pnpm up --latest", "lock": "pnpm --ignore-workspace install", diff --git a/play.ts b/play.ts index b001465..a940479 100644 --- a/play.ts +++ b/play.ts @@ -94,17 +94,17 @@ Usage: node play.ts [options] Test the Pullfrog action with the inline playFixture. Options: - --raw [prompt] Use raw string as prompt instead of playFixture + --raw [input] Use raw string as prompt, or JSON object as full fixture --local, -l Run locally (default: runs in Docker) -h, --help Show this help message Environment: PLAY_LOCAL=1 Same as --local - PLAY_FIXTURE JSON fixture passed by test runner (internal) Examples: - node play.ts # Run inline playFixture - node play.ts --raw "Hello world" # Use raw string as prompt + node play.ts # Run inline playFixture + node play.ts --raw "Hello world" # Use raw string as prompt + node play.ts --raw '{"prompt":"Hello","timeout":"5s"}' # Use JSON fixture `); process.exit(0); } @@ -228,15 +228,16 @@ Examples: process.exit(result.status ?? 1); } - // check for fixture passed via env var (from test runner) - if (process.env.PLAY_FIXTURE) { - const fixtureFromEnv = JSON.parse(process.env.PLAY_FIXTURE) as Inputs; - const result = await run(fixtureFromEnv); - process.exit(result.success ? 0 : 1); - } - if (args["--raw"]) { - const result = await run(args["--raw"]); + const raw = args["--raw"]; + // try to parse as JSON, otherwise treat as prompt string + let input: Inputs | string = raw; + try { + input = JSON.parse(raw) as Inputs; + } catch { + // not valid JSON, use as prompt string + } + const result = await run(input); process.exit(result.success ? 0 : 1); } diff --git a/test/agnostic/timeout.ts b/test/agnostic/timeout.ts new file mode 100644 index 0000000..e6ecfd1 --- /dev/null +++ b/test/agnostic/timeout.ts @@ -0,0 +1,33 @@ +import type { AgentResult, TestRunnerOptions, ValidationCheck } from "../utils.ts"; +import { defineFixture } from "../utils.ts"; + +/** + * timeout test - validates timeout enforcement works correctly. + * sets a very short timeout (5s) and gives the agent a task that takes longer. + * the run should fail with a timeout error. + */ + +const fixture = defineFixture( + { + prompt: `Call the select_mode tool with modeName "Build", then analyze the mode instructions. +Then call select_mode with modeName "Review" and compare the two modes in detail. +Finally say "TIMEOUT TEST COMPLETED".`, + timeout: "5s", + effort: "mini", + }, + { localOnly: true } +); + +function validator(result: AgentResult): ValidationCheck[] { + // run should have failed due to timeout + const timedOut = !result.success && /timed out/i.test(result.output); + return [{ name: "timeout_triggered", passed: timedOut }]; +} + +export const test: TestRunnerOptions = { + name: "timeout tests", + fixture, + validator, + // agnostic tests expect failures, so override the default success check + expectFailure: true, +}; diff --git a/test/nobash.ts b/test/crossagent/nobash.ts similarity index 84% rename from test/nobash.ts rename to test/crossagent/nobash.ts index 860559a..8f9e956 100644 --- a/test/nobash.ts +++ b/test/crossagent/nobash.ts @@ -1,5 +1,5 @@ -import type { AgentResult, ValidationCheck } from "./utils.ts"; -import { defineFixture, generateAgentUuids, getAgentOutput, runTests } from "./utils.ts"; +import type { AgentResult, TestRunnerOptions, ValidationCheck } from "../utils.ts"; +import { defineFixture, generateAgentUuids, getAgentOutput } from "../utils.ts"; /** * nobash test - validates agents respect bash=disabled setting. @@ -32,9 +32,9 @@ function validator(result: AgentResult): ValidationCheck[] { return [{ name: "no_bash", passed: bashNotExecuted }]; } -runTests({ +export const test: TestRunnerOptions = { name: "nobash tests", fixture, validator, agentEnv, -}); +}; diff --git a/test/restricted.ts b/test/crossagent/restricted.ts similarity index 89% rename from test/restricted.ts rename to test/crossagent/restricted.ts index 114366b..f90fe32 100644 --- a/test/restricted.ts +++ b/test/crossagent/restricted.ts @@ -1,5 +1,5 @@ -import type { AgentResult, ValidationCheck } from "./utils.ts"; -import { defineFixture, generateAgentUuids, getAgentOutput, runTests } from "./utils.ts"; +import type { AgentResult, TestRunnerOptions, ValidationCheck } from "../utils.ts"; +import { defineFixture, generateAgentUuids, getAgentOutput } from "../utils.ts"; /** * restricted test - validates bash=restricted environment filtering. @@ -48,9 +48,9 @@ function validator(result: AgentResult): ValidationCheck[] { ]; } -runTests({ +export const test: TestRunnerOptions = { name: "restricted tests", fixture, validator, agentEnv, -}); +}; diff --git a/test/smoke.ts b/test/crossagent/smoke.ts similarity index 84% rename from test/smoke.ts rename to test/crossagent/smoke.ts index 6dd2f5f..e826cff 100644 --- a/test/smoke.ts +++ b/test/crossagent/smoke.ts @@ -1,5 +1,5 @@ -import type { AgentResult, ValidationCheck } from "./utils.ts"; -import { defineFixture, runTests } from "./utils.ts"; +import type { AgentResult, TestRunnerOptions, ValidationCheck } from "../utils.ts"; +import { defineFixture } from "../utils.ts"; /** * smoke test - validates agent can connect to API and call MCP tools. @@ -29,8 +29,8 @@ function validator(result: AgentResult): ValidationCheck[] { ]; } -runTests({ +export const test: TestRunnerOptions = { name: "smoke tests", fixture, validator, -}); +}; diff --git a/test/run.ts b/test/run.ts new file mode 100644 index 0000000..c949096 --- /dev/null +++ b/test/run.ts @@ -0,0 +1,184 @@ +import { existsSync, readdirSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { config } from "dotenv"; +import { + agents, + printResults, + printSingleValidation, + runAgentStreaming, + runAllAgentsStreaming, + type TestRunnerOptions, + type ValidateResultOptions, + type ValidationResult, + validateResult, +} from "./utils.ts"; + +/** + * unified test runner for all agent tests. + * + * usage: node test/run.ts [agent] + * + * tests are organized into two directories: + * - crossagent/ - tests that run across all agents (smoke, nobash, restricted) + * - agnostic/ - tests that only need to run once with any agent (timeout) + * + * the runner automatically detects which type of test based on directory. + * + * examples: + * node test/run.ts smoke claude # run smoke test for claude only + * node test/run.ts smoke # run smoke test for all agents + * node test/run.ts timeout claude # run timeout test with claude + * node test/run.ts timeout # run timeout test with default agent + */ + +const __dirname = dirname(fileURLToPath(import.meta.url)); +export const actionDir = join(__dirname, ".."); + +// load .env files +config({ path: join(actionDir, ".env") }); +config({ path: join(actionDir, "..", ".env") }); + +type TestType = "crossagent" | "agnostic"; + +type TestInfo = { + type: TestType; + config: TestRunnerOptions; +}; + +async function loadTest(testName: string): Promise { + // check crossagent directory first + const crossagentPath = join(__dirname, "crossagent", `${testName}.ts`); + if (existsSync(crossagentPath)) { + const module = await import(crossagentPath); + return { type: "crossagent", config: module.test }; + } + + // check agnostic directory + const agnosticPath = join(__dirname, "agnostic", `${testName}.ts`); + if (existsSync(agnosticPath)) { + const module = await import(agnosticPath); + return { type: "agnostic", config: module.test }; + } + + return null; +} + +function listAvailableTests(): string[] { + const tests: string[] = []; + + // list crossagent tests + const crossagentDir = join(__dirname, "crossagent"); + if (existsSync(crossagentDir)) { + for (const file of readdirSync(crossagentDir)) { + if (file.endsWith(".ts")) { + tests.push(file.replace(".ts", "")); + } + } + } + + // list agnostic tests + const agnosticDir = join(__dirname, "agnostic"); + if (existsSync(agnosticDir)) { + for (const file of readdirSync(agnosticDir)) { + if (file.endsWith(".ts")) { + tests.push(file.replace(".ts", "")); + } + } + } + + return tests; +} + +async function runAgnosticTest( + config: TestRunnerOptions, + agent: string +): Promise { + const env = { ...config.env, ...config.agentEnv?.get(agent) }; + const result = await runAgentStreaming(agent, { fixture: config.fixture, env }); + const checks = config.validator(result); + const allPassed = checks.every((c) => c.passed); + + // for tests with expectFailure: passed = agent failed AND all validation checks pass + // for normal tests: passed = agent succeeded AND all validation checks pass + const passed = config.expectFailure ? !result.success && allPassed : result.success && allPassed; + + return { + agent: result.agent, + passed, + checks, + output: result.output, + }; +} + +async function main(): Promise { + const testName = process.argv[2]; + const agentArg = process.argv[3]; + + if (!testName) { + const available = listAvailableTests(); + console.error(`usage: node test/run.ts [agent]`); + console.error(`available tests: ${available.join(", ")}`); + process.exit(1); + } + + const testInfo = await loadTest(testName); + if (!testInfo) { + const available = listAvailableTests(); + console.error(`unknown test: ${testName}`); + console.error(`available tests: ${available.join(", ")}`); + process.exit(1); + } + + if (agentArg && !agents.includes(agentArg as (typeof agents)[number])) { + console.error(`unknown agent: ${agentArg}`); + console.error(`available agents: ${agents.join(", ")}`); + process.exit(1); + } + + const config = testInfo.config; + + if (testInfo.type === "crossagent") { + // crossagent tests: run for specified agent or all agents + const validateOptions: ValidateResultOptions = { expectFailure: config.expectFailure }; + if (agentArg) { + console.log(`running ${config.name} for: ${agentArg}\n`); + const env = { ...config.env, ...config.agentEnv?.get(agentArg) }; + const result = await runAgentStreaming(agentArg, { fixture: config.fixture, env }); + const validation = validateResult(result, config.validator, validateOptions); + console.log(); + printSingleValidation(validation); + printResults([validation]); + process.exit(validation.passed ? 0 : 1); + } else { + // run all agents + console.log(`running ${config.name} for all agents...\n`); + const results = await runAllAgentsStreaming({ + fixture: config.fixture, + env: config.env, + agentEnv: config.agentEnv, + }); + const validations = results.map((r) => validateResult(r, config.validator, validateOptions)); + console.log(); + for (const v of validations) { + printSingleValidation(v); + } + printResults(validations); + const allPassed = validations.every((v) => v.passed); + process.exit(allPassed ? 0 : 1); + } + } else { + // agnostic tests: run once with specified agent or default + const agent = agentArg ?? agents[0]; + console.log(`running ${config.name} for: ${agent}\n`); + + const validation = await runAgnosticTest(config, agent); + + console.log(); + printSingleValidation(validation); + printResults([validation]); + process.exit(validation.passed ? 0 : 1); + } +} + +main(); diff --git a/test/utils.ts b/test/utils.ts index 079365b..25f3e33 100644 --- a/test/utils.ts +++ b/test/utils.ts @@ -2,17 +2,12 @@ import { spawn } from "node:child_process"; import { randomUUID } from "node:crypto"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; -import { config } from "dotenv"; import { agentsManifest } from "../external.ts"; import type { Inputs } from "../main.ts"; const __dirname = dirname(fileURLToPath(import.meta.url)); export const actionDir = join(__dirname, ".."); -// load .env files -config({ path: join(actionDir, ".env") }); -config({ path: join(actionDir, "..", ".env") }); - const LOCAL_TEST_WARNING = "This is a local test - do not post any comments to GitHub."; export type FixtureOptions = { @@ -116,18 +111,25 @@ export interface RunOptions { env?: Record | undefined; } +const DEFAULT_TEST_TIMEOUT = "10m"; + // run agent and stream output with prefix labels export async function runAgentStreaming(agent: string, options: RunOptions): Promise { return new Promise((resolve) => { const chunks: Buffer[] = []; const prefix = getAgentPrefix(agent); - const child = spawn("node", ["play.ts"], { + // apply default timeout if not specified in fixture + const fixture: Inputs = { + ...options.fixture, + timeout: options.fixture.timeout ?? DEFAULT_TEST_TIMEOUT, + }; + + const child = spawn("node", ["play.ts", "--raw", JSON.stringify(fixture)], { cwd: actionDir, env: { ...process.env, AGENT_OVERRIDE: agent, - PLAY_FIXTURE: JSON.stringify(options.fixture), ...options.env, }, stdio: "pipe", @@ -174,12 +176,17 @@ export async function runAgent(agent: string, options: RunOptions): Promise { const chunks: Buffer[] = []; - const child = spawn("node", ["play.ts"], { + // apply default timeout if not specified in fixture + const fixture: Inputs = { + ...options.fixture, + timeout: options.fixture.timeout ?? DEFAULT_TEST_TIMEOUT, + }; + + const child = spawn("node", ["play.ts", "--raw", JSON.stringify(fixture)], { cwd: actionDir, env: { ...process.env, AGENT_OVERRIDE: agent, - PLAY_FIXTURE: JSON.stringify(options.fixture), ...options.env, }, stdio: "pipe", @@ -198,13 +205,29 @@ export async function runAgent(agent: string, options: RunOptions): Promise c.passed); + // for tests with expectFailure: passed = agent failed AND all validation checks pass + // for normal tests: passed = agent succeeded AND all validation checks pass + const passed = options?.expectFailure + ? !result.success && allPassed + : result.success && allPassed; + return { agent: result.agent, - passed: result.success && allPassed, + passed, checks, output: result.output, }; @@ -234,43 +257,9 @@ export interface TestRunnerOptions { env?: Record; // per-agent env vars (for unique markers) agentEnv?: Map>; -} - -export async function runTests(options: TestRunnerOptions): Promise { - const agentArg = process.argv[2]; - - if (agentArg) { - // single agent mode - if (!agents.includes(agentArg as (typeof agents)[number])) { - console.error(`unknown agent: ${agentArg}`); - console.error(`available agents: ${agents.join(", ")}`); - process.exit(1); - } - console.log(`running ${options.name} for: ${agentArg}\n`); - const env = { ...options.env, ...options.agentEnv?.get(agentArg) }; - const result = await runAgentStreaming(agentArg, { fixture: options.fixture, env }); - const validation = validateResult(result, options.validator); - console.log(); - printSingleValidation(validation); - process.exit(validation.passed ? 0 : 1); - } - - // parallel mode with streaming - console.log(`running ${options.name} for: ${agents.join(", ")}\n`); - - const results = await runAllAgentsStreaming({ - fixture: options.fixture, - env: options.env, - agentEnv: options.agentEnv, - }); - - console.log(); - const validations = results.map((r) => validateResult(r, options.validator)); - - printResults(validations); - - const failed = validations.filter((v) => !v.passed); - process.exit(failed.length > 0 ? 1 : 0); + // if true, test passes when agent fails AND validation checks pass + // (used for tests like timeout that expect the agent run to fail) + expectFailure?: boolean; } export function printSingleValidation(validation: ValidationResult): void { diff --git a/utils/buildPullfrogFooter.ts b/utils/buildPullfrogFooter.ts index d90cfcc..c3e5fe9 100644 --- a/utils/buildPullfrogFooter.ts +++ b/utils/buildPullfrogFooter.ts @@ -1,6 +1,6 @@ export const PULLFROG_DIVIDER = ""; -const FROG_LOGO = `Pullfrog`; +const FROG_LOGO = `Pullfrog`; export interface AgentInfo { displayName: string; diff --git a/utils/payload.test.ts b/utils/payload.test.ts index 12dc149..c1a3597 100644 --- a/utils/payload.test.ts +++ b/utils/payload.test.ts @@ -24,6 +24,10 @@ describe("Inputs schema", () => { ["effort", "mini"], ["effort", "auto"], ["effort", "max"], + ["timeout", "10m"], + ["timeout", "1h30m"], + ["timeout", "30s"], + ["timeout", undefined], ["agent", "claude"], ["agent", "codex"], ["agent", "cursor"], @@ -63,6 +67,9 @@ describe("JsonPayload schema", () => { ["effort", "mini"], ["effort", "auto"], ["effort", "max"], + ["timeout", "10m"], + ["timeout", "1h30m"], + ["timeout", "30s"], ["event", { trigger: "unknown" }], ] as const)("should accept optional %s with value %s", (prop, value) => { const input = { "~pullfrog": true, version: "1.2.3", [prop]: value }; diff --git a/utils/payload.ts b/utils/payload.ts index 8726e37..9532f96 100644 --- a/utils/payload.ts +++ b/utils/payload.ts @@ -22,6 +22,7 @@ export const JsonPayload = type({ "repoInstructions?": "string", "event?": "object", "effort?": Effort.or("undefined"), + "timeout?": type.string.or("undefined"), }); // permission levels that indicate collaborator status (have push access) @@ -41,6 +42,7 @@ function isCollaborator(event: PayloadEvent): boolean { export const Inputs = type({ prompt: "string", "effort?": Effort.or("undefined"), + "timeout?": type.string.or("undefined"), "agent?": AgentName.or("undefined"), "web?": ToolPermissionInput.or("undefined"), "search?": ToolPermissionInput.or("undefined"), @@ -70,6 +72,7 @@ export function resolvePayload(repoSettings: RepoSettings) { const inputs = Inputs.assert({ prompt: core.getInput("prompt", { required: true }), effort: core.getInput("effort") || undefined, + timeout: core.getInput("timeout") || undefined, agent: core.getInput("agent") || undefined, cwd: core.getInput("cwd") || undefined, web: core.getInput("web") || undefined, @@ -145,6 +148,7 @@ export function resolvePayload(repoSettings: RepoSettings) { repoInstructions: jsonPayload?.repoInstructions, event, effort: inputs.effort ?? jsonPayload?.effort ?? "auto", + timeout: inputs.timeout ?? jsonPayload?.timeout, cwd: resolveCwd(inputs.cwd), // permissions: inputs > repoSettings > fallbacks diff --git a/utils/time.test.ts b/utils/time.test.ts new file mode 100644 index 0000000..e91c987 --- /dev/null +++ b/utils/time.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, it } from "vitest"; +import { isValidTimeString, parseTimeString } from "./time.ts"; + +describe("parseTimeString", () => { + it.each([ + ["10m", 600000], // 10 minutes + ["1h", 3600000], // 1 hour + ["30s", 30000], // 30 seconds + ["1h30m", 5400000], // 1 hour 30 minutes + ["10m12s", 612000], // 10 minutes 12 seconds + ["1h30m45s", 5445000], // 1 hour 30 minutes 45 seconds + ["2h", 7200000], // 2 hours + ["90m", 5400000], // 90 minutes + ["0m", 0], // 0 minutes (edge case) + ["0s", 0], // 0 seconds (edge case) + ])("parses '%s' to %d ms", (input, expected) => { + expect(parseTimeString(input)).toBe(expected); + }); + + it.each([ + [""], // empty string + ["abc"], // no numbers + ["10"], // no unit + ["10x"], // invalid unit + ["h10m"], // hours without number + ["m10"], // units before number + ["10 m"], // space between number and unit + ["-10m"], // negative number + ["10.5m"], // decimal + ["10m 30s"], // space between components + ])("returns null for invalid input '%s'", (input) => { + expect(parseTimeString(input)).toBeNull(); + }); +}); + +describe("isValidTimeString", () => { + it.each(["10m", "1h", "30s", "1h30m", "10m12s", "1h30m45s"])( + "returns true for valid '%s'", + (input) => { + expect(isValidTimeString(input)).toBe(true); + } + ); + + it.each(["", "abc", "10", "10x", "-10m", "10.5m"])("returns false for invalid '%s'", (input) => { + expect(isValidTimeString(input)).toBe(false); + }); +}); diff --git a/utils/time.ts b/utils/time.ts new file mode 100644 index 0000000..48efb2a --- /dev/null +++ b/utils/time.ts @@ -0,0 +1,33 @@ +/** + * time string parsing utilities for timeout configuration. + * supports formats like "10m", "1h30m", "10m12s", "30s". + */ + +// special value indicating timeout is explicitly disabled via #notimeout macro +export const TIMEOUT_DISABLED = "none"; + +// time string regex: supports formats like "10m", "1h30m", "10m12s", "30s" +// at least one component (hours, minutes, or seconds) is required +const TIME_STRING_REGEX = /^(?:(\d+)h)?(?:(\d+)m)?(?:(\d+)s)?$/; + +/** + * parse a time string like "10m", "1h30m", "10m12s" into milliseconds. + * returns null if the string is not a valid time format. + */ +export function parseTimeString(input: string): number | null { + const match = input.match(TIME_STRING_REGEX); + if (!match || (!match[1] && !match[2] && !match[3])) return null; + + const hours = parseInt(match[1] || "0", 10); + const minutes = parseInt(match[2] || "0", 10); + const seconds = parseInt(match[3] || "0", 10); + + return (hours * 3600 + minutes * 60 + seconds) * 1000; +} + +/** + * check if a string is a valid time format. + */ +export function isValidTimeString(input: string): boolean { + return parseTimeString(input) !== null; +}