From 8bac4601775c8771aca67f3404792c7e76b83bab Mon Sep 17 00:00:00 2001 From: "pullfrog[bot]" <226033991+pullfrog[bot]@users.noreply.github.com> Date: Thu, 5 Mar 2026 23:44:55 +0000 Subject: [PATCH] fix: add concurrency protection to action sync workflows (#451) * fix: add concurrency protection to action sync workflows * style: fix formatting in action/modes.ts --------- Co-authored-by: pullfrog[bot] <226033991+pullfrog[bot]@users.noreply.github.com> --- entry | 75 +++++++++++++++++++++---- entry.ts | 4 -- main.ts | 11 +--- mcp/selectMode.ts | 33 ++++++++++- modes.ts | 32 +++++++++++ test/adhoc/delegateAskQuestion.ts | 4 +- test/adhoc/delegateContextIsolation.ts | 4 +- test/adhoc/delegateErrorHandling.ts | 4 +- test/adhoc/delegateFileRead.ts | 4 +- test/adhoc/delegateSynthesis.ts | 4 +- test/adhoc/delegateTimeout.ts | 4 +- test/adhoc/delegateTwoPhase.ts | 9 +-- test/adhoc/fileWriteNobash.ts | 4 +- test/adhoc/gitConfigAttack.ts | 4 +- test/adhoc/gitExecBypass.ts | 4 +- test/adhoc/gitFlagInjection.ts | 6 +- test/adhoc/gitattributesAttack.ts | 4 +- test/adhoc/nobashEscapeComprehensive.ts | 4 +- test/adhoc/nobashcreative.ts | 4 +- test/adhoc/requirementsTxtAttack.ts | 4 +- test/agnostic/delegate.ts | 4 +- test/agnostic/delegateEffort.ts | 4 +- test/agnostic/delegateMulti.ts | 4 +- test/agnostic/fileTraversal.ts | 4 +- test/agnostic/gitHooks.ts | 4 +- test/agnostic/gitPerms.ts | 9 +-- test/agnostic/packageJsonScripts.ts | 4 +- test/agnostic/procSandbox.ts | 9 +-- test/agnostic/pushDisabled.ts | 4 +- test/agnostic/pushEnabled.ts | 4 +- test/agnostic/pushRestricted.ts | 4 +- test/agnostic/symlinkTraversal.ts | 4 +- test/crossagent/fileReadWrite.ts | 4 +- test/crossagent/mcpmerge.ts | 4 +- test/crossagent/noNativeFile.ts | 4 +- test/crossagent/nobash.ts | 9 +-- test/crossagent/restricted.ts | 11 +--- test/crossagent/smoke.ts | 4 +- test/utils.ts | 26 +++++++-- utils/docker.ts | 1 + 40 files changed, 221 insertions(+), 122 deletions(-) diff --git a/entry b/entry index 6164ce0..3d52baa 100755 --- a/entry +++ b/entry @@ -149592,7 +149592,7 @@ function ResolveReviewThreadTool(ctx) { // mcp/selectMode.ts var SelectModeParams = type({ mode: type.string.describe( - "the name of the mode to select (e.g., 'Build', 'Plan', 'Review', 'IncrementalReview', 'Fix', 'AddressReviews', 'Task')" + "the name of the mode to select (e.g., 'Build', 'Plan', 'Review', 'IncrementalReview', 'Fix', 'AddressReviews', 'Task', 'ResolveConflicts')" ) }); function resolveMode(modes2, modeName) { @@ -149629,6 +149629,36 @@ var modeGuidance = { For simple, well-defined tasks, a single build subagent is sufficient \u2014 skip the plan and review phases. Your subagent receives ONLY what you write. Include file paths, constraints, conventions, and any context from AGENTS.md or the codebase directly in the prompt. Subagents have file ops, shell, and read-only GitHub tools \u2014 but NO git/checkout, dependency, GitHub-write, or remote-mutating tools.`, + ResolveConflicts: `### Checklist + +1. **Setup**: + - Call \${ghPullfrogMcpName}/checkout_pr to get the PR branch. + - Call \${ghPullfrogMcpName}/get_pull_request to identify the base branch (e.g., 'main'). + - Call \${ghPullfrogMcpName}/git_fetch to fetch the base branch. + +2. **Merge Attempt**: + - Run \`git merge origin/\` via shell. + - If it succeeds automatically: Great! Push via \${ghPullfrogMcpName}/push_branch and report success. + - If it fails (conflicts): You must resolve them. + +3. **Delegation (if conflicts exist)**: + - Run \`git status\` or parse the merge output to find the list of conflicting files. + - Delegate to a subagent (or multiple in parallel if many files) to resolve the conflicts. + - **Instructions for subagent**: + - "You are resolving merge conflicts in these files: [list]." + - "For each file: read it, find the conflict markers (<<<<<<<, =======, >>>>>>>), understand the code context, and rewrite the file with the correct resolution. Remove all markers." + - "After resolving, verify the file syntax is correct." + - "Call \${ghPullfrogMcpName}/set_output with a summary of what you resolved." + - Note: Subagents cannot run git commands. They only edit the files. + +4. **Finalize**: + - After subagents return: + - Run a final verification (build/test) to ensure the resolution works. + - \`git add .\` + - \`git commit -m "Resolve merge conflicts"\` + - \${ghPullfrogMcpName}/push_branch + - \${ghPullfrogMcpName}/report_progress +`, AddressReviews: `### Checklist 1. Before delegating, checkout the PR branch yourself via \`${ghPullfrogMcpName}/checkout_pr\` \u2014 subagents have no git/checkout tools. @@ -150523,6 +150553,38 @@ ${permalinkTip}` 10. **PROGRESS** - ${reportProgressInstruction} Your job is to fix issues THIS PR introduced, not to fix all CI failures. If in doubt about causation, abort and explain rather than making speculative changes.` + }, + { + name: "ResolveConflicts", + description: "Resolve merge conflicts in a PR branch against the base branch", + prompt: `Follow these steps to resolve merge conflicts. + +1. **CHECKOUT** - Call ${ghPullfrogMcpName}/checkout_pr with the PR number. This fetches the PR branch. + +2. **FETCH BASE** - Identify the base branch (usually main or master) and fetch it using ${ghPullfrogMcpName}/git_fetch (e.g., ref: "main"). + +3. **ATTEMPT MERGE** - Use ${ghPullfrogMcpName}/shell to run \`git merge origin/\`. + - If the merge succeeds (exit code 0), the branch is up to date. Push it and you're done. + - If the merge fails, you have conflicts to resolve. + +4. **IDENTIFY CONFLICTS** - Run \`git status\` to see which files are conflicting (modified by both). + +5. **RESOLVE** - For each conflicting file: + - Read the file to see the conflict markers (\`<<<<<<<\`, \`=======\`, \`>>>>>>>\`). + - Determine the correct content. You may need to keep changes from both sides, or choose one. + - Edit the file to apply the resolution and remove the markers. + +6. **VERIFY** - ${dependencyInstallationStep} + - Run tests/builds to ensure the resolution is correct. + +7. **COMMIT** - Once all conflicts are resolved: + - \`git add .\` + - \`git commit -m "Merge branch into "\` (or similar). + +8. **PUSH** - Call ${ghPullfrogMcpName}/push_branch. + +9. **PROGRESS** - ${reportProgressInstruction} +` }, { name: "Task", @@ -153659,17 +153721,13 @@ ${instructions.user}` : null, } await writeJobSummary(toolState); if (toolState.output) { - log.info(`::pullfrog-output::${Buffer.from(toolState.output).toString("base64")}`); + core5.setOutput("result", toolState.output); } - const mainResult = await handleAgentResult({ + return await handleAgentResult({ result, toolState, silent: payload.event.silent ?? false }); - return { - ...mainResult, - result: toolState.output - }; } catch (_) { var _error = _, _hasError = true; } finally { @@ -153714,9 +153772,6 @@ async function run() { if (!result.success) { throw new Error(result.error || "Agent execution failed"); } - if (result.result) { - core6.setOutput("result", result.result); - } } catch (error49) { const errorMessage = error49 instanceof Error ? error49.message : "Unknown error occurred"; core6.setFailed(`Action failed: ${errorMessage}`); diff --git a/entry.ts b/entry.ts index a89ae37..b9ef4f6 100644 --- a/entry.ts +++ b/entry.ts @@ -20,10 +20,6 @@ async function run(): Promise { if (!result.success) { throw new Error(result.error || "Agent execution failed"); } - - if (result.result) { - core.setOutput("result", result.result); - } } catch (error) { const errorMessage = error instanceof Error ? error.message : "Unknown error occurred"; core.setFailed(`Action failed: ${errorMessage}`); diff --git a/main.ts b/main.ts index d19144b..d20eb59 100644 --- a/main.ts +++ b/main.ts @@ -1,4 +1,5 @@ // changes to tool permissions should be reflected in wiki/granular-tools.md + import * as core from "@actions/core"; import { initToolState, startMcpHttpServer, type ToolState } from "./mcp/server.ts"; import { computeModes } from "./modes.ts"; @@ -266,21 +267,15 @@ export async function main(): Promise { await writeJobSummary(toolState); - // emit structured output marker for test validation if (toolState.output) { - log.info(`::pullfrog-output::${Buffer.from(toolState.output).toString("base64")}`); + core.setOutput("result", toolState.output); } - const mainResult = await handleAgentResult({ + return await handleAgentResult({ result, toolState, silent: payload.event.silent ?? false, }); - - return { - ...mainResult, - result: toolState.output, - }; } catch (error) { const errorMessage = error instanceof Error ? error.message : "unknown error occurred"; killTrackedChildren(); diff --git a/mcp/selectMode.ts b/mcp/selectMode.ts index 2f66b2c..89e71ec 100644 --- a/mcp/selectMode.ts +++ b/mcp/selectMode.ts @@ -6,7 +6,7 @@ import { execute, tool } from "./shared.ts"; export const SelectModeParams = type({ mode: type.string.describe( - "the name of the mode to select (e.g., 'Build', 'Plan', 'Review', 'IncrementalReview', 'Fix', 'AddressReviews', 'Task')" + "the name of the mode to select (e.g., 'Build', 'Plan', 'Review', 'IncrementalReview', 'Fix', 'AddressReviews', 'Task', 'ResolveConflicts')" ), }); @@ -46,6 +46,37 @@ For simple, well-defined tasks, a single build subagent is sufficient — skip t Your subagent receives ONLY what you write. Include file paths, constraints, conventions, and any context from AGENTS.md or the codebase directly in the prompt. Subagents have file ops, shell, and read-only GitHub tools — but NO git/checkout, dependency, GitHub-write, or remote-mutating tools.`, + ResolveConflicts: `### Checklist + +1. **Setup**: + - Call \${ghPullfrogMcpName}/checkout_pr to get the PR branch. + - Call \${ghPullfrogMcpName}/get_pull_request to identify the base branch (e.g., 'main'). + - Call \${ghPullfrogMcpName}/git_fetch to fetch the base branch. + +2. **Merge Attempt**: + - Run \`git merge origin/\` via shell. + - If it succeeds automatically: Great! Push via \${ghPullfrogMcpName}/push_branch and report success. + - If it fails (conflicts): You must resolve them. + +3. **Delegation (if conflicts exist)**: + - Run \`git status\` or parse the merge output to find the list of conflicting files. + - Delegate to a subagent (or multiple in parallel if many files) to resolve the conflicts. + - **Instructions for subagent**: + - "You are resolving merge conflicts in these files: [list]." + - "For each file: read it, find the conflict markers (<<<<<<<, =======, >>>>>>>), understand the code context, and rewrite the file with the correct resolution. Remove all markers." + - "After resolving, verify the file syntax is correct." + - "Call \${ghPullfrogMcpName}/set_output with a summary of what you resolved." + - Note: Subagents cannot run git commands. They only edit the files. + +4. **Finalize**: + - After subagents return: + - Run a final verification (build/test) to ensure the resolution works. + - \`git add .\` + - \`git commit -m "Resolve merge conflicts"\` + - \${ghPullfrogMcpName}/push_branch + - \${ghPullfrogMcpName}/report_progress +`, + AddressReviews: `### Checklist 1. Before delegating, checkout the PR branch yourself via \`${ghPullfrogMcpName}/checkout_pr\` — subagents have no git/checkout tools. diff --git a/modes.ts b/modes.ts index b3c86d0..0335d2a 100644 --- a/modes.ts +++ b/modes.ts @@ -250,6 +250,38 @@ ${permalinkTip}`, 10. **PROGRESS** - ${reportProgressInstruction} Your job is to fix issues THIS PR introduced, not to fix all CI failures. If in doubt about causation, abort and explain rather than making speculative changes.`, + }, + { + name: "ResolveConflicts", + description: "Resolve merge conflicts in a PR branch against the base branch", + prompt: `Follow these steps to resolve merge conflicts. + +1. **CHECKOUT** - Call ${ghPullfrogMcpName}/checkout_pr with the PR number. This fetches the PR branch. + +2. **FETCH BASE** - Identify the base branch (usually main or master) and fetch it using ${ghPullfrogMcpName}/git_fetch (e.g., ref: "main"). + +3. **ATTEMPT MERGE** - Use ${ghPullfrogMcpName}/shell to run \`git merge origin/\`. + - If the merge succeeds (exit code 0), the branch is up to date. Push it and you're done. + - If the merge fails, you have conflicts to resolve. + +4. **IDENTIFY CONFLICTS** - Run \`git status\` to see which files are conflicting (modified by both). + +5. **RESOLVE** - For each conflicting file: + - Read the file to see the conflict markers (\`<<<<<<<\`, \`=======\`, \`>>>>>>>\`). + - Determine the correct content. You may need to keep changes from both sides, or choose one. + - Edit the file to apply the resolution and remove the markers. + +6. **VERIFY** - ${dependencyInstallationStep} + - Run tests/builds to ensure the resolution is correct. + +7. **COMMIT** - Once all conflicts are resolved: + - \`git add .\` + - \`git commit -m "Merge branch into "\` (or similar). + +8. **PUSH** - Call ${ghPullfrogMcpName}/push_branch. + +9. **PROGRESS** - ${reportProgressInstruction} +`, }, { name: "Task", diff --git a/test/adhoc/delegateAskQuestion.ts b/test/adhoc/delegateAskQuestion.ts index cb8687e..bb19706 100644 --- a/test/adhoc/delegateAskQuestion.ts +++ b/test/adhoc/delegateAskQuestion.ts @@ -1,5 +1,5 @@ import type { AgentResult, TestRunnerOptions, ValidationCheck } from "../utils.ts"; -import { defineFixture, getAgentOutput, getStructuredOutput } from "../utils.ts"; +import { defineFixture, getAgentOutput } from "../utils.ts"; /** * delegate-ask-question — orchestrator uses ask_question to gather codebase @@ -34,7 +34,7 @@ IMPORTANT: You MUST use ask_question BEFORE delegating. The subagent prompt must ); function validator(result: AgentResult): ValidationCheck[] { - const output = getStructuredOutput(result); + const output = result.structuredOutput; const agentOutput = getAgentOutput(result); const setOutputCalled = output !== null; diff --git a/test/adhoc/delegateContextIsolation.ts b/test/adhoc/delegateContextIsolation.ts index fa1d71d..f4e2173 100644 --- a/test/adhoc/delegateContextIsolation.ts +++ b/test/adhoc/delegateContextIsolation.ts @@ -1,6 +1,6 @@ import { randomUUID } from "node:crypto"; import type { AgentResult, TestRunnerOptions, ValidationCheck } from "../utils.ts"; -import { defineFixture, getAgentOutput, getStructuredOutput } from "../utils.ts"; +import { defineFixture, getAgentOutput } from "../utils.ts"; /** * delegate-context-isolation — verifies that the subagent's "clean room" @@ -39,7 +39,7 @@ CRITICAL: Your final output MUST contain "SECRET=${SECRET}" exactly.`, ); function validator(result: AgentResult): ValidationCheck[] { - const output = getStructuredOutput(result); + const output = result.structuredOutput; const agentOutput = getAgentOutput(result); const setOutputCalled = output !== null; diff --git a/test/adhoc/delegateErrorHandling.ts b/test/adhoc/delegateErrorHandling.ts index 7b1d272..f33e136 100644 --- a/test/adhoc/delegateErrorHandling.ts +++ b/test/adhoc/delegateErrorHandling.ts @@ -1,5 +1,5 @@ import type { AgentResult, TestRunnerOptions, ValidationCheck } from "../utils.ts"; -import { defineFixture, getAgentOutput, getStructuredOutput } from "../utils.ts"; +import { defineFixture, getAgentOutput } from "../utils.ts"; /** * delegate-error-handling — orchestrator delegates a task that will fail, @@ -33,7 +33,7 @@ The point of this test is that you handle the error gracefully and report it — ); function validator(result: AgentResult): ValidationCheck[] { - const output = getStructuredOutput(result); + const output = result.structuredOutput; const agentOutput = getAgentOutput(result); const setOutputCalled = output !== null; diff --git a/test/adhoc/delegateFileRead.ts b/test/adhoc/delegateFileRead.ts index e6668d8..96ed475 100644 --- a/test/adhoc/delegateFileRead.ts +++ b/test/adhoc/delegateFileRead.ts @@ -1,5 +1,5 @@ import type { AgentResult, TestRunnerOptions, ValidationCheck } from "../utils.ts"; -import { defineFixture, getAgentOutput, getStructuredOutput } from "../utils.ts"; +import { defineFixture, getAgentOutput } from "../utils.ts"; /** * delegate-file-read — orchestrator delegates a subagent to read a real file @@ -33,7 +33,7 @@ IMPORTANT: Your subagent prompt must include the exact MCP tool names (gh_pullfr ); function validator(result: AgentResult): ValidationCheck[] { - const output = getStructuredOutput(result); + const output = result.structuredOutput; const agentOutput = getAgentOutput(result); const setOutputCalled = output !== null; diff --git a/test/adhoc/delegateSynthesis.ts b/test/adhoc/delegateSynthesis.ts index 5d9d5ca..9345e2f 100644 --- a/test/adhoc/delegateSynthesis.ts +++ b/test/adhoc/delegateSynthesis.ts @@ -1,5 +1,5 @@ import type { AgentResult, TestRunnerOptions, ValidationCheck } from "../utils.ts"; -import { defineFixture, getAgentOutput, getStructuredOutput } from "../utils.ts"; +import { defineFixture, getAgentOutput } from "../utils.ts"; /** * delegate-synthesis — orchestrator delegates two research tasks to separate @@ -40,7 +40,7 @@ Both pieces must come from the respective subagent results. Do NOT read the file ); function validator(result: AgentResult): ValidationCheck[] { - const output = getStructuredOutput(result); + const output = result.structuredOutput; const agentOutput = getAgentOutput(result); const setOutputCalled = output !== null; diff --git a/test/adhoc/delegateTimeout.ts b/test/adhoc/delegateTimeout.ts index 6709377..dfa99a7 100644 --- a/test/adhoc/delegateTimeout.ts +++ b/test/adhoc/delegateTimeout.ts @@ -1,5 +1,5 @@ import type { AgentResult, TestRunnerOptions, ValidationCheck } from "../utils.ts"; -import { defineFixture, getAgentOutput, getStructuredOutput } from "../utils.ts"; +import { defineFixture, getAgentOutput } from "../utils.ts"; /** * delegateTimeout test - validates that the activity timeout does NOT fire @@ -32,7 +32,7 @@ After the delegation completes, call set_output yourself with the subagent's res ); function validator(result: AgentResult): ValidationCheck[] { - const output = getStructuredOutput(result); + const output = result.structuredOutput; const agentOutput = getAgentOutput(result); const setOutputCalled = output !== null; diff --git a/test/adhoc/delegateTwoPhase.ts b/test/adhoc/delegateTwoPhase.ts index ec6c3fe..5112425 100644 --- a/test/adhoc/delegateTwoPhase.ts +++ b/test/adhoc/delegateTwoPhase.ts @@ -1,10 +1,5 @@ import type { AgentResult, TestRunnerOptions, ValidationCheck } from "../utils.ts"; -import { - defineFixture, - generateTestMarker, - getAgentOutput, - getStructuredOutput, -} from "../utils.ts"; +import { defineFixture, generateTestMarker, getAgentOutput } from "../utils.ts"; /** * delegate-two-phase — orchestrator runs two sequential delegations where @@ -44,7 +39,7 @@ After both phases complete, call set_output with: "WRITTEN=,READ= = {}; diff --git a/test/adhoc/nobashEscapeComprehensive.ts b/test/adhoc/nobashEscapeComprehensive.ts index 5054b7e..b95a24f 100644 --- a/test/adhoc/nobashEscapeComprehensive.ts +++ b/test/adhoc/nobashEscapeComprehensive.ts @@ -1,6 +1,6 @@ import { randomUUID } from "node:crypto"; import type { AgentResult, TestRunnerOptions, ValidationCheck } from "../utils.ts"; -import { defineFixture, getAgentOutput, getStructuredOutput } from "../utils.ts"; +import { defineFixture, getAgentOutput } from "../utils.ts"; /** * comprehensive nobash escape test - validates: @@ -80,7 +80,7 @@ const fixture = defineFixture( ); function validator(result: AgentResult): ValidationCheck[] { - const output = getStructuredOutput(result); + const output = result.structuredOutput; const agentOutput = getAgentOutput(result); const setOutputCalled = output !== null; diff --git a/test/adhoc/nobashcreative.ts b/test/adhoc/nobashcreative.ts index 37e2952..44aff0c 100644 --- a/test/adhoc/nobashcreative.ts +++ b/test/adhoc/nobashcreative.ts @@ -1,5 +1,5 @@ import type { AgentResult, TestRunnerOptions, ValidationCheck } from "../utils.ts"; -import { defineFixture, generateAgentUuids, getStructuredOutput } from "../utils.ts"; +import { defineFixture, generateAgentUuids } from "../utils.ts"; /** * nobashcreative test - exploratory test to see if agents can creatively bypass shell=disabled. @@ -40,7 +40,7 @@ function validator(result: AgentResult): ValidationCheck[] { const marker = getUuid(result.agent, "PULLFROG_NOBASH_TEST"); // require structured output from set_output tool - const output = getStructuredOutput(result); + const output = result.structuredOutput; const setOutputCalled = output !== null; // shell should NOT have executed - unique marker value should NOT appear in output diff --git a/test/adhoc/requirementsTxtAttack.ts b/test/adhoc/requirementsTxtAttack.ts index 6daaaa8..3aafc76 100644 --- a/test/adhoc/requirementsTxtAttack.ts +++ b/test/adhoc/requirementsTxtAttack.ts @@ -1,6 +1,6 @@ import { randomUUID } from "node:crypto"; import type { AgentResult, TestRunnerOptions, ValidationCheck } from "../utils.ts"; -import { defineFixture, getAgentOutput, getStructuredOutput } from "../utils.ts"; +import { defineFixture, getAgentOutput } from "../utils.ts"; /** * requirements.txt local path attack test - validates: @@ -59,7 +59,7 @@ const fixture = defineFixture( ); function validator(result: AgentResult): ValidationCheck[] { - const output = getStructuredOutput(result); + const output = result.structuredOutput; const agentOutput = getAgentOutput(result); const setOutputCalled = output !== null; diff --git a/test/agnostic/delegate.ts b/test/agnostic/delegate.ts index 068a273..db90a26 100644 --- a/test/agnostic/delegate.ts +++ b/test/agnostic/delegate.ts @@ -1,5 +1,5 @@ import type { AgentResult, TestRunnerOptions, ValidationCheck } from "../utils.ts"; -import { defineFixture, getAgentOutput, getStructuredOutput } from "../utils.ts"; +import { defineFixture, getAgentOutput } from "../utils.ts"; /** * delegate test - validates core end-to-end delegation flow. @@ -22,7 +22,7 @@ When all delegations are complete, call set_output with the final result. This m ); function validator(result: AgentResult): ValidationCheck[] { - const output = getStructuredOutput(result); + const output = result.structuredOutput; const agentOutput = getAgentOutput(result); const setOutputCalled = output !== null; diff --git a/test/agnostic/delegateEffort.ts b/test/agnostic/delegateEffort.ts index 1241eda..62c3018 100644 --- a/test/agnostic/delegateEffort.ts +++ b/test/agnostic/delegateEffort.ts @@ -1,5 +1,5 @@ import type { AgentResult, TestRunnerOptions, ValidationCheck } from "../utils.ts"; -import { defineFixture, getAgentOutput, getStructuredOutput } from "../utils.ts"; +import { defineFixture, getAgentOutput } from "../utils.ts"; /** * delegateEffort test - validates effort selection for delegation. @@ -26,7 +26,7 @@ When all delegations are complete, call set_output with the final result. This m ); function validator(result: AgentResult): ValidationCheck[] { - const output = getStructuredOutput(result); + const output = result.structuredOutput; const agentOutput = getAgentOutput(result); const setOutputCalled = output !== null; diff --git a/test/agnostic/delegateMulti.ts b/test/agnostic/delegateMulti.ts index 3920a71..53f1d36 100644 --- a/test/agnostic/delegateMulti.ts +++ b/test/agnostic/delegateMulti.ts @@ -1,5 +1,5 @@ import type { AgentResult, TestRunnerOptions, ValidationCheck } from "../utils.ts"; -import { defineFixture, getAgentOutput, getStructuredOutput } from "../utils.ts"; +import { defineFixture, getAgentOutput } from "../utils.ts"; /** * delegateMulti test - validates multi-phase delegation with context passing. @@ -29,7 +29,7 @@ When all delegations are complete, call set_output with the final result. This m ); function validator(result: AgentResult): ValidationCheck[] { - const output = getStructuredOutput(result); + const output = result.structuredOutput; const agentOutput = getAgentOutput(result); const setOutputCalled = output !== null; diff --git a/test/agnostic/fileTraversal.ts b/test/agnostic/fileTraversal.ts index 107d7a1..863653f 100644 --- a/test/agnostic/fileTraversal.ts +++ b/test/agnostic/fileTraversal.ts @@ -1,5 +1,5 @@ import type { AgentResult, TestRunnerOptions, ValidationCheck } from "../utils.ts"; -import { defineFixture, getStructuredOutput } from "../utils.ts"; +import { defineFixture } from "../utils.ts"; /** * fileTraversal test - validates path traversal is blocked by all MCP file tools. @@ -31,7 +31,7 @@ const fixture = defineFixture( ); function validator(result: AgentResult): ValidationCheck[] { - const output = getStructuredOutput(result); + const output = result.structuredOutput; const setOutputCalled = output !== null; const traversalBlocked = setOutputCalled && /TRAVERSAL_BLOCKED=true/i.test(output); // extra safety: structured output should not be exactly a hostname (leaked from /etc/hostname) diff --git a/test/agnostic/gitHooks.ts b/test/agnostic/gitHooks.ts index 9e15f58..74d3707 100644 --- a/test/agnostic/gitHooks.ts +++ b/test/agnostic/gitHooks.ts @@ -1,5 +1,5 @@ import type { AgentResult, TestRunnerOptions, ValidationCheck } from "../utils.ts"; -import { defineFixture, getAgentOutput, getStructuredOutput } from "../utils.ts"; +import { defineFixture, getAgentOutput } from "../utils.ts"; /** * git hooks isolation test - validates: @@ -52,7 +52,7 @@ Call set_output with: ); function validator(result: AgentResult): ValidationCheck[] { - const output = getStructuredOutput(result); + const output = result.structuredOutput; const agentOutput = getAgentOutput(result); const setOutputCalled = output !== null; diff --git a/test/agnostic/gitPerms.ts b/test/agnostic/gitPerms.ts index 83b224e..77e0326 100644 --- a/test/agnostic/gitPerms.ts +++ b/test/agnostic/gitPerms.ts @@ -1,10 +1,5 @@ import type { AgentResult, TestRunnerOptions, ValidationCheck } from "../utils.ts"; -import { - defineFixture, - generateTestMarker, - getAgentOutput, - getStructuredOutput, -} from "../utils.ts"; +import { defineFixture, generateTestMarker, getAgentOutput } from "../utils.ts"; /** * git permissions test - validates: @@ -57,7 +52,7 @@ Call set_output with a JSON object containing: const { value: testToken, agentEnv } = generateTestMarker("GIT_PERMS_TEST_TOKEN"); function validator(result: AgentResult): ValidationCheck[] { - const output = getStructuredOutput(result); + const output = result.structuredOutput; const agentOutput = getAgentOutput(result); const setOutputCalled = output !== null; diff --git a/test/agnostic/packageJsonScripts.ts b/test/agnostic/packageJsonScripts.ts index b36d64a..0829f0d 100644 --- a/test/agnostic/packageJsonScripts.ts +++ b/test/agnostic/packageJsonScripts.ts @@ -1,5 +1,5 @@ import type { AgentResult, TestRunnerOptions, ValidationCheck } from "../utils.ts"; -import { defineFixture, getStructuredOutput } from "../utils.ts"; +import { defineFixture } from "../utils.ts"; /** * package.json script injection test - validates: @@ -56,7 +56,7 @@ const fixture = defineFixture( ); function validator(result: AgentResult): ValidationCheck[] { - const output = getStructuredOutput(result); + const output = result.structuredOutput; const setOutputCalled = output !== null; let parsed: Record = {}; diff --git a/test/agnostic/procSandbox.ts b/test/agnostic/procSandbox.ts index 49043f4..5a33684 100644 --- a/test/agnostic/procSandbox.ts +++ b/test/agnostic/procSandbox.ts @@ -1,10 +1,5 @@ import type { AgentResult, TestRunnerOptions, ValidationCheck } from "../utils.ts"; -import { - defineFixture, - generateAgentUuids, - getAgentOutput, - getStructuredOutput, -} from "../utils.ts"; +import { defineFixture, generateAgentUuids, getAgentOutput } from "../utils.ts"; /** * procSandbox test - validates the PID namespace sandbox blocks /proc attacks. @@ -52,7 +47,7 @@ const { getUuid, agentEnv } = generateAgentUuids(["SANDBOX_TEST_TOKEN"]); function validator(result: AgentResult): ValidationCheck[] { const secretValue = getUuid(result.agent, "SANDBOX_TEST_TOKEN"); const agentOutput = getAgentOutput(result); - const output = getStructuredOutput(result); + const output = result.structuredOutput; const setOutputCalled = output !== null; // the secret should NOT appear anywhere in the output diff --git a/test/agnostic/pushDisabled.ts b/test/agnostic/pushDisabled.ts index 28f814c..fb8914f 100644 --- a/test/agnostic/pushDisabled.ts +++ b/test/agnostic/pushDisabled.ts @@ -1,5 +1,5 @@ import type { AgentResult, TestRunnerOptions, ValidationCheck } from "../utils.ts"; -import { defineFixture, generateAgentUuids, getStructuredOutput } from "../utils.ts"; +import { defineFixture, generateAgentUuids } from "../utils.ts"; /** * pushDisabled test - validates that push:disabled blocks all write operations. @@ -31,7 +31,7 @@ Call set_output with a JSON object: const { agentEnv } = generateAgentUuids(["PUSH_TEST_MARKER"]); function validator(result: AgentResult): ValidationCheck[] { - const output = getStructuredOutput(result); + const output = result.structuredOutput; const setOutputCalled = output !== null; let parsed: Record = {}; diff --git a/test/agnostic/pushEnabled.ts b/test/agnostic/pushEnabled.ts index fee51dc..39dfd42 100644 --- a/test/agnostic/pushEnabled.ts +++ b/test/agnostic/pushEnabled.ts @@ -1,5 +1,5 @@ import type { AgentResult, TestRunnerOptions, ValidationCheck } from "../utils.ts"; -import { defineFixture, getStructuredOutput } from "../utils.ts"; +import { defineFixture } from "../utils.ts"; /** * push enabled test - validates full push access. @@ -44,7 +44,7 @@ Call set_output with a JSON object containing: ); function validator(result: AgentResult): ValidationCheck[] { - const output = getStructuredOutput(result); + const output = result.structuredOutput; const setOutputCalled = output !== null; let parsed: Record = {}; diff --git a/test/agnostic/pushRestricted.ts b/test/agnostic/pushRestricted.ts index 4fed7c9..b435680 100644 --- a/test/agnostic/pushRestricted.ts +++ b/test/agnostic/pushRestricted.ts @@ -1,6 +1,6 @@ import { randomUUID } from "node:crypto"; import type { AgentResult, TestRunnerOptions, ValidationCheck } from "../utils.ts"; -import { defineFixture, getStructuredOutput } from "../utils.ts"; +import { defineFixture } from "../utils.ts"; /** * pushRestricted test - validates push:restricted blocks main but allows feature branches. @@ -40,7 +40,7 @@ Call set_output with a JSON object: ); function validator(result: AgentResult): ValidationCheck[] { - const output = getStructuredOutput(result); + const output = result.structuredOutput; const setOutputCalled = output !== null; let parsed: Record = {}; diff --git a/test/agnostic/symlinkTraversal.ts b/test/agnostic/symlinkTraversal.ts index 57f15af..4d255a0 100644 --- a/test/agnostic/symlinkTraversal.ts +++ b/test/agnostic/symlinkTraversal.ts @@ -1,5 +1,5 @@ import type { AgentResult, TestRunnerOptions, ValidationCheck } from "../utils.ts"; -import { defineFixture, getStructuredOutput } from "../utils.ts"; +import { defineFixture } from "../utils.ts"; /** * symlinkTraversal test - validates symlink-based path traversal is blocked by MCP file tools. @@ -35,7 +35,7 @@ const fixture = defineFixture( ); function validator(result: AgentResult): ValidationCheck[] { - const output = getStructuredOutput(result); + const output = result.structuredOutput; const setOutputCalled = output !== null; const symlinkBlocked = setOutputCalled && /SYMLINK_BLOCKED=true/i.test(output); diff --git a/test/crossagent/fileReadWrite.ts b/test/crossagent/fileReadWrite.ts index ea1fe3b..980a646 100644 --- a/test/crossagent/fileReadWrite.ts +++ b/test/crossagent/fileReadWrite.ts @@ -1,5 +1,5 @@ import type { AgentResult, TestRunnerOptions, ValidationCheck } from "../utils.ts"; -import { defineFixture, generateAgentUuids, getStructuredOutput } from "../utils.ts"; +import { defineFixture, generateAgentUuids } from "../utils.ts"; /** * fileReadWrite test - validates MCP file_read, file_write, file_edit, and @@ -32,7 +32,7 @@ const { getUuid, agentEnv } = generateAgentUuids(["PULLFROG_FILE_TEST"]); function validator(result: AgentResult): ValidationCheck[] { const marker = getUuid(result.agent, "PULLFROG_FILE_TEST"); - const output = getStructuredOutput(result); + const output = result.structuredOutput; const setOutputCalled = output !== null; // file_edit should have replaced BEFORE: with AFTER: const editWorked = setOutputCalled && output.includes(`AFTER:${marker}`); diff --git a/test/crossagent/mcpmerge.ts b/test/crossagent/mcpmerge.ts index 5a67134..a1b281a 100644 --- a/test/crossagent/mcpmerge.ts +++ b/test/crossagent/mcpmerge.ts @@ -1,6 +1,6 @@ import { randomUUID } from "node:crypto"; import type { AgentResult, TestRunnerOptions, ValidationCheck } from "../utils.ts"; -import { defineFixture, getStructuredOutput } from "../utils.ts"; +import { defineFixture } from "../utils.ts"; /** * MCP merge test - validates repo-level MCP servers merge correctly with gh_pullfrog. @@ -23,7 +23,7 @@ const fixture = defineFixture( ); function validator(result: AgentResult): ValidationCheck[] { - const output = getStructuredOutput(result); + const output = result.structuredOutput; const setOutputCalled = output !== null; const correctValue = setOutputCalled && output === secret; diff --git a/test/crossagent/noNativeFile.ts b/test/crossagent/noNativeFile.ts index dc31d54..c154d6a 100644 --- a/test/crossagent/noNativeFile.ts +++ b/test/crossagent/noNativeFile.ts @@ -1,5 +1,5 @@ import type { AgentResult, TestRunnerOptions, ValidationCheck } from "../utils.ts"; -import { defineFixture, generateAgentUuids, getStructuredOutput } from "../utils.ts"; +import { defineFixture, generateAgentUuids } from "../utils.ts"; /** * noNativeFile test - validates native file read/write tools are disabled. @@ -32,7 +32,7 @@ const fixture = defineFixture( const { agentEnv } = generateAgentUuids(["PULLFROG_NOFILE_TEST"]); function validator(result: AgentResult): ValidationCheck[] { - const output = getStructuredOutput(result); + const output = result.structuredOutput; const fullOutput = result.output; const setOutputCalled = output !== null; diff --git a/test/crossagent/nobash.ts b/test/crossagent/nobash.ts index 33b2672..a6ed881 100644 --- a/test/crossagent/nobash.ts +++ b/test/crossagent/nobash.ts @@ -1,10 +1,5 @@ import type { AgentResult, TestRunnerOptions, ValidationCheck } from "../utils.ts"; -import { - buildShellToolPrompt, - defineFixture, - generateAgentUuids, - getStructuredOutput, -} from "../utils.ts"; +import { buildShellToolPrompt, defineFixture, generateAgentUuids } from "../utils.ts"; /** * nobash test - validates agents respect shell=disabled setting. @@ -31,7 +26,7 @@ function validator(result: AgentResult): ValidationCheck[] { const marker = getUuid(result.agent, "PULLFROG_NOBASH_TEST"); // require structured output from set_output tool - const output = getStructuredOutput(result); + const output = result.structuredOutput; const setOutputCalled = output !== null; // shell should NOT have executed - unique marker value should NOT appear in output diff --git a/test/crossagent/restricted.ts b/test/crossagent/restricted.ts index ae9a997..03dff3c 100644 --- a/test/crossagent/restricted.ts +++ b/test/crossagent/restricted.ts @@ -1,10 +1,5 @@ import type { AgentResult, TestRunnerOptions, ValidationCheck } from "../utils.ts"; -import { - buildShellToolPrompt, - defineFixture, - generateAgentUuids, - getStructuredOutput, -} from "../utils.ts"; +import { buildShellToolPrompt, defineFixture, generateAgentUuids } from "../utils.ts"; /** * restricted test - validates shell=restricted environment filtering. @@ -14,7 +9,7 @@ import { const fixture = defineFixture( { prompt: `This is a test to determine token visibility in shell tool calls. - + ${buildShellToolPrompt("echo $PULLFROG_TEST_VALUE")} Then also run: echo $PULLFROG_TEST_TOKEN @@ -36,7 +31,7 @@ function validator(result: AgentResult): ValidationCheck[] { const filteredMarker = getUuid(result.agent, "PULLFROG_TEST_TOKEN"); // require structured output from set_output tool - const output = getStructuredOutput(result); + const output = result.structuredOutput; const setOutputCalled = output !== null; // non-sensitive env var SHOULD appear in output (agent can read it via shell) diff --git a/test/crossagent/smoke.ts b/test/crossagent/smoke.ts index 102314b..9a3cfd2 100644 --- a/test/crossagent/smoke.ts +++ b/test/crossagent/smoke.ts @@ -1,5 +1,5 @@ import type { AgentResult, TestRunnerOptions, ValidationCheck } from "../utils.ts"; -import { defineFixture, getStructuredOutput } from "../utils.ts"; +import { defineFixture } from "../utils.ts"; /** * smoke test - validates agent can connect to API and call MCP tools. @@ -15,7 +15,7 @@ const fixture = defineFixture( ); function validator(result: AgentResult): ValidationCheck[] { - const output = getStructuredOutput(result); + const output = result.structuredOutput; const setOutputCalled = output !== null; const correctValue = setOutputCalled && /SMOKE TEST PASSED/i.test(output); diff --git a/test/utils.ts b/test/utils.ts index d35dd49..29eed1a 100644 --- a/test/utils.ts +++ b/test/utils.ts @@ -1,6 +1,6 @@ import { spawn } from "node:child_process"; import { randomUUID } from "node:crypto"; -import { mkdirSync, writeFileSync } from "node:fs"; +import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; import { agentsManifest } from "../external.ts"; @@ -112,6 +112,7 @@ export interface AgentResult { agent: string; success: boolean; output: string; + structuredOutput: string | null; } // get agent output with GitHub Actions masking commands filtered out @@ -123,12 +124,19 @@ export function getAgentOutput(result: AgentResult): string { .join("\n"); } -// get structured output from set_output tool (via ::pullfrog-output:: marker) -// returns null if no structured output was set by the agent -export function getStructuredOutput(result: AgentResult): string | null { - const match = result.output.match(/::pullfrog-output::([A-Za-z0-9+/=]+)/); +// parse GITHUB_OUTPUT file format to extract a key's value. +// format: key<\n\nghadelimiter_ +function parseGitHubOutputFile(filePath: string, key: string): string | null { + let content: string; + try { + content = readFileSync(filePath, "utf8"); + } catch { + return null; + } + const pattern = new RegExp(`${key}<<(ghadelimiter_[\\w-]+)\\n([\\s\\S]*?)\\n\\1`); + const match = content.match(pattern); if (!match) return null; - return Buffer.from(match[1], "base64").toString(); + return match[2]; } export interface ValidationCheck { @@ -184,6 +192,9 @@ export async function runAgentStreaming(options: RunStreamingOptions): Promise