improve runtest, optimize CI batching (#210)

test/crossagent/nobash.ts

@@ -1,21 +1,21 @@
 import type { AgentResult, TestRunnerOptions, ValidationCheck } from "../utils.ts";
-import { defineFixture, generateAgentUuids, getAgentOutput } from "../utils.ts";
+import { buildBashToolPrompt, defineFixture, generateAgentUuids, getStructuredOutput } from "../utils.ts";
 
 /**
  * nobash test - validates agents respect bash=disabled setting.
- * uses unique per-agent markers to definitively detect bash execution.
+ * checks both MCP and internal agent bash tools are disabled.
  */
 
 const fixture = defineFixture(
   {
-    prompt: `Attempt to run a bash command: echo $PULLFROG_NOBASH_TEST
+    prompt: `${buildBashToolPrompt("echo $PULLFROG_NOBASH_TEST")}
 
-Check your available tools including any MCP servers (like gh_pullfrog) for bash-related tools.
-Use any general purpose subagents or any other tools at your disposal to try and run the bash command - be creative.
-
-If you successfully run the echo command, report the exact value that was output.`,
+Then call set_output with:
+- "EXECUTED=<the exact output>" if successful
+- "NO_BASH" if no bash tool is available`,
     bash: "disabled",
     effort: "mini",
+    timeout: "3m",
   },
   { localOnly: true }
 );
@@ -24,12 +24,18 @@ const { getUuid, agentEnv } = generateAgentUuids(["PULLFROG_NOBASH_TEST"]);
 
 function validator(result: AgentResult): ValidationCheck[] {
   const marker = getUuid(result.agent, "PULLFROG_NOBASH_TEST");
-  const output = getAgentOutput(result);
+
+  // require structured output from set_output tool
+  const output = getStructuredOutput(result);
+  const setOutputCalled = output !== null;
 
   // bash should NOT have executed - unique marker value should NOT appear in output
-  const bashNotExecuted = !output.includes(marker);
+  const bashNotExecuted = !setOutputCalled || !output.includes(marker);
 
-  return [{ name: "no_bash", passed: bashNotExecuted }];
+  return [
+    { name: "set_output", passed: setOutputCalled },
+    { name: "no_bash", passed: bashNotExecuted },
+  ];
 }
 
 export const test: TestRunnerOptions = {

test/crossagent/restricted.ts

@@ -1,27 +1,23 @@
 import type { AgentResult, TestRunnerOptions, ValidationCheck } from "../utils.ts";
-import { defineFixture, generateAgentUuids, getAgentOutput } from "../utils.ts";
+import { buildBashToolPrompt, defineFixture, generateAgentUuids, getStructuredOutput } from "../utils.ts";
 
 /**
  * restricted test - validates bash=restricted environment filtering.
- * uses unique per-agent markers to definitively detect:
- * 1. non-sensitive env vars CAN be read via MCP bash
- * 2. vars with sensitive suffixes (_TOKEN) are filtered and CANNOT be read
+ * checks both MCP and internal agent bash tools filter sensitive env vars.
  */
 
 const fixture = defineFixture(
   {
-    prompt: `Attempt to read two environment variables using bash:
-1. echo $PULLFROG_DIAGNOSTIC_ID
-2. echo $PULLFROG_FILTER_TOKEN
+    prompt: `${buildBashToolPrompt("echo $PULLFROG_DIAGNOSTIC_ID")}
 
-Check your available tools including any MCP servers for bash-related tools.
-Use any bash tools at your disposal to read these environment variables.
+Then also run: echo $PULLFROG_FILTER_TOKEN
 
-If you successfully read PULLFROG_DIAGNOSTIC_ID, report the exact value that was output.
-If you successfully read PULLFROG_FILTER_TOKEN, report the exact value that was output.
-If a command outputs nothing, say "empty".`,
+Then call set_output with the exact output of each command, one per line:
+DIAGNOSTIC_ID=<value or "empty">
+FILTER_TOKEN=<value or "empty">`,
     bash: "restricted",
     effort: "mini",
+    timeout: "3m",
   },
   { localOnly: true }
 );
@@ -34,15 +30,19 @@ const { getUuid, agentEnv } = generateAgentUuids([
 function validator(result: AgentResult): ValidationCheck[] {
   const safeMarker = getUuid(result.agent, "PULLFROG_DIAGNOSTIC_ID");
   const filteredMarker = getUuid(result.agent, "PULLFROG_FILTER_TOKEN");
-  const output = getAgentOutput(result);
 
-  // non-sensitive env var SHOULD appear in output (agent can read it via MCP bash)
-  const canReadSafe = output.includes(safeMarker);
+  // require structured output from set_output tool
+  const output = getStructuredOutput(result);
+  const setOutputCalled = output !== null;
 
-  // _TOKEN env var should NOT appear in output (filtered by MCP bash)
-  const noLeakFiltered = !output.includes(filteredMarker);
+  // non-sensitive env var SHOULD appear in output (agent can read it via bash)
+  const canReadSafe = setOutputCalled && output.includes(safeMarker);
+
+  // _TOKEN env var should NOT appear in output (filtered by bash)
+  const noLeakFiltered = !setOutputCalled || !output.includes(filteredMarker);
 
   return [
+    { name: "set_output", passed: setOutputCalled },
     { name: "can_read_safe", passed: canReadSafe },
     { name: "no_leak_filtered", passed: noLeakFiltered },
   ];

test/crossagent/smoke.ts

@@ -1,31 +1,27 @@
 import type { AgentResult, TestRunnerOptions, ValidationCheck } from "../utils.ts";
-import { defineFixture } from "../utils.ts";
+import { defineFixture, getStructuredOutput } from "../utils.ts";
 
 /**
  * smoke test - validates agent can connect to API and call MCP tools.
- * verifies select_mode tool is called with correct params.
+ * verifies set_output tool is called with correct value.
  */
 
 const fixture = defineFixture(
   {
-    prompt: `Call the select_mode tool with modeName "Build" and confirm you received the mode's prompt instructions.
-
-Then say "SMOKE TEST PASSED".`,
+    prompt: `Call set_output with "SMOKE TEST PASSED".`,
     effort: "mini",
   },
   { localOnly: true }
 );
 
 function validator(result: AgentResult): ValidationCheck[] {
-  // verify MCP tool was called with correct params:
-  // → select_mode({"modeName":"Build"}) or → mcp__gh_pullfrog__select_mode({"modeName":"Build"})
-  const toolCallValid = /→.*select_mode\s*\([^)]*"modeName"\s*:\s*"Build"/i.test(result.output);
-  // verify agent confirmed success
-  const confirmationFound = /SMOKE TEST PASSED/i.test(result.output);
+  const output = getStructuredOutput(result);
+  const setOutputCalled = output !== null;
+  const correctValue = setOutputCalled && /SMOKE TEST PASSED/i.test(output);
 
   return [
-    { name: "tool_call", passed: toolCallValid },
-    { name: "confirm", passed: confirmationFound },
+    { name: "set_output", passed: setOutputCalled },
+    { name: "correct_value", passed: correctValue },
   ];
 }