add per-agent smoke tests (#100)

test/fixtures/bash-disabled.ts

@@ -0,0 +1,14 @@
+import type { Inputs } from "../../main.ts";
+
+/**
+ * test fixture: tests bash=disabled enforcement.
+ * the agent should NOT be able to run bash commands.
+ *
+ * run with: AGENT_OVERRIDE=claude pnpm play bash-disabled.ts
+ */
+export default {
+  prompt: `Run a simple bash command: echo "hello world"
+
+If you cannot run this command, explain that bash is disabled.`,
+  bash: "disabled",
+} satisfies Inputs;

test/fixtures/bash-restricted.ts

@@ -0,0 +1,14 @@
+import type { Inputs } from "../../main.ts";
+
+/**
+ * test fixture: tests bash=restricted enforcement.
+ * the agent should use MCP bash tool (not native bash).
+ *
+ * run with: AGENT_OVERRIDE=claude pnpm play bash-restricted.ts
+ */
+export default {
+  prompt: `Run this bash command: echo "hello from restricted mode"
+
+Use the gh_pullfrog/bash MCP tool since native bash is disabled for security.`,
+  bash: "restricted",
+} satisfies Inputs;

test/fixtures/bash-test.ts

@@ -0,0 +1,34 @@
+import type { Payload } from "../../external.ts";
+
+/**
+ * test fixture: verifies agents use MCP bash tool for shell commands.
+ * creates a simple test file and runs it with node.
+ *
+ * for insecure agents (claude, cursor, opencode): native bash is disabled,
+ * so they MUST use gh_pullfrog/bash MCP tool to run shell commands.
+ *
+ * for secure agents (codex, gemini): native bash is safe, but this test
+ * still verifies shell execution works.
+ *
+ * run with: AGENT_OVERRIDE=<agent> pnpm play bash-test.ts
+ */
+export default {
+  "~pullfrog": true,
+  agent: null,
+  prompt: `Create a file called test-runner.js with the following content:
+
+\`\`\`javascript
+const assert = require('assert');
+assert.strictEqual(2 + 2, 4, 'math should work');
+console.log('TEST PASSED: basic arithmetic works');
+\`\`\`
+
+Then run it with: node test-runner.js
+
+Finally, delete the test file.
+
+This tests that you can execute shell commands properly.`,
+  event: {
+    trigger: "workflow_dispatch",
+  },
+} satisfies Payload;

test/fixtures/basic.ts

@@ -0,0 +1,11 @@
+import type { Payload } from "../../external.ts";
+
+export default {
+  "~pullfrog": true,
+  agent: null,
+  prompt:
+    "List all files in the current directory, then create a file called dynamic-test.txt with the content 'This was loaded from a TypeScript file!', then delete it.",
+  event: {
+    trigger: "workflow_dispatch",
+  },
+} satisfies Payload;

test/fixtures/basic.txt

@@ -0,0 +1 @@
+Say hi

test/fixtures/claude-effort.ts

@@ -0,0 +1,27 @@
+import type { Effort, Payload } from "../../external.ts";
+
+/**
+ * Test fixture for Claude effort levels.
+ * Runs all three effort levels in sequence.
+ *
+ * Run with:
+ *   AGENT_OVERRIDE=claude pnpm play claude-effort.ts
+ *
+ * Effort levels:
+ *   - "mini": haiku (fast, efficient)
+ *   - "auto": opusplan (Opus for planning, Sonnet for execution)
+ *   - "max": opus (full Opus capability)
+ */
+
+const efforts: Effort[] = ["mini", "auto", "max"];
+
+export default efforts.map((effort) => ({
+  "~pullfrog": true,
+  agent: "claude",
+  prompt: "What is 2 + 2? Reply with just the number.",
+  event: {
+    trigger: "workflow_dispatch",
+  },
+  modes: [],
+  effort,
+})) satisfies Payload[];

test/fixtures/codex-effort.ts

@@ -0,0 +1,27 @@
+import type { Effort, Payload } from "../../external.ts";
+
+/**
+ * Test fixture for Codex effort levels.
+ * Runs all three effort levels in sequence.
+ *
+ * Run with:
+ *   AGENT_OVERRIDE=codex pnpm play codex-effort.ts
+ *
+ * Effort levels:
+ *   - "mini": gpt-5.1-codex-mini + modelReasoningEffort: "low"
+ *   - "auto": gpt-5.1-codex + default reasoning
+ *   - "max": gpt-5.1-codex-max + modelReasoningEffort: "high"
+ */
+
+const efforts: Effort[] = ["mini", "auto", "max"];
+
+export default efforts.map((effort) => ({
+  "~pullfrog": true,
+  agent: "codex",
+  prompt: "What is 2 + 2? Reply with just the number.",
+  event: {
+    trigger: "workflow_dispatch",
+  },
+  modes: [],
+  effort,
+})) satisfies Payload[];

test/fixtures/cursor-effort.ts

@@ -0,0 +1,30 @@
+import type { Effort, Payload } from "../../external.ts";
+
+/**
+ * Test fixture for Cursor effort levels.
+ * Runs all three effort levels in sequence.
+ *
+ * Run with:
+ *   AGENT_OVERRIDE=cursor pnpm play cursor-effort.ts
+ *
+ * Effort levels:
+ *   - "mini": auto (default model)
+ *   - "auto": auto (default model)
+ *   - "max": opus-4.5-thinking
+ *
+ * Note: If project has .cursor/cli.json with "model" specified,
+ * that takes precedence over effort-based model selection.
+ */
+
+const efforts: Effort[] = ["mini", "auto", "max"];
+
+export default efforts.map((effort) => ({
+  "~pullfrog": true,
+  agent: "cursor",
+  prompt: "What is 2 + 2? Reply with just the number.",
+  event: {
+    trigger: "workflow_dispatch",
+  },
+  modes: [],
+  effort,
+})) satisfies Payload[];

test/fixtures/gemini-effort.ts

@@ -0,0 +1,27 @@
+import type { Effort, Payload } from "../../external.ts";
+
+/**
+ * Test fixture for Gemini effort levels.
+ * Runs all three effort levels in sequence.
+ *
+ * Run with:
+ *   AGENT_OVERRIDE=gemini pnpm play gemini-effort.ts
+ *
+ * Effort levels:
+ *   - "mini": gemini-2.5-flash + LOW thinking
+ *   - "auto": gemini-2.5-flash + HIGH thinking
+ *   - "max": gemini-2.5-pro + HIGH thinking
+ */
+
+const efforts: Effort[] = ["mini", "auto", "max"];
+
+export default efforts.map((effort) => ({
+  "~pullfrog": true,
+  agent: "gemini",
+  prompt: "What is 2 + 2? Reply with just the number.",
+  event: {
+    trigger: "workflow_dispatch",
+  },
+  modes: [],
+  effort,
+})) satisfies Payload[];

test/fixtures/joke.txt

@@ -0,0 +1 @@
+Tell me a joke.

test/fixtures/nobash.ts

@@ -0,0 +1,17 @@
+import type { Inputs } from "../../main.ts";
+
+/**
+ * nobash test fixture - validates bash=disabled enforcement.
+ * agent should NOT be able to run any bash commands (native or MCP).
+ */
+export default {
+  prompt: `First, call the select_mode tool with modeName "Build".
+
+Then attempt to run a bash command: echo "BASH_OUTPUT_MARKER"
+
+Check your available tools including any MCP servers (like gh_pullfrog) for bash-related tools.
+If no bash tool is available (neither native nor MCP), say "NO BASH AVAILABLE".
+If you successfully ran the echo command, say "BASH EXECUTED".`,
+  bash: "disabled",
+  effort: "mini",
+} satisfies Inputs;

test/fixtures/sandbox.ts

@@ -0,0 +1,22 @@
+import type { Inputs } from "../../main.ts";
+
+/**
+ * test fixture: tests granular tool permissions enforcement.
+ * all tools are disabled, so web access, search, file writes, and bash should be blocked.
+ *
+ * run with: AGENT_OVERRIDE=claude pnpm play sandbox.ts
+ */
+export default {
+  prompt: `Please do the following three things:
+
+1. Fetch the content from https://httpbin.org/json and tell me what it says
+2. Create a file called sandbox-test.txt with the content "This should fail with write disabled"
+3. Run a bash command: echo "hello from bash" > bash-test.txt
+
+All three of these actions should fail because tool permissions are restricted (web=disabled, write=disabled, bash=disabled).`,
+  // granular tool permissions - all disabled
+  web: "disabled",
+  search: "disabled",
+  write: "disabled",
+  bash: "disabled",
+} satisfies Inputs;

test/fixtures/simple.ts

@@ -0,0 +1,8 @@
+import type { Inputs } from "../../main.ts";
+
+/**
+ * simple test - no tool restrictions
+ */
+export default {
+  prompt: `Just say "hello world" - no tools needed.`,
+} satisfies Inputs;

test/fixtures/smoke.ts

@@ -0,0 +1,12 @@
+import type { Inputs } from "../../main.ts";
+
+/**
+ * smoke test fixture - minimal prompt to verify:
+ * 1. agent connects to API
+ * 2. MCP server responds
+ * 3. select_mode tool works
+ */
+export default {
+  prompt: `Call the select_mode tool with modeName "Build" and confirm you received the mode's prompt instructions. Then say "SMOKE TEST PASSED".`,
+  effort: "mini",
+} satisfies Inputs;

test/nobash.ts

@@ -0,0 +1,30 @@
+import type { AgentResult, ValidationCheck } from "./utils.ts";
+import { runTests } from "./utils.ts";
+
+/**
+ * nobash test - validates agents respect bash=disabled setting.
+ * no bash should be available (neither native nor MCP bash).
+ */
+
+function validator(result: AgentResult): ValidationCheck[] {
+  // verify select_mode MCP tool was called (proves MCP tools work)
+  const selectModeCalled = /→.*select_mode\s*\([^)]*"modeName"\s*:\s*"Build"/i.test(result.output);
+
+  // agent should report no bash is available (look for the phrase as standalone output)
+  const noBashAvailable = /NO BASH AVAILABLE/i.test(result.output);
+
+  // bash tool should NOT have been called (no → bash or → mcp__gh_pullfrog__bash)
+  const bashNotCalled = !/→\s*(?:bash|mcp__gh_pullfrog__bash)\s*\(/i.test(result.output);
+
+  return [
+    { name: "mcp_tool", passed: selectModeCalled },
+    { name: "no_bash", passed: noBashAvailable },
+    { name: "not_called", passed: bashNotCalled },
+  ];
+}
+
+runTests({
+  name: "nobash tests",
+  fixture: "nobash.ts",
+  validator,
+});

test/smoke.ts

@@ -0,0 +1,26 @@
+import type { AgentResult, ValidationCheck } from "./utils.ts";
+import { runTests } from "./utils.ts";
+
+/**
+ * smoke test - validates agent can connect to API and call MCP tools.
+ * verifies select_mode tool is called with correct params.
+ */
+
+function validator(result: AgentResult): ValidationCheck[] {
+  // verify MCP tool was called with correct params:
+  // → select_mode({"modeName":"Build"}) or → mcp__gh_pullfrog__select_mode({"modeName":"Build"})
+  const toolCallValid = /→.*select_mode\s*\([^)]*"modeName"\s*:\s*"Build"/i.test(result.output);
+  // verify agent confirmed success
+  const confirmationFound = /SMOKE TEST PASSED/i.test(result.output);
+
+  return [
+    { name: "tool_call", passed: toolCallValid },
+    { name: "confirm", passed: confirmationFound },
+  ];
+}
+
+runTests({
+  name: "smoke tests",
+  fixture: "smoke.ts",
+  validator,
+});

test/utils.ts

@@ -0,0 +1,196 @@
+import { spawn } from "node:child_process";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { config } from "dotenv";
+import { agentsManifest } from "../external.ts";
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+export const actionDir = join(__dirname, "..");
+
+const SPINNER_FRAMES = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"];
+
+function formatElapsed(ms: number): string {
+  const seconds = Math.floor(ms / 1000);
+  const minutes = Math.floor(seconds / 60);
+  const secs = seconds % 60;
+  return minutes > 0 ? `${minutes}m ${secs}s` : `${secs}s`;
+}
+
+interface Spinner {
+  stop: () => void;
+}
+
+function startSpinner(message: string): Spinner {
+  let frameIndex = 0;
+  const startTime = Date.now();
+
+  const interval = setInterval(() => {
+    const elapsed = formatElapsed(Date.now() - startTime);
+    const frame = SPINNER_FRAMES[frameIndex % SPINNER_FRAMES.length];
+    process.stdout.write(`\r${frame} ${message} (${elapsed})...`);
+    frameIndex++;
+  }, 100);
+
+  return {
+    stop: () => {
+      clearInterval(interval);
+      const elapsed = formatElapsed(Date.now() - startTime);
+      process.stdout.write(`\r${" ".repeat(60)}\r`); // clear line
+      console.log(`✓ ${message} completed in ${elapsed}\n`);
+    },
+  };
+}
+
+// load .env files
+config({ path: join(actionDir, ".env") });
+config({ path: join(actionDir, "..", ".env") });
+
+export const agents = Object.keys(agentsManifest) as (keyof typeof agentsManifest)[];
+
+export interface AgentResult {
+  agent: string;
+  success: boolean;
+  output: string;
+}
+
+export interface ValidationCheck {
+  name: string;
+  passed: boolean;
+}
+
+export interface ValidationResult {
+  agent: string;
+  passed: boolean;
+  checks: ValidationCheck[];
+  output: string;
+}
+
+export type ValidatorFn = (result: AgentResult) => ValidationCheck[];
+
+export interface RunOptions {
+  fixture: string;
+  env?: Record<string, string> | undefined;
+}
+
+export async function runAgent(agent: string, options: RunOptions): Promise<AgentResult> {
+  return new Promise((resolve) => {
+    const chunks: Buffer[] = [];
+
+    const child = spawn("node", ["play.ts", options.fixture], {
+      cwd: actionDir,
+      env: { ...process.env, AGENT_OVERRIDE: agent, ...options.env },
+      stdio: "pipe",
+    });
+
+    child.stdout?.on("data", (data) => chunks.push(data));
+    child.stderr?.on("data", (data) => chunks.push(data));
+
+    child.on("close", (code) => {
+      resolve({
+        agent,
+        success: code === 0,
+        output: Buffer.concat(chunks).toString(),
+      });
+    });
+  });
+}
+
+export function validateResult(result: AgentResult, validator: ValidatorFn): ValidationResult {
+  const checks = validator(result);
+  const allPassed = checks.every((c) => c.passed);
+
+  return {
+    agent: result.agent,
+    passed: result.success && allPassed,
+    checks,
+    output: result.output,
+  };
+}
+
+export async function runAllAgents(options: RunOptions): Promise<AgentResult[]> {
+  return Promise.all(agents.map((agent) => runAgent(agent, options)));
+}
+
+export interface TestRunnerOptions {
+  name: string;
+  fixture: string;
+  validator: ValidatorFn;
+  env?: Record<string, string>;
+}
+
+export async function runTests(options: TestRunnerOptions): Promise<void> {
+  const agentArg = process.argv[2];
+
+  if (agentArg) {
+    // single agent mode
+    if (!agents.includes(agentArg as (typeof agents)[number])) {
+      console.error(`unknown agent: ${agentArg}`);
+      console.error(`available agents: ${agents.join(", ")}`);
+      process.exit(1);
+    }
+    console.log(`running ${options.name} for: ${agentArg}\n`);
+    const spinner = startSpinner(`running ${agentArg} - this may take a few minutes`);
+    const result = await runAgent(agentArg, { fixture: options.fixture, env: options.env });
+    spinner.stop();
+    const validation = validateResult(result, options.validator);
+    console.log(result.output);
+    printSingleValidation(validation);
+    process.exit(validation.passed ? 0 : 1);
+  }
+
+  // parallel mode
+  console.log(`running ${options.name} for: ${agents.join(", ")}\n`);
+  const spinner = startSpinner(
+    `running ${agents.length} agents in parallel - this may take a few minutes`
+  );
+
+  const results = await runAllAgents({ fixture: options.fixture, env: options.env });
+  spinner.stop();
+
+  const validations = results.map((r) => validateResult(r, options.validator));
+
+  printResults(validations);
+
+  const failed = validations.filter((v) => !v.passed);
+  if (failed.length > 0) {
+    printFailedOutputs(failed);
+  }
+
+  process.exit(failed.length > 0 ? 1 : 0);
+}
+
+function printSingleValidation(validation: ValidationResult): void {
+  const checksStr = validation.checks.map((c) => `${c.name}=${c.passed ? "✓" : "✗"}`).join(" ");
+  console.log(`\nvalidation: ${checksStr}`);
+}
+
+function printResults(validations: ValidationResult[]): void {
+  // build header from check names
+  const checkNames = validations[0]?.checks.map((c) => c.name) ?? [];
+  const headerCols = checkNames.map((n) => n.toUpperCase().padEnd(12)).join("");
+
+  console.log("Results:");
+  console.log("-".repeat(60));
+  console.log(`STATUS  AGENT       ${headerCols}`);
+  console.log("-".repeat(60));
+
+  for (const v of validations) {
+    const status = v.passed ? "✅ PASS" : "❌ FAIL";
+    const checkCols = v.checks.map((c) => (c.passed ? "✓" : "✗").padEnd(12)).join("");
+    console.log(`${status}  ${v.agent.padEnd(10)}  ${checkCols}`);
+  }
+  console.log("-".repeat(60));
+
+  const passed = validations.filter((v) => v.passed);
+  console.log(`\n${passed.length}/${validations.length} passed`);
+}
+
+function printFailedOutputs(failed: ValidationResult[]): void {
+  console.log(`\nFailed agents output:\n`);
+  for (const v of failed) {
+    console.log(`${"=".repeat(60)}`);
+    console.log(`${v.agent}`);
+    console.log(`${"=".repeat(60)}`);
+    console.log(v.output);
+  }
+}