add per-agent smoke tests (#100)

.github/workflows/test.yml

@@ -9,9 +9,18 @@ on:
 permissions:
   contents: read
 
+defaults:
+  run:
+    working-directory: action
+
 jobs:
   test:
     runs-on: ubuntu-latest
+    env:
+      ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+      CURSOR_API_KEY: ${{ secrets.CURSOR_API_KEY }}
+      GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -24,6 +33,7 @@ jobs:
         with:
           node-version: "24"
           cache: "pnpm"
+          cache-dependency-path: action/pnpm-lock.yaml
 
       - name: Install dependencies
         run: pnpm install --frozen-lockfile
@@ -33,3 +43,9 @@ jobs:
 
       - name: Run tests
         run: pnpm test
+
+      - name: Run smoke tests
+        run: pnpm smoke
+
+      - name: Run nobash tests
+        run: pnpm nobash

entry

@@ -119438,9 +119438,11 @@ async function startMcpHttpServer(ctx) {
     AddLabelsTool(ctx),
     CreateBranchTool(ctx),
     CommitFilesTool(ctx),
-    PushBranchTool(ctx),
-    BashTool(ctx)
+    PushBranchTool(ctx)
   ];
+  if (ctx.tools.bash !== "disabled") {
+    tools.push(BashTool(ctx));
+  }
   if (!ctx.disableProgressComment) {
     tools.push(ReportProgressTool(ctx));
   }
@@ -119971,6 +119973,8 @@ var package_default = {
     typecheck: "tsc --noEmit",
     build: "node esbuild.config.js",
     play: "node play.ts",
+    smoke: "node test/smoke.ts",
+    nobash: "node test/nobash.ts",
     scratch: "node scratch.ts",
     upDeps: "pnpm up --latest",
     lock: "pnpm --ignore-workspace install",
@@ -138503,7 +138507,8 @@ async function main(core5) {
           modes: modes2,
           toolState,
           runId,
-          jobId
+          jobId,
+          tools
         };
         const mcpHttpServer = __using(_stack, await startMcpHttpServer(toolContext), true);
         log.info(`\xBB MCP server started at ${mcpHttpServer.url}`);

main.ts

@@ -86,6 +86,7 @@ export async function main(core: {
       toolState,
       runId,
       jobId,
+      tools,
     };
 
     await using mcpHttpServer = await startMcpHttpServer(toolContext);

mcp/server.ts

@@ -24,6 +24,7 @@ export interface ToolState {
 }
 
 export interface ToolContext {
+  tools: ToolPermissions;
   owner: string;
   name: string;
   repo: { default_branch: string; private: boolean };
@@ -38,6 +39,7 @@ export interface ToolContext {
   jobId: string | undefined;
 }
 
+import type { ToolPermissions } from "../agents/shared.ts";
 import { BashTool } from "./bash.ts";
 import { CheckoutPrTool } from "./checkout.ts";
 import { GetCheckSuiteLogsTool } from "./checkSuite.ts";
@@ -129,9 +131,16 @@ export async function startMcpHttpServer(
     CreateBranchTool(ctx),
     CommitFilesTool(ctx),
     PushBranchTool(ctx),
-    BashTool(ctx),
   ];
 
+  // only add BashTool if bash is not disabled
+  // - "enabled": native bash + MCP bash
+  // - "restricted": MCP bash only (native blocked by agent)
+  // - "disabled": no bash at all
+  if (ctx.tools.bash !== "disabled") {
+    tools.push(BashTool(ctx));
+  }
+
   if (!ctx.disableProgressComment) {
     tools.push(ReportProgressTool(ctx));
   }

package.json

@@ -17,6 +17,8 @@
     "typecheck": "tsc --noEmit",
     "build": "node esbuild.config.js",
     "play": "node play.ts",
+    "smoke": "node test/smoke.ts",
+    "nobash": "node test/nobash.ts",
     "scratch": "node scratch.ts",
     "upDeps": "pnpm up --latest",
     "lock": "pnpm --ignore-workspace install",

play.ts

@@ -1,5 +1,7 @@
 import { spawnSync } from "node:child_process";
-import { existsSync, readFileSync } from "node:fs";
+import { existsSync, readFileSync, rmSync } from "node:fs";
+import { mkdtemp } from "node:fs/promises";
+import { platform, tmpdir } from "node:os";
 import { dirname, extname, join, resolve } from "node:path";
 import { fileURLToPath, pathToFileURL } from "node:url";
 import { fromHere } from "@ark/fs";
@@ -19,11 +21,14 @@ config();
 config({ path: join(__dirname, "..", ".env") });
 
 export async function run(inputsOrPrompt: Inputs | string): Promise<AgentResult> {
-  try {
-    const tempDir = join(__dirname, ".temp");
-    setupTestRepo({ tempDir });
-
+  // create unique temp directory path in OS temp location for parallel execution
+  // use a parent dir from mkdtemp, then clone into a 'repo' subdirectory
+  const tempParent = await mkdtemp(join(tmpdir(), "pullfrog-play-"));
+  const tempDir = join(tempParent, "repo");
   const originalCwd = process.cwd();
+
+  try {
+    setupTestRepo({ tempDir });
     process.chdir(tempDir);
 
     // allow passing full Inputs object or just a prompt string
@@ -59,6 +64,10 @@ export async function run(inputsOrPrompt: Inputs | string): Promise<AgentResult>
     const errorMessage = (err as Error).message;
     log.error(`Error: ${errorMessage}`);
     return { success: false, error: errorMessage, output: undefined };
+  } finally {
+    // cleanup temp directory
+    process.chdir(originalCwd);
+    rmSync(tempParent, { recursive: true, force: true });
   }
 }
 
@@ -82,7 +91,7 @@ Arguments:
 
 Options:
   --raw [prompt]          Use raw string as prompt instead of loading from file
-  --local, -l             Run locally on macOS (default: runs in Docker)
+  --local, -l             Run locally (default: runs in Docker)
   -h, --help              Show this help message
 
 Environment:
@@ -115,26 +124,72 @@ Examples:
       value !== undefined ? ["-e", `${key}=${value}`] : []
     );
 
-    // SSH for git - use SSH agent forwarding (no passphrase prompts)
+    // SSH for git - platform-specific handling
     const sshFlags: string[] = [];
+    let sshSetupCmd = "";
+    const plat = platform();
     const home = process.env.HOME;
+
+    if (plat === "win32") {
+      throw new Error(
+        "Docker mode is not supported on native Windows. Use WSL2 or set PLAY_LOCAL=1."
+      );
+    } else if (plat === "darwin") {
+      // macOS: Docker Desktop SSH agent forwarding
       if (home) {
-      const sshDir = join(home, ".ssh");
-      // mount known_hosts for host key verification
-      const knownHostsPath = join(sshDir, "known_hosts");
+        const knownHostsPath = join(home, ".ssh", "known_hosts");
         if (existsSync(knownHostsPath)) {
           sshFlags.push("-v", `${knownHostsPath}:/root/.ssh/known_hosts:ro`);
         }
       }
-    // forward SSH agent from Docker Desktop on macOS
       sshFlags.push(
         "-v",
         "/run/host-services/ssh-auth.sock:/run/host-services/ssh-auth.sock",
         "-e",
         "SSH_AUTH_SOCK=/run/host-services/ssh-auth.sock"
       );
+    } else {
+      // Linux/WSL: copy .ssh files into container with correct permissions
+      if (home) {
+        const sshDir = join(home, ".ssh");
+        if (existsSync(sshDir)) {
+          sshFlags.push("-v", `${sshDir}:/tmp/.ssh-host:ro`);
+          // copy ssh keys, add github.com to known_hosts, set GIT_SSH_COMMAND to use them
+          sshSetupCmd =
+            "mkdir -p /tmp/home/.ssh && cp /tmp/.ssh-host/id_* /tmp/home/.ssh/ 2>/dev/null; chmod 600 /tmp/home/.ssh/id_* 2>/dev/null; " +
+            "ssh-keyscan -t ed25519,rsa github.com >> /tmp/home/.ssh/known_hosts 2>/dev/null; chmod 644 /tmp/home/.ssh/known_hosts; " +
+            "export GIT_SSH_COMMAND='ssh -i /tmp/home/.ssh/id_rsa -o UserKnownHostsFile=/tmp/home/.ssh/known_hosts -o StrictHostKeyChecking=no'; ";
+        }
+      }
+    }
 
-    const ttyFlags = process.stdin.isTTY ? ["-it"] : [];
+    // always allocate a pseudo-TTY - Claude Code may require it
+    const ttyFlags = ["-t"];
+
+    // run as current user to avoid Claude CLI's root user restriction
+    const uid = process.getuid?.() ?? 1000;
+    const gid = process.getgid?.() ?? 1000;
+
+    // use agent-specific volume to avoid conflicts when running in parallel
+    const agentOverride = process.env.AGENT_OVERRIDE ?? "default";
+    const volumeName = `pullfrog-action-node-modules-${agentOverride}`;
+
+    // initialize volume with correct ownership (runs as root briefly)
+    spawnSync(
+      "docker",
+      [
+        "run",
+        "--rm",
+        "-v",
+        `${volumeName}:/app/action/node_modules`,
+        "node:24",
+        "chown",
+        "-R",
+        `${uid}:${gid}`,
+        "/app/action/node_modules",
+      ],
+      { stdio: "ignore", cwd: __dirname }
+    );
 
     const result = spawnSync(
       "docker",
@@ -142,24 +197,24 @@ Examples:
         "run",
         "--rm",
         ...ttyFlags,
+        "--user",
+        `${uid}:${gid}`,
         "-v",
         `${__dirname}:/app/action:cached`,
         "-v",
-        "pullfrog-action-node-modules:/app/action/node_modules",
+        `${volumeName}:/app/action/node_modules`,
         "-w",
         "/app/action",
         ...envFlags,
         ...sshFlags,
         "-e",
         "COREPACK_ENABLE_DOWNLOAD_PROMPT=0",
-        "--cap-add",
-        "SYS_ADMIN",
-        "--security-opt",
-        "seccomp:unconfined",
+        "-e",
+        `HOME=/tmp/home`,
         "node:24",
         "bash",
         "-c",
-        `corepack enable pnpm >/dev/null 2>&1 && pnpm install --frozen-lockfile && ${nodeCmd}`,
+        `${sshSetupCmd}mkdir -p /tmp/home/.config /tmp/home/.cache && corepack pnpm install --frozen-lockfile --ignore-scripts && ${nodeCmd}`,
       ],
       { stdio: "inherit", cwd: __dirname }
     );
@@ -177,7 +232,7 @@ Examples:
     const ext = extname(filePath).toLowerCase();
     let resolvedPath: string;
 
-    const fixturesPath = fromHere("fixtures", filePath);
+    const fixturesPath = fromHere("test", "fixtures", filePath);
     if (existsSync(fixturesPath)) {
       resolvedPath = fixturesPath;
     } else if (existsSync(filePath)) {

test/fixtures/bash-disabled.ts

@@ -1,4 +1,4 @@
-import type { Inputs } from "../main.ts";
+import type { Inputs } from "../../main.ts";
 
 /**
  * test fixture: tests bash=disabled enforcement.

test/fixtures/bash-restricted.ts

@@ -1,4 +1,4 @@
-import type { Inputs } from "../main.ts";
+import type { Inputs } from "../../main.ts";
 
 /**
  * test fixture: tests bash=restricted enforcement.

test/fixtures/bash-test.ts

@@ -1,4 +1,4 @@
-import type { Payload } from "../external.ts";
+import type { Payload } from "../../external.ts";
 
 /**
  * test fixture: verifies agents use MCP bash tool for shell commands.

test/fixtures/basic.ts

@@ -1,4 +1,4 @@
-import type { Payload } from "../external.ts";
+import type { Payload } from "../../external.ts";
 
 export default {
   "~pullfrog": true,

test/fixtures/basic.txt

@@ -0,0 +1 @@
+Say hi

test/fixtures/claude-effort.ts

@@ -1,4 +1,4 @@
-import type { Effort, Payload } from "../external.ts";
+import type { Effort, Payload } from "../../external.ts";
 
 /**
  * Test fixture for Claude effort levels.

test/fixtures/codex-effort.ts

@@ -1,4 +1,4 @@
-import type { Effort, Payload } from "../external.ts";
+import type { Effort, Payload } from "../../external.ts";
 
 /**
  * Test fixture for Codex effort levels.

test/fixtures/cursor-effort.ts

@@ -1,4 +1,4 @@
-import type { Effort, Payload } from "../external.ts";
+import type { Effort, Payload } from "../../external.ts";
 
 /**
  * Test fixture for Cursor effort levels.

test/fixtures/gemini-effort.ts

@@ -1,4 +1,4 @@
-import type { Effort, Payload } from "../external.ts";
+import type { Effort, Payload } from "../../external.ts";
 
 /**
  * Test fixture for Gemini effort levels.

test/fixtures/nobash.ts

@@ -0,0 +1,17 @@
+import type { Inputs } from "../../main.ts";
+
+/**
+ * nobash test fixture - validates bash=disabled enforcement.
+ * agent should NOT be able to run any bash commands (native or MCP).
+ */
+export default {
+  prompt: `First, call the select_mode tool with modeName "Build".
+
+Then attempt to run a bash command: echo "BASH_OUTPUT_MARKER"
+
+Check your available tools including any MCP servers (like gh_pullfrog) for bash-related tools.
+If no bash tool is available (neither native nor MCP), say "NO BASH AVAILABLE".
+If you successfully ran the echo command, say "BASH EXECUTED".`,
+  bash: "disabled",
+  effort: "mini",
+} satisfies Inputs;

test/fixtures/sandbox.ts

@@ -1,4 +1,4 @@
-import type { Inputs } from "../main.ts";
+import type { Inputs } from "../../main.ts";
 
 /**
  * test fixture: tests granular tool permissions enforcement.

test/fixtures/simple.ts

@@ -1,4 +1,4 @@
-import type { Inputs } from "../main.ts";
+import type { Inputs } from "../../main.ts";
 
 /**
  * simple test - no tool restrictions

test/fixtures/smoke.ts

@@ -0,0 +1,12 @@
+import type { Inputs } from "../../main.ts";
+
+/**
+ * smoke test fixture - minimal prompt to verify:
+ * 1. agent connects to API
+ * 2. MCP server responds
+ * 3. select_mode tool works
+ */
+export default {
+  prompt: `Call the select_mode tool with modeName "Build" and confirm you received the mode's prompt instructions. Then say "SMOKE TEST PASSED".`,
+  effort: "mini",
+} satisfies Inputs;

test/nobash.ts

@@ -0,0 +1,30 @@
+import type { AgentResult, ValidationCheck } from "./utils.ts";
+import { runTests } from "./utils.ts";
+
+/**
+ * nobash test - validates agents respect bash=disabled setting.
+ * no bash should be available (neither native nor MCP bash).
+ */
+
+function validator(result: AgentResult): ValidationCheck[] {
+  // verify select_mode MCP tool was called (proves MCP tools work)
+  const selectModeCalled = /→.*select_mode\s*\([^)]*"modeName"\s*:\s*"Build"/i.test(result.output);
+
+  // agent should report no bash is available (look for the phrase as standalone output)
+  const noBashAvailable = /NO BASH AVAILABLE/i.test(result.output);
+
+  // bash tool should NOT have been called (no → bash or → mcp__gh_pullfrog__bash)
+  const bashNotCalled = !/→\s*(?:bash|mcp__gh_pullfrog__bash)\s*\(/i.test(result.output);
+
+  return [
+    { name: "mcp_tool", passed: selectModeCalled },
+    { name: "no_bash", passed: noBashAvailable },
+    { name: "not_called", passed: bashNotCalled },
+  ];
+}
+
+runTests({
+  name: "nobash tests",
+  fixture: "nobash.ts",
+  validator,
+});

test/smoke.ts

@@ -0,0 +1,26 @@
+import type { AgentResult, ValidationCheck } from "./utils.ts";
+import { runTests } from "./utils.ts";
+
+/**
+ * smoke test - validates agent can connect to API and call MCP tools.
+ * verifies select_mode tool is called with correct params.
+ */
+
+function validator(result: AgentResult): ValidationCheck[] {
+  // verify MCP tool was called with correct params:
+  // → select_mode({"modeName":"Build"}) or → mcp__gh_pullfrog__select_mode({"modeName":"Build"})
+  const toolCallValid = /→.*select_mode\s*\([^)]*"modeName"\s*:\s*"Build"/i.test(result.output);
+  // verify agent confirmed success
+  const confirmationFound = /SMOKE TEST PASSED/i.test(result.output);
+
+  return [
+    { name: "tool_call", passed: toolCallValid },
+    { name: "confirm", passed: confirmationFound },
+  ];
+}
+
+runTests({
+  name: "smoke tests",
+  fixture: "smoke.ts",
+  validator,
+});

test/utils.ts

@@ -0,0 +1,196 @@
+import { spawn } from "node:child_process";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { config } from "dotenv";
+import { agentsManifest } from "../external.ts";
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+export const actionDir = join(__dirname, "..");
+
+const SPINNER_FRAMES = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"];
+
+function formatElapsed(ms: number): string {
+  const seconds = Math.floor(ms / 1000);
+  const minutes = Math.floor(seconds / 60);
+  const secs = seconds % 60;
+  return minutes > 0 ? `${minutes}m ${secs}s` : `${secs}s`;
+}
+
+interface Spinner {
+  stop: () => void;
+}
+
+function startSpinner(message: string): Spinner {
+  let frameIndex = 0;
+  const startTime = Date.now();
+
+  const interval = setInterval(() => {
+    const elapsed = formatElapsed(Date.now() - startTime);
+    const frame = SPINNER_FRAMES[frameIndex % SPINNER_FRAMES.length];
+    process.stdout.write(`\r${frame} ${message} (${elapsed})...`);
+    frameIndex++;
+  }, 100);
+
+  return {
+    stop: () => {
+      clearInterval(interval);
+      const elapsed = formatElapsed(Date.now() - startTime);
+      process.stdout.write(`\r${" ".repeat(60)}\r`); // clear line
+      console.log(`✓ ${message} completed in ${elapsed}\n`);
+    },
+  };
+}
+
+// load .env files
+config({ path: join(actionDir, ".env") });
+config({ path: join(actionDir, "..", ".env") });
+
+export const agents = Object.keys(agentsManifest) as (keyof typeof agentsManifest)[];
+
+export interface AgentResult {
+  agent: string;
+  success: boolean;
+  output: string;
+}
+
+export interface ValidationCheck {
+  name: string;
+  passed: boolean;
+}
+
+export interface ValidationResult {
+  agent: string;
+  passed: boolean;
+  checks: ValidationCheck[];
+  output: string;
+}
+
+export type ValidatorFn = (result: AgentResult) => ValidationCheck[];
+
+export interface RunOptions {
+  fixture: string;
+  env?: Record<string, string> | undefined;
+}
+
+export async function runAgent(agent: string, options: RunOptions): Promise<AgentResult> {
+  return new Promise((resolve) => {
+    const chunks: Buffer[] = [];
+
+    const child = spawn("node", ["play.ts", options.fixture], {
+      cwd: actionDir,
+      env: { ...process.env, AGENT_OVERRIDE: agent, ...options.env },
+      stdio: "pipe",
+    });
+
+    child.stdout?.on("data", (data) => chunks.push(data));
+    child.stderr?.on("data", (data) => chunks.push(data));
+
+    child.on("close", (code) => {
+      resolve({
+        agent,
+        success: code === 0,
+        output: Buffer.concat(chunks).toString(),
+      });
+    });
+  });
+}
+
+export function validateResult(result: AgentResult, validator: ValidatorFn): ValidationResult {
+  const checks = validator(result);
+  const allPassed = checks.every((c) => c.passed);
+
+  return {
+    agent: result.agent,
+    passed: result.success && allPassed,
+    checks,
+    output: result.output,
+  };
+}
+
+export async function runAllAgents(options: RunOptions): Promise<AgentResult[]> {
+  return Promise.all(agents.map((agent) => runAgent(agent, options)));
+}
+
+export interface TestRunnerOptions {
+  name: string;
+  fixture: string;
+  validator: ValidatorFn;
+  env?: Record<string, string>;
+}
+
+export async function runTests(options: TestRunnerOptions): Promise<void> {
+  const agentArg = process.argv[2];
+
+  if (agentArg) {
+    // single agent mode
+    if (!agents.includes(agentArg as (typeof agents)[number])) {
+      console.error(`unknown agent: ${agentArg}`);
+      console.error(`available agents: ${agents.join(", ")}`);
+      process.exit(1);
+    }
+    console.log(`running ${options.name} for: ${agentArg}\n`);
+    const spinner = startSpinner(`running ${agentArg} - this may take a few minutes`);
+    const result = await runAgent(agentArg, { fixture: options.fixture, env: options.env });
+    spinner.stop();
+    const validation = validateResult(result, options.validator);
+    console.log(result.output);
+    printSingleValidation(validation);
+    process.exit(validation.passed ? 0 : 1);
+  }
+
+  // parallel mode
+  console.log(`running ${options.name} for: ${agents.join(", ")}\n`);
+  const spinner = startSpinner(
+    `running ${agents.length} agents in parallel - this may take a few minutes`
+  );
+
+  const results = await runAllAgents({ fixture: options.fixture, env: options.env });
+  spinner.stop();
+
+  const validations = results.map((r) => validateResult(r, options.validator));
+
+  printResults(validations);
+
+  const failed = validations.filter((v) => !v.passed);
+  if (failed.length > 0) {
+    printFailedOutputs(failed);
+  }
+
+  process.exit(failed.length > 0 ? 1 : 0);
+}
+
+function printSingleValidation(validation: ValidationResult): void {
+  const checksStr = validation.checks.map((c) => `${c.name}=${c.passed ? "✓" : "✗"}`).join(" ");
+  console.log(`\nvalidation: ${checksStr}`);
+}
+
+function printResults(validations: ValidationResult[]): void {
+  // build header from check names
+  const checkNames = validations[0]?.checks.map((c) => c.name) ?? [];
+  const headerCols = checkNames.map((n) => n.toUpperCase().padEnd(12)).join("");
+
+  console.log("Results:");
+  console.log("-".repeat(60));
+  console.log(`STATUS  AGENT       ${headerCols}`);
+  console.log("-".repeat(60));
+
+  for (const v of validations) {
+    const status = v.passed ? "✅ PASS" : "❌ FAIL";
+    const checkCols = v.checks.map((c) => (c.passed ? "✓" : "✗").padEnd(12)).join("");
+    console.log(`${status}  ${v.agent.padEnd(10)}  ${checkCols}`);
+  }
+  console.log("-".repeat(60));
+
+  const passed = validations.filter((v) => v.passed);
+  console.log(`\n${passed.length}/${validations.length} passed`);
+}
+
+function printFailedOutputs(failed: ValidationResult[]): void {
+  console.log(`\nFailed agents output:\n`);
+  for (const v of failed) {
+    console.log(`${"=".repeat(60)}`);
+    console.log(`${v.agent}`);
+    console.log(`${"=".repeat(60)}`);
+    console.log(v.output);
+  }
+}

utils/setup.ts

@@ -1,5 +1,4 @@
 import { execSync } from "node:child_process";
-import { existsSync, rmSync } from "node:fs";
 import { mkdtemp } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
@@ -31,11 +30,7 @@ export function setupTestRepo(options: SetupOptions): void {
   const { tempDir } = options;
   const repo = process.env.GITHUB_REPOSITORY;
   if (!repo) throw new Error("GITHUB_REPOSITORY is required");
-  if (existsSync(tempDir)) {
-    log.info("» removing existing .temp directory...");
-    rmSync(tempDir, { recursive: true, force: true });
-  }
-  log.info(`» cloning ${repo} into .temp...`);
+  log.info(`» cloning ${repo} into ${tempDir}...`);
   $("git", ["clone", `git@github.com:${repo}.git`, tempDir]);
 }