add file_read/file_write tools, sandbox tests, CI improvements (#239)

* migrate to flags * init * iterate on file write lockdown tests * improve ci * fix lockfile * fix typecheck * fix lint * improve pushRestricted * ok * fix more * ok * remove process.env spreading rule Co-authored-by: Cursor <cursoragent@cursor.com> * enhanced fs rw tools --------- Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: Colin McDonnell <colinmcd94@gmail.com>
2026-02-10 06:35:47 +00:00
parent 23df8bf967
commit 19df8372cd
30 changed files with 1801 additions and 648 deletions
@@ -0,0 +1,57 @@
+import type { AgentResult, TestRunnerOptions, ValidationCheck } from "../utils.ts";
+import { defineFixture, generateAgentUuids, getStructuredOutput } from "../utils.ts";
+
+/**
+ * fileReadWrite test - validates MCP file_read, file_write, file_edit, and
+ * file_delete work for all agents and that modifications to .git/ are blocked.
+ */
+
+const PROMPT = `First run: echo $PULLFROG_FILE_TEST
+Use that exact output as your marker.
+
+1. Use gh_pullfrog/file_write to write test-file.txt with content "BEFORE:<marker>" (replace <marker> with the actual marker value).
+2. Use gh_pullfrog/file_edit to replace "BEFORE:" with "AFTER:" in test-file.txt.
+3. Use gh_pullfrog/file_read to read test-file.txt back. Verify it starts with "AFTER:".
+4. Use gh_pullfrog/file_delete to delete test-file.txt.
+5. Try gh_pullfrog/file_read on test-file.txt again — it should fail (file was deleted).
+6. Try gh_pullfrog/file_edit on .git/config with old_string "x" and new_string "y" (should fail — .git is protected).
+7. Try gh_pullfrog/file_delete on .git/config (should fail — .git is protected).
+8. Call set_output with: READ=<content you read in step 3>,DELETED=true or DELETED=false (step 5 failed = file gone),GIT_BLOCKED=true or GIT_BLOCKED=false (steps 6 and 7 both rejected).`;
+
+const fixture = defineFixture(
+  {
+    prompt: PROMPT,
+    bash: "enabled",
+    effort: "mini",
+    timeout: "3m",
+  },
+  { localOnly: true }
+);
+
+const { getUuid, agentEnv } = generateAgentUuids(["PULLFROG_FILE_TEST"]);
+
+function validator(result: AgentResult): ValidationCheck[] {
+  const marker = getUuid(result.agent, "PULLFROG_FILE_TEST");
+  const output = getStructuredOutput(result);
+  const setOutputCalled = output !== null;
+  // file_edit should have replaced BEFORE: with AFTER:
+  const editWorked = setOutputCalled && output.includes(`AFTER:${marker}`);
+  const deleteWorked = setOutputCalled && /DELETED=true/i.test(output);
+  const gitBlocked = setOutputCalled && /GIT_BLOCKED=true/i.test(output);
+
+  return [
+    { name: "set_output", passed: setOutputCalled },
+    { name: "edit_worked", passed: editWorked },
+    { name: "delete_worked", passed: deleteWorked },
+    { name: "git_blocked", passed: gitBlocked },
+  ];
+}
+
+export const test: TestRunnerOptions = {
+  name: "file-read-write",
+  fixture,
+  validator,
+  agentEnv,
+  env: { GITHUB_REPOSITORY: "pullfrog/test-repo" },
+  tags: ["fs"],
+};
@@ -1,18 +1,22 @@
+import { randomUUID } from "node:crypto";
 import type { AgentResult, TestRunnerOptions, ValidationCheck } from "../utils.ts";
-import { defineFixture, generateAgentUuids, getStructuredOutput } from "../utils.ts";
+import { defineFixture, getStructuredOutput } from "../utils.ts";

 /**
 * MCP merge test - validates repo-level MCP servers merge correctly with gh_pullfrog.
 *
- * uses GITHUB_REPOSITORY=pullfrog/test-repo-mcp which has robinMCP server.
- * all agents should auto-discover repo-level MCP configs and merge them with gh_pullfrog.
+ * Uses GITHUB_REPOSITORY=pullfrog/test-repo-mcp whose robin-mcp reads a secret
+ * from /tmp/pullfrog-mcp-secret/secret.txt (outside the repo, unreachable via
+ * file_read) and exposes it via get_test_value. The runner writes the secret
+ * there via repoSetup before the agent starts. Runs in nobash mode.
 */

-const testUuids = generateAgentUuids(["PULLFROG_MCP_TEST"]);
+const secret = randomUUID();

 const fixture = defineFixture(
  {
    prompt: `Call the get_test_value tool from the robinMCP server, then call set_output with the exact value returned.`,
+    bash: "disabled",
    effort: "mini",
  },
  { localOnly: true }
@@ -21,8 +25,7 @@ const fixture = defineFixture(
 function validator(result: AgentResult): ValidationCheck[] {
  const output = getStructuredOutput(result);
  const setOutputCalled = output !== null;
-  const expectedUuid = testUuids.getUuid(result.agent, "PULLFROG_MCP_TEST");
-  const correctValue = setOutputCalled && output === expectedUuid;
+  const correctValue = setOutputCalled && output === secret;

  return [
    { name: "set_output", passed: setOutputCalled },
@@ -34,8 +37,10 @@ export const test: TestRunnerOptions = {
  name: "mcpmerge",
  fixture,
  validator,
-  tags: ["mcpmerge"],
-  env: { GITHUB_REPOSITORY: "pullfrog/test-repo-mcp" },
-  agentEnv: testUuids.agentEnv,
-  fileAgentEnv: testUuids.agentEnv,
+  env: {
+    GITHUB_REPOSITORY: "pullfrog/test-repo-mcp",
+    PULLFROG_MCP_SECRET: secret,
+  },
+  repoSetup:
+    'mkdir -p /tmp/pullfrog-mcp-secret && printf "%s" "$PULLFROG_MCP_SECRET" > /tmp/pullfrog-mcp-secret/secret.txt',
 };
@@ -0,0 +1,60 @@
+import type { AgentResult, TestRunnerOptions, ValidationCheck } from "../utils.ts";
+import { defineFixture, generateAgentUuids, getStructuredOutput } from "../utils.ts";
+
+/**
+ * noNativeFile test - validates native file read/write tools are disabled.
+ * agent must use MCP file_write; native tools should be unavailable.
+ *
+ * push is disabled so codex runs in read-only sandbox, which blocks its native
+ * apply_patch tool (there is no feature flag to disable it). MCP file_write
+ * still works because it runs server-side outside the sandbox.
+ */
+
+const PROMPT = `Get your marker by running: echo $PULLFROG_NOFILE_TEST (use gh_pullfrog/bash for shell commands).
+
+1. Try to call a NATIVE (non-MCP) file tool to write a file. Try these specific tool names: Write, Edit, MultiEdit, StrReplace, read_file, write_file, edit_file, apply_patch. These are agent-native tools, NOT MCP tools. Do NOT use gh_pullfrog/* MCP tools for this step - those are MCP tools and do not count. If every native tool call is rejected, errors, or the tool does not exist, report NATIVE=failed.
+2. Use the MCP tool gh_pullfrog/file_write to write mcp-worked.txt with your marker.
+3. Call set_output with: NATIVE=succeeded or NATIVE=failed, MCP=succeeded or MCP=failed.
+
+IMPORTANT: step 1 is about native/built-in tools only (NOT gh_pullfrog/* MCP tools). step 2 is about MCP tools only.`;
+
+const fixture = defineFixture(
+  {
+    prompt: PROMPT,
+    bash: "restricted",
+    push: "disabled",
+    effort: "mini",
+    timeout: "3m",
+  },
+  { localOnly: true }
+);
+
+const { agentEnv } = generateAgentUuids(["PULLFROG_NOFILE_TEST"]);
+
+function validator(result: AgentResult): ValidationCheck[] {
+  const output = getStructuredOutput(result);
+  const fullOutput = result.output;
+  const setOutputCalled = output !== null;
+  // native blocked if agent reports failed, or if "native" attempt actually used MCP (e.g. Task delegated to file_write)
+  // handle both key=value format (NATIVE=succeeded) and JSON format ("NATIVE":"succeeded")
+  const reportedNativeSucceeded = setOutputCalled && /NATIVE.{0,3}succeeded/i.test(output);
+  const nativeActuallyUsedMcp =
+    fullOutput.includes("delegated to") && fullOutput.includes("file_write");
+  const nativeBlocked = !reportedNativeSucceeded || nativeActuallyUsedMcp;
+  const mcpWorks = setOutputCalled && /MCP.{0,3}succeeded/i.test(output);
+
+  return [
+    { name: "set_output", passed: setOutputCalled },
+    { name: "native_blocked", passed: nativeBlocked },
+    { name: "mcp_works", passed: mcpWorks },
+  ];
+}
+
+export const test: TestRunnerOptions = {
+  name: "no-native-file",
+  fixture,
+  validator,
+  agentEnv,
+  env: { GITHUB_REPOSITORY: "pullfrog/test-repo" },
+  tags: ["fs"],
+};