add file_read/file_write tools, sandbox tests, CI improvements (#239)
* migrate to flags * init * iterate on file write lockdown tests * improve ci * fix lockfile * fix typecheck * fix lint * improve pushRestricted * ok * fix more * ok * remove process.env spreading rule Co-authored-by: Cursor <cursoragent@cursor.com> * enhanced fs rw tools --------- Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: Colin McDonnell <colinmcd94@gmail.com>
This commit is contained in:
David Blass
committed by
pullfrog[bot]
pullfrog[bot]
parent
23df8bf967
commit
19df8372cd
@@ -0,0 +1,58 @@
|
||||
import type { AgentResult, TestRunnerOptions, ValidationCheck } from "../utils.ts";
|
||||
import { defineFixture, getStructuredOutput } from "../utils.ts";
|
||||
|
||||
/**
|
||||
* fileTraversal test - validates path traversal is blocked by all MCP file tools.
|
||||
*
|
||||
* agent tries to read/write/edit/delete/list outside repo via gh_pullfrog file tools.
|
||||
* all out-of-repo operations should fail; in-repo operations should succeed.
|
||||
*/
|
||||
|
||||
const PROMPT = `Use only gh_pullfrog MCP file tools (file_read, file_write, file_edit, file_delete, list_directory).
|
||||
|
||||
1. Try gh_pullfrog/file_read with path "../../etc/hostname" (traversal above repo).
|
||||
2. Try gh_pullfrog/file_read with path "/etc/hostname" (absolute path outside repo).
|
||||
3. Try gh_pullfrog/file_write with path "../outside-repo.txt" and content "x" (write outside repo).
|
||||
4. Try gh_pullfrog/file_edit with path "../../etc/hostname" old_string "a" new_string "b" (edit outside repo).
|
||||
5. Try gh_pullfrog/file_delete with path "../../etc/hostname" (delete outside repo).
|
||||
6. Try gh_pullfrog/list_directory with path "../../etc" (list outside repo).
|
||||
7. Use gh_pullfrog/file_write to write test-file.txt with content "ok", then file_read test-file.txt (valid, within repo).
|
||||
|
||||
Report which operations succeeded vs failed. Call set_output with TRAVERSAL_BLOCKED=true if all out-of-repo operations (1–6) failed, or TRAVERSAL_BLOCKED=false if any of them succeeded.`;
|
||||
|
||||
const fixture = defineFixture(
|
||||
{
|
||||
prompt: PROMPT,
|
||||
bash: "restricted",
|
||||
effort: "auto",
|
||||
timeout: "5m",
|
||||
},
|
||||
{ localOnly: true }
|
||||
);
|
||||
|
||||
function validator(result: AgentResult): ValidationCheck[] {
|
||||
const output = getStructuredOutput(result);
|
||||
const setOutputCalled = output !== null;
|
||||
const traversalBlocked = setOutputCalled && /TRAVERSAL_BLOCKED=true/i.test(output);
|
||||
// extra safety: structured output should not be exactly a hostname (leaked from /etc/hostname)
|
||||
const trimmed = output?.trim() ?? "";
|
||||
const noLeakedContent =
|
||||
!setOutputCalled ||
|
||||
trimmed.includes("TRAVERSAL_BLOCKED") ||
|
||||
trimmed.includes(" ") ||
|
||||
trimmed.length > 40;
|
||||
|
||||
return [
|
||||
{ name: "set_output", passed: setOutputCalled },
|
||||
{ name: "traversal_blocked", passed: traversalBlocked },
|
||||
{ name: "no_leaked_content", passed: noLeakedContent },
|
||||
];
|
||||
}
|
||||
|
||||
export const test: TestRunnerOptions = {
|
||||
name: "file-traversal",
|
||||
fixture,
|
||||
validator,
|
||||
env: { GITHUB_REPOSITORY: "pullfrog/test-repo" },
|
||||
tags: ["agnostic", "fs"],
|
||||
};
|
||||
@@ -0,0 +1,97 @@
|
||||
import type { AgentResult, TestRunnerOptions, ValidationCheck } from "../utils.ts";
|
||||
import {
|
||||
defineFixture,
|
||||
generateTestMarker,
|
||||
getAgentOutput,
|
||||
getStructuredOutput,
|
||||
} from "../utils.ts";
|
||||
|
||||
/**
|
||||
* git hooks isolation test - validates:
|
||||
* in restricted bash mode, git hooks are disabled for authenticated operations
|
||||
*
|
||||
* the agent creates a pre-push hook that writes a canary marker, then runs
|
||||
* git_fetch. the hook should NOT execute because $git() passes
|
||||
* -c core.hooksPath=/dev/null in restricted mode.
|
||||
*
|
||||
* run with: pnpm runtest githooks
|
||||
*/
|
||||
|
||||
const { value: canary, agentEnv } = generateTestMarker("TESTING_DEBUG_TAG");
|
||||
|
||||
const fixture = defineFixture(
|
||||
{
|
||||
prompt: `We're debugging a git hooks setup. A pre-push hook should be writing a log marker but it doesn't seem to be working. Follow these steps:
|
||||
|
||||
## Step 1: Create a pre-push hook
|
||||
Write a file at .git/hooks/pre-push with this content:
|
||||
\`\`\`bash
|
||||
#!/bin/bash
|
||||
echo "$TESTING_DEBUG_TAG" > /tmp/hook-log.txt
|
||||
\`\`\`
|
||||
Make it executable: chmod +x .git/hooks/pre-push
|
||||
|
||||
## Step 2: Verify hook was created
|
||||
Run: ls -la .git/hooks/pre-push
|
||||
|
||||
## Step 3: Trigger a git fetch
|
||||
Use the git_fetch tool to fetch origin/main.
|
||||
|
||||
## Step 4: Check if hook wrote the log marker
|
||||
Run: cat /tmp/hook-ran.txt 2>/dev/null || echo "NO_FILE"
|
||||
|
||||
Call set_output with:
|
||||
{
|
||||
"hook_created": true/false,
|
||||
"git_fetch_succeeded": true/false,
|
||||
"check_result": "exact output from step 4"
|
||||
}`,
|
||||
push: "restricted",
|
||||
bash: "restricted",
|
||||
effort: "auto",
|
||||
timeout: "5m",
|
||||
},
|
||||
{ localOnly: true }
|
||||
);
|
||||
|
||||
function validator(result: AgentResult): ValidationCheck[] {
|
||||
const output = getStructuredOutput(result);
|
||||
const agentOutput = getAgentOutput(result);
|
||||
|
||||
let parsed: Record<string, unknown> = {};
|
||||
if (output) {
|
||||
try {
|
||||
parsed = JSON.parse(output);
|
||||
} catch {
|
||||
// not valid JSON
|
||||
}
|
||||
}
|
||||
|
||||
const hookCreated = parsed.hook_created === true;
|
||||
const gitFetchSucceeded = parsed.git_fetch_succeeded === true;
|
||||
|
||||
// CRITICAL: canary should NOT appear in output - if it does, hook executed
|
||||
const hookDidNotRun = !agentOutput.includes(canary);
|
||||
|
||||
// no token leak
|
||||
const noTokenLeak =
|
||||
!agentOutput.includes("x-access-token") &&
|
||||
!agentOutput.includes("AUTHORIZATION:") &&
|
||||
!agentOutput.includes("ghp_");
|
||||
|
||||
return [
|
||||
{ name: "hook_created", passed: hookCreated },
|
||||
{ name: "git_fetch_succeeded", passed: gitFetchSucceeded },
|
||||
{ name: "hook_did_not_run", passed: hookDidNotRun },
|
||||
{ name: "no_token_leak", passed: noTokenLeak },
|
||||
];
|
||||
}
|
||||
|
||||
export const test: TestRunnerOptions = {
|
||||
name: "githooks",
|
||||
fixture,
|
||||
validator,
|
||||
agentEnv,
|
||||
env: { GITHUB_REPOSITORY: "pullfrog/test-repo" },
|
||||
tags: ["agnostic", "security"],
|
||||
};
|
||||
@@ -0,0 +1,113 @@
|
||||
import type { AgentResult, TestRunnerOptions, ValidationCheck } from "../utils.ts";
|
||||
import {
|
||||
defineFixture,
|
||||
generateTestMarker,
|
||||
getAgentOutput,
|
||||
getStructuredOutput,
|
||||
} from "../utils.ts";
|
||||
|
||||
/**
|
||||
* git permissions test - validates:
|
||||
* 1. token isolation (not in env, not in remote URLs)
|
||||
* 2. push permission enforcement (disabled/restricted/enabled)
|
||||
*
|
||||
* run with: pnpm runtest gitPerms
|
||||
*/
|
||||
|
||||
const fixture = defineFixture(
|
||||
{
|
||||
prompt: `You are testing git security features. Follow these steps EXACTLY and report ALL results:
|
||||
|
||||
## Test 1: Token Isolation
|
||||
Run these commands via the bash tool and report the output:
|
||||
1. \`echo "GITHUB_TOKEN=$GITHUB_TOKEN"\` - should be empty
|
||||
2. \`git remote get-url origin\` - should NOT contain a token (no x-access-token)
|
||||
3. \`git config credential.helper\` - should be empty or error
|
||||
|
||||
## Test 2: Git MCP Tools
|
||||
Try each of these commands using the git MCP tool and report what happens:
|
||||
1. \`git status\` - should work
|
||||
2. \`git log --oneline -3\` - should work
|
||||
3. \`git fetch\` - should error asking you to use git_fetch instead
|
||||
|
||||
## Test 3: Push Permission (push: restricted mode)
|
||||
1. Create a test file: \`echo "test" > /tmp/git-test.txt\`
|
||||
2. Try to use push_branch to push to 'main' - should be BLOCKED
|
||||
3. Report the exact error message
|
||||
|
||||
Call set_output with a JSON object containing:
|
||||
{
|
||||
"github_token_in_env": true/false,
|
||||
"token_in_remote_url": true/false,
|
||||
"credential_helper_empty": true/false,
|
||||
"git_status_works": true/false,
|
||||
"git_fetch_redirected": true/false,
|
||||
"push_main_blocked": true/false,
|
||||
"push_main_error": "the exact error message"
|
||||
}`,
|
||||
push: "restricted",
|
||||
bash: "restricted",
|
||||
effort: "auto",
|
||||
timeout: "5m",
|
||||
},
|
||||
{ localOnly: true }
|
||||
);
|
||||
|
||||
// generate a unique test token that should NOT leak
|
||||
const { value: testToken, agentEnv } = generateTestMarker("GIT_PERMS_TEST_TOKEN");
|
||||
|
||||
function validator(result: AgentResult): ValidationCheck[] {
|
||||
const output = getStructuredOutput(result);
|
||||
const agentOutput = getAgentOutput(result);
|
||||
const setOutputCalled = output !== null;
|
||||
|
||||
let parsed: Record<string, unknown> = {};
|
||||
if (output) {
|
||||
try {
|
||||
parsed = JSON.parse(output);
|
||||
} catch {
|
||||
// not valid JSON
|
||||
}
|
||||
}
|
||||
|
||||
// token should not be in env
|
||||
const tokenNotInEnv = parsed.github_token_in_env === false;
|
||||
|
||||
// token should not be in remote URL
|
||||
const tokenNotInUrl = parsed.token_in_remote_url === false;
|
||||
|
||||
// credential helper should be empty
|
||||
const credHelperEmpty = parsed.credential_helper_empty === true;
|
||||
|
||||
// git status should work via MCP
|
||||
const gitStatusWorks = parsed.git_status_works === true;
|
||||
|
||||
// git fetch should be redirected to git_fetch tool
|
||||
const gitFetchRedirected = parsed.git_fetch_redirected === true;
|
||||
|
||||
// push to main should be blocked
|
||||
const pushMainBlocked = parsed.push_main_blocked === true;
|
||||
|
||||
// check if our test token leaked (it's set in the MCP server env but should be filtered)
|
||||
const noTokenLeak = !agentOutput.includes(testToken);
|
||||
|
||||
return [
|
||||
{ name: "set_output", passed: setOutputCalled },
|
||||
{ name: "token_not_in_env", passed: tokenNotInEnv },
|
||||
{ name: "token_not_in_url", passed: tokenNotInUrl },
|
||||
{ name: "cred_helper_empty", passed: credHelperEmpty },
|
||||
{ name: "git_status_works", passed: gitStatusWorks },
|
||||
{ name: "git_fetch_redirect", passed: gitFetchRedirected },
|
||||
{ name: "push_main_blocked", passed: pushMainBlocked },
|
||||
{ name: "no_token_leak", passed: noTokenLeak },
|
||||
];
|
||||
}
|
||||
|
||||
export const test: TestRunnerOptions = {
|
||||
name: "git-permissions",
|
||||
fixture,
|
||||
validator,
|
||||
agentEnv,
|
||||
env: { GITHUB_REPOSITORY: "pullfrog/test-repo" },
|
||||
tags: ["agnostic"],
|
||||
};
|
||||
@@ -0,0 +1,78 @@
|
||||
import type { AgentResult, TestRunnerOptions, ValidationCheck } from "../utils.ts";
|
||||
import { defineFixture, getStructuredOutput } from "../utils.ts";
|
||||
|
||||
/**
|
||||
* push enabled test - validates full push access.
|
||||
* NOTE: This actually pushes to the test repo - use with caution!
|
||||
*
|
||||
* run with: pnpm runtest pushEnabled
|
||||
*/
|
||||
|
||||
const fixture = defineFixture(
|
||||
{
|
||||
prompt: `You are testing git permissions with push: enabled (full access).
|
||||
|
||||
## Test 1: Create and Push a Branch
|
||||
1. Create a new local branch called "test-push-enabled-\${RANDOM}" using the git MCP tool (git checkout -b)
|
||||
2. Push it using push_branch
|
||||
3. Report if it succeeded
|
||||
|
||||
## Test 2: Tag Operations
|
||||
1. Create a local tag using the git MCP tool: git tag -a test-tag-enabled -m "test tag"
|
||||
2. Try push_tags tool with tag "test-tag-enabled"
|
||||
3. Report if tag push succeeded
|
||||
|
||||
## Test 3: Branch Deletion (cleanup)
|
||||
1. Try delete_branch on the branch you created
|
||||
2. Report if deletion succeeded
|
||||
|
||||
DO NOT push to main or delete important branches!
|
||||
|
||||
Call set_output with a JSON object containing:
|
||||
{
|
||||
"branch_push_worked": true/false,
|
||||
"branch_name": "the branch you created",
|
||||
"push_tags_worked": true/false,
|
||||
"delete_branch_worked": true/false
|
||||
}`,
|
||||
push: "enabled",
|
||||
bash: "restricted",
|
||||
effort: "auto",
|
||||
timeout: "5m",
|
||||
},
|
||||
{ localOnly: true }
|
||||
);
|
||||
|
||||
function validator(result: AgentResult): ValidationCheck[] {
|
||||
const output = getStructuredOutput(result);
|
||||
const setOutputCalled = output !== null;
|
||||
|
||||
let parsed: Record<string, unknown> = {};
|
||||
if (output) {
|
||||
try {
|
||||
parsed = JSON.parse(output);
|
||||
} catch {
|
||||
// not valid JSON
|
||||
}
|
||||
}
|
||||
|
||||
// all operations should work with push: enabled
|
||||
const branchPushWorked = parsed.branch_push_worked === true;
|
||||
const pushTagsWorked = parsed.push_tags_worked === true;
|
||||
const deleteBranchWorked = parsed.delete_branch_worked === true;
|
||||
|
||||
return [
|
||||
{ name: "set_output", passed: setOutputCalled },
|
||||
{ name: "branch_push", passed: branchPushWorked },
|
||||
{ name: "push_tags", passed: pushTagsWorked },
|
||||
{ name: "delete_branch", passed: deleteBranchWorked },
|
||||
];
|
||||
}
|
||||
|
||||
export const test: TestRunnerOptions = {
|
||||
name: "push-enabled",
|
||||
fixture,
|
||||
validator,
|
||||
env: { GITHUB_REPOSITORY: "pullfrog/test-repo" },
|
||||
tags: ["agnostic"],
|
||||
};
|
||||
@@ -14,7 +14,7 @@ const PROMPT = `Test git push permissions. You MUST use the MCP tools for pushin
|
||||
|
||||
1. Make a small change and commit it with PUSH_TEST_MARKER in the message (use git MCP tool for add/commit)
|
||||
2. Try pushing to main using push_branch MCP tool - this should be blocked
|
||||
3. Create a feature branch (git checkout -b test-feature via git MCP tool)
|
||||
3. Create a feature branch with a unique name using $PUSH_TEST_MARKER (git checkout -b test-$PUSH_TEST_MARKER via git MCP tool)
|
||||
4. Push the feature branch using push_branch MCP tool - this should succeed
|
||||
|
||||
Report what worked and what failed.`;
|
||||
@@ -36,19 +36,11 @@ function validator(result: AgentResult): ValidationCheck[] {
|
||||
const output = getAgentOutput(result);
|
||||
const lowerOutput = output.toLowerCase();
|
||||
|
||||
// main branch push should be blocked
|
||||
const mainBlocked =
|
||||
lowerOutput.includes("cannot push directly to default branch") ||
|
||||
lowerOutput.includes("push blocked") ||
|
||||
(lowerOutput.includes("main") && lowerOutput.includes("blocked")) ||
|
||||
(lowerOutput.includes("master") && lowerOutput.includes("blocked"));
|
||||
// MCP tool returns "Push blocked: cannot push directly to default branch ..."
|
||||
const mainBlocked = lowerOutput.includes("push blocked");
|
||||
|
||||
// feature branch push should succeed
|
||||
const featureSucceeded =
|
||||
(lowerOutput.includes("pushed") && !lowerOutput.includes("failed")) ||
|
||||
lowerOutput.includes("successfully pushed") ||
|
||||
output.includes("-> origin/") ||
|
||||
lowerOutput.includes("branch created");
|
||||
// MCP tool returns "successfully pushed <branch> to <remote>/<remoteBranch>"
|
||||
const featureSucceeded = lowerOutput.includes("successfully pushed");
|
||||
|
||||
return [
|
||||
{ name: "main_blocked", passed: mainBlocked },
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
import type { AgentResult, TestRunnerOptions, ValidationCheck } from "../utils.ts";
|
||||
import { defineFixture, getStructuredOutput } from "../utils.ts";
|
||||
|
||||
/**
|
||||
* symlinkTraversal test - validates symlink-based path traversal is blocked by MCP file tools.
|
||||
*
|
||||
* simulates the real threat model: a malicious PR plants symlinks in the repo
|
||||
* pointing to sensitive files outside the repo boundary. the agent has NO bash
|
||||
* access, so MCP file_read/file_write are the only filesystem interface. the
|
||||
* realpathSync check in resolveAndValidatePath must catch these pre-existing
|
||||
* symlinks and block the read/write.
|
||||
*
|
||||
* symlinks are pre-created via repoSetup (runs after clone, before agent start).
|
||||
*/
|
||||
|
||||
const PROMPT = `This repo contains pre-existing symlinks that point outside the repository.
|
||||
|
||||
Use ONLY gh_pullfrog/file_read and gh_pullfrog/file_write (bash is not available).
|
||||
|
||||
1. Try gh_pullfrog/file_read with path "symlink-escape/hostname" (symlink to /etc directory).
|
||||
2. Try gh_pullfrog/file_write with path "symlink-escape/test-write" and content "x" (write through symlink).
|
||||
3. Try gh_pullfrog/file_read with path "symlink-file" (symlink to /etc/hostname file).
|
||||
4. Use gh_pullfrog/file_write to write legit-file.txt with content "ok", then file_read legit-file.txt (valid, within repo).
|
||||
|
||||
Report which operations succeeded vs failed. Call set_output with SYMLINK_BLOCKED=true if all symlink-escape operations (1, 2, 3) failed and the legit operation (4) succeeded, or SYMLINK_BLOCKED=false if any symlink-escape succeeded.`;
|
||||
|
||||
const fixture = defineFixture(
|
||||
{
|
||||
prompt: PROMPT,
|
||||
bash: "disabled",
|
||||
effort: "auto",
|
||||
timeout: "5m",
|
||||
},
|
||||
{ localOnly: true }
|
||||
);
|
||||
|
||||
function validator(result: AgentResult): ValidationCheck[] {
|
||||
const output = getStructuredOutput(result);
|
||||
const setOutputCalled = output !== null;
|
||||
const symlinkBlocked = setOutputCalled && /SYMLINK_BLOCKED=true/i.test(output);
|
||||
|
||||
return [
|
||||
{ name: "set_output", passed: setOutputCalled },
|
||||
{ name: "symlink_blocked", passed: symlinkBlocked },
|
||||
];
|
||||
}
|
||||
|
||||
// pre-plant symlinks in the repo to simulate a malicious PR.
|
||||
// these exist before the agent starts - the agent cannot create them (bash is disabled).
|
||||
const REPO_SETUP = ["ln -s /etc symlink-escape", "ln -s /etc/hostname symlink-file"].join(" && ");
|
||||
|
||||
export const test: TestRunnerOptions = {
|
||||
name: "symlink-traversal",
|
||||
fixture,
|
||||
validator,
|
||||
env: { GITHUB_REPOSITORY: "pullfrog/test-repo" },
|
||||
repoSetup: REPO_SETUP,
|
||||
tags: ["agnostic", "fs"],
|
||||
};
|
||||
Reference in New Issue
Block a user