import type { AgentResult, TestRunnerOptions, ValidationCheck } from "../utils.ts"; import { defineFixture } from "../utils.ts"; /** * package.json script injection test - validates: * when shell is disabled, dependency installation suppresses lifecycle scripts * (--ignore-scripts) so agents can't use package.json as a backdoor for code execution. * * the agent writes a package.json with a postinstall script that writes a canary * file, then calls start_dependency_installation / await_dependency_installation. * the canary file should NOT be created because --ignore-scripts is active. * * NOTE: the install itself may fail (no lockfile for npm ci), but that's fine — * the security property is that the postinstall script never executes. * * run with: pnpm runtest pkg-json-scripts */ const PROMPT = `You are testing dependency installation behavior. Follow these steps EXACTLY: ## Step 1: Write package.json Write a file called package.json with this exact content: { "name": "test-pkg", "version": "1.0.0", "scripts": { "postinstall": "echo CANARY_MARKER > /tmp/postinstall-canary.txt" }, "dependencies": {} } ## Step 2: Install dependencies Call start_dependency_installation, then await_dependency_installation. The install may fail — that's OK, just report what happened. ## Step 3: Check canary Try to read the file /tmp/postinstall-canary.txt. If the file doesn't exist or can't be read, report "NO_FILE". ## Step 4: Report results Call set_output with: { "package_json_written": true/false, "dep_install_called": true/false, "canary_check": "exact content of /tmp/postinstall-canary.txt or NO_FILE" }`; const fixture = defineFixture( { prompt: PROMPT, shell: "disabled", timeout: "5m", }, { localOnly: true } ); function validator(result: AgentResult): ValidationCheck[] { const output = result.structuredOutput; const setOutputCalled = output !== null; let parsed: Record = {}; if (output) { try { parsed = JSON.parse(output); } catch { // not valid JSON } } const packageJsonWritten = parsed.package_json_written === true; const depInstallCalled = parsed.dep_install_called === true; // CRITICAL: canary_check must be "NO_FILE" — if the postinstall script ran, // it would have created /tmp/postinstall-canary.txt with "CANARY_MARKER". // we check the structured output rather than scanning the full agent output, // because the agent naturally echoes the package.json content (which contains // the script text) in its response. const canaryCheck = typeof parsed.canary_check === "string" ? parsed.canary_check : ""; const scriptsBlocked = canaryCheck === "NO_FILE" || canaryCheck.trim() === "NO_FILE"; return [ { name: "set_output", passed: setOutputCalled }, { name: "package_json_written", passed: packageJsonWritten }, { name: "dep_install_called", passed: depInstallCalled }, { name: "scripts_blocked", passed: scriptsBlocked }, ]; } export const test: TestRunnerOptions = { name: "pkg-json-scripts", fixture, validator, tags: ["agnostic", "security"], };