import type { AgentResult, TestRunnerOptions, ValidationCheck } from "../utils.ts"; import { buildShellToolPrompt, defineFixture, generateAgentUuids } from "../utils.ts"; /** * restricted test - validates shell=restricted environment filtering. * checks both MCP and internal agent shell tools filter sensitive env vars. */ const fixture = defineFixture( { prompt: `This is a test to determine token visibility in shell tool calls. ${buildShellToolPrompt("echo $RUNNER_TEST_VALUE")} Then also run: echo $RUNNER_TEST_TOKEN Then call set_output with the exact output of each command, one per line: DIAGNOSTIC_ID= FILTER_TOKEN=`, shell: "restricted", timeout: "3m", }, { localOnly: true } ); const { getUuid, agentEnv } = generateAgentUuids(["RUNNER_TEST_VALUE", "RUNNER_TEST_TOKEN"]); function validator(result: AgentResult): ValidationCheck[] { const safeMarker = getUuid(result.agent, "RUNNER_TEST_VALUE"); const filteredMarker = getUuid(result.agent, "RUNNER_TEST_TOKEN"); // require structured output from set_output tool const output = result.structuredOutput; const setOutputCalled = output !== null; // non-sensitive env var SHOULD appear in output (agent can read it via shell) const canReadSafe = setOutputCalled && output.includes(safeMarker); // _TOKEN env var should NOT appear in output (filtered by shell) const noLeakFiltered = !setOutputCalled || !output.includes(filteredMarker); return [ { name: "set_output", passed: setOutputCalled }, { name: "can_read_safe", passed: canReadSafe }, { name: "no_leak_filtered", passed: noLeakFiltered }, ]; } export const test: TestRunnerOptions = { name: "restricted", fixture, validator, agentEnv, env: { PULLFROG_DISABLE_SECURITY_INSTRUCTIONS: "1" }, coverage: [ "action/utils/normalizeEnv.ts", "action/mcp/shell.ts", "action/agents/{claude,opencode,opencode_v2}.ts", ], };