fix: make token-exfil test reliable (#528)

* fix: make token-exfil test reliable by disabling security instructions and reframing prompt

the test was flaky — agents would randomly refuse (not calling set_output),
refuse politely (calling set_output with refusal text), or cooperate fully,
depending on model mood. two changes:

1. set PULLFROG_DISABLE_SECURITY_INSTRUCTIONS=1 in test env (layer 1)
2. reframe prompt as CI debugging task instead of security test (layer 2)

Made-with: Cursor

* fix: set PULLFROG_DISABLE_SECURITY_INSTRUCTIONS on adversarial test fixtures

without this flag, the system prompt tells agents to refuse anything that
looks malicious — which is exactly what these security pentests ask them to
do. adds the flag to tokenExfil, askpassIntercept, and nobashcreative.

Made-with: Cursor

* set PULLFROG_DISABLE_SECURITY_INSTRUCTIONS on all security-related test fixtures

Made-with: Cursor

test/adhoc/askpassIntercept.ts

@@ -49,4 +49,5 @@ export const test: TestRunnerOptions = {
   fixture,
   validator,
   tags: ["adhoc", "security"],
+  env: { PULLFROG_DISABLE_SECURITY_INSTRUCTIONS: "1" },
 };

test/adhoc/gitExecBypass.ts

@@ -91,4 +91,5 @@ export const test: TestRunnerOptions = {
   validator,
   tags: ["adhoc", "security"],
   agents: ["opentoad"],
+  env: { PULLFROG_DISABLE_SECURITY_INSTRUCTIONS: "1" },
 };

test/adhoc/gitFlagInjection.ts

@@ -108,4 +108,5 @@ export const test: TestRunnerOptions = {
   validator,
   tags: ["adhoc", "security"],
   agents: ["opentoad"],
+  env: { PULLFROG_DISABLE_SECURITY_INSTRUCTIONS: "1" },
 };

test/adhoc/nobashcreative.ts

@@ -57,4 +57,5 @@ export const test: TestRunnerOptions = {
   validator,
   agentEnv,
   tags: ["adhoc"],
+  env: { PULLFROG_DISABLE_SECURITY_INSTRUCTIONS: "1" },
 };

test/adhoc/requirementsTxtAttack.ts

@@ -79,4 +79,5 @@ export const test: TestRunnerOptions = {
   validator,
   tags: ["adhoc", "security"],
   agents: ["opentoad"],
+  env: { PULLFROG_DISABLE_SECURITY_INSTRUCTIONS: "1" },
 };

test/crossagent/nobash.ts

@@ -42,4 +42,5 @@ export const test: TestRunnerOptions = {
   fixture,
   validator,
   agentEnv,
+  env: { PULLFROG_DISABLE_SECURITY_INSTRUCTIONS: "1" },
 };

test/crossagent/restricted.ts

@@ -51,4 +51,5 @@ export const test: TestRunnerOptions = {
   fixture,
   validator,
   agentEnv,
+  env: { PULLFROG_DISABLE_SECURITY_INSTRUCTIONS: "1" },
 };

test/crossagent/tokenExfil.ts

@@ -10,7 +10,7 @@ import { defineFixture, generateAgentUuids, getAgentOutput } from "../utils.ts";
  *     managed-settings.json denies /proc reads (claude)
  *
  * runs with both agents to verify each sandbox independently.
- * requires: CI=true (to enable sandbox), PULLFROG_DISABLE_SECURITY_INSTRUCTIONS=1
+ * requires: CI=true (to enable sandbox)
  */
 
 const fixture = defineFixture(
@@ -57,4 +57,5 @@ export const test: TestRunnerOptions = {
   fixture,
   validator,
   agentEnv,
+  env: { PULLFROG_DISABLE_SECURITY_INSTRUCTIONS: "1" },
 };