add per-agent smoke tests (#100)
This commit is contained in:
David Blass
committed by
pullfrog[bot]
pullfrog[bot]
parent
9e019d89d2
commit
f34379415e
@@ -1,14 +0,0 @@
|
||||
import type { Inputs } from "../main.ts";
|
||||
|
||||
/**
|
||||
* test fixture: tests bash=disabled enforcement.
|
||||
* the agent should NOT be able to run bash commands.
|
||||
*
|
||||
* run with: AGENT_OVERRIDE=claude pnpm play bash-disabled.ts
|
||||
*/
|
||||
export default {
|
||||
prompt: `Run a simple bash command: echo "hello world"
|
||||
|
||||
If you cannot run this command, explain that bash is disabled.`,
|
||||
bash: "disabled",
|
||||
} satisfies Inputs;
|
||||
@@ -1,14 +0,0 @@
|
||||
import type { Inputs } from "../main.ts";
|
||||
|
||||
/**
|
||||
* test fixture: tests bash=restricted enforcement.
|
||||
* the agent should use MCP bash tool (not native bash).
|
||||
*
|
||||
* run with: AGENT_OVERRIDE=claude pnpm play bash-restricted.ts
|
||||
*/
|
||||
export default {
|
||||
prompt: `Run this bash command: echo "hello from restricted mode"
|
||||
|
||||
Use the gh_pullfrog/bash MCP tool since native bash is disabled for security.`,
|
||||
bash: "restricted",
|
||||
} satisfies Inputs;
|
||||
@@ -1,34 +0,0 @@
|
||||
import type { Payload } from "../external.ts";
|
||||
|
||||
/**
|
||||
* test fixture: verifies agents use MCP bash tool for shell commands.
|
||||
* creates a simple test file and runs it with node.
|
||||
*
|
||||
* for insecure agents (claude, cursor, opencode): native bash is disabled,
|
||||
* so they MUST use gh_pullfrog/bash MCP tool to run shell commands.
|
||||
*
|
||||
* for secure agents (codex, gemini): native bash is safe, but this test
|
||||
* still verifies shell execution works.
|
||||
*
|
||||
* run with: AGENT_OVERRIDE=<agent> pnpm play bash-test.ts
|
||||
*/
|
||||
export default {
|
||||
"~pullfrog": true,
|
||||
agent: null,
|
||||
prompt: `Create a file called test-runner.js with the following content:
|
||||
|
||||
\`\`\`javascript
|
||||
const assert = require('assert');
|
||||
assert.strictEqual(2 + 2, 4, 'math should work');
|
||||
console.log('TEST PASSED: basic arithmetic works');
|
||||
\`\`\`
|
||||
|
||||
Then run it with: node test-runner.js
|
||||
|
||||
Finally, delete the test file.
|
||||
|
||||
This tests that you can execute shell commands properly.`,
|
||||
event: {
|
||||
trigger: "workflow_dispatch",
|
||||
},
|
||||
} satisfies Payload;
|
||||
@@ -1,11 +0,0 @@
|
||||
import type { Payload } from "../external.ts";
|
||||
|
||||
export default {
|
||||
"~pullfrog": true,
|
||||
agent: null,
|
||||
prompt:
|
||||
"List all files in the current directory, then create a file called dynamic-test.txt with the content 'This was loaded from a TypeScript file!', then delete it.",
|
||||
event: {
|
||||
trigger: "workflow_dispatch",
|
||||
},
|
||||
} satisfies Payload;
|
||||
@@ -1,27 +0,0 @@
|
||||
import type { Effort, Payload } from "../external.ts";
|
||||
|
||||
/**
|
||||
* Test fixture for Claude effort levels.
|
||||
* Runs all three effort levels in sequence.
|
||||
*
|
||||
* Run with:
|
||||
* AGENT_OVERRIDE=claude pnpm play claude-effort.ts
|
||||
*
|
||||
* Effort levels:
|
||||
* - "mini": haiku (fast, efficient)
|
||||
* - "auto": opusplan (Opus for planning, Sonnet for execution)
|
||||
* - "max": opus (full Opus capability)
|
||||
*/
|
||||
|
||||
const efforts: Effort[] = ["mini", "auto", "max"];
|
||||
|
||||
export default efforts.map((effort) => ({
|
||||
"~pullfrog": true,
|
||||
agent: "claude",
|
||||
prompt: "What is 2 + 2? Reply with just the number.",
|
||||
event: {
|
||||
trigger: "workflow_dispatch",
|
||||
},
|
||||
modes: [],
|
||||
effort,
|
||||
})) satisfies Payload[];
|
||||
@@ -1,27 +0,0 @@
|
||||
import type { Effort, Payload } from "../external.ts";
|
||||
|
||||
/**
|
||||
* Test fixture for Codex effort levels.
|
||||
* Runs all three effort levels in sequence.
|
||||
*
|
||||
* Run with:
|
||||
* AGENT_OVERRIDE=codex pnpm play codex-effort.ts
|
||||
*
|
||||
* Effort levels:
|
||||
* - "mini": gpt-5.1-codex-mini + modelReasoningEffort: "low"
|
||||
* - "auto": gpt-5.1-codex + default reasoning
|
||||
* - "max": gpt-5.1-codex-max + modelReasoningEffort: "high"
|
||||
*/
|
||||
|
||||
const efforts: Effort[] = ["mini", "auto", "max"];
|
||||
|
||||
export default efforts.map((effort) => ({
|
||||
"~pullfrog": true,
|
||||
agent: "codex",
|
||||
prompt: "What is 2 + 2? Reply with just the number.",
|
||||
event: {
|
||||
trigger: "workflow_dispatch",
|
||||
},
|
||||
modes: [],
|
||||
effort,
|
||||
})) satisfies Payload[];
|
||||
@@ -1,30 +0,0 @@
|
||||
import type { Effort, Payload } from "../external.ts";
|
||||
|
||||
/**
|
||||
* Test fixture for Cursor effort levels.
|
||||
* Runs all three effort levels in sequence.
|
||||
*
|
||||
* Run with:
|
||||
* AGENT_OVERRIDE=cursor pnpm play cursor-effort.ts
|
||||
*
|
||||
* Effort levels:
|
||||
* - "mini": auto (default model)
|
||||
* - "auto": auto (default model)
|
||||
* - "max": opus-4.5-thinking
|
||||
*
|
||||
* Note: If project has .cursor/cli.json with "model" specified,
|
||||
* that takes precedence over effort-based model selection.
|
||||
*/
|
||||
|
||||
const efforts: Effort[] = ["mini", "auto", "max"];
|
||||
|
||||
export default efforts.map((effort) => ({
|
||||
"~pullfrog": true,
|
||||
agent: "cursor",
|
||||
prompt: "What is 2 + 2? Reply with just the number.",
|
||||
event: {
|
||||
trigger: "workflow_dispatch",
|
||||
},
|
||||
modes: [],
|
||||
effort,
|
||||
})) satisfies Payload[];
|
||||
@@ -1,27 +0,0 @@
|
||||
import type { Effort, Payload } from "../external.ts";
|
||||
|
||||
/**
|
||||
* Test fixture for Gemini effort levels.
|
||||
* Runs all three effort levels in sequence.
|
||||
*
|
||||
* Run with:
|
||||
* AGENT_OVERRIDE=gemini pnpm play gemini-effort.ts
|
||||
*
|
||||
* Effort levels:
|
||||
* - "mini": gemini-2.5-flash + LOW thinking
|
||||
* - "auto": gemini-2.5-flash + HIGH thinking
|
||||
* - "max": gemini-2.5-pro + HIGH thinking
|
||||
*/
|
||||
|
||||
const efforts: Effort[] = ["mini", "auto", "max"];
|
||||
|
||||
export default efforts.map((effort) => ({
|
||||
"~pullfrog": true,
|
||||
agent: "gemini",
|
||||
prompt: "What is 2 + 2? Reply with just the number.",
|
||||
event: {
|
||||
trigger: "workflow_dispatch",
|
||||
},
|
||||
modes: [],
|
||||
effort,
|
||||
})) satisfies Payload[];
|
||||
@@ -1 +0,0 @@
|
||||
Tell me a joke.
|
||||
@@ -1,22 +0,0 @@
|
||||
import type { Inputs } from "../main.ts";
|
||||
|
||||
/**
|
||||
* test fixture: tests granular tool permissions enforcement.
|
||||
* all tools are disabled, so web access, search, file writes, and bash should be blocked.
|
||||
*
|
||||
* run with: AGENT_OVERRIDE=claude pnpm play sandbox.ts
|
||||
*/
|
||||
export default {
|
||||
prompt: `Please do the following three things:
|
||||
|
||||
1. Fetch the content from https://httpbin.org/json and tell me what it says
|
||||
2. Create a file called sandbox-test.txt with the content "This should fail with write disabled"
|
||||
3. Run a bash command: echo "hello from bash" > bash-test.txt
|
||||
|
||||
All three of these actions should fail because tool permissions are restricted (web=disabled, write=disabled, bash=disabled).`,
|
||||
// granular tool permissions - all disabled
|
||||
web: "disabled",
|
||||
search: "disabled",
|
||||
write: "disabled",
|
||||
bash: "disabled",
|
||||
} satisfies Inputs;
|
||||
@@ -1,8 +0,0 @@
|
||||
import type { Inputs } from "../main.ts";
|
||||
|
||||
/**
|
||||
* simple test - no tool restrictions
|
||||
*/
|
||||
export default {
|
||||
prompt: `Just say "hello world" - no tools needed.`,
|
||||
} satisfies Inputs;
|
||||
Reference in New Issue
Block a user