Colin McDonnell
4b3c5ca905
* rename agent key to opencode and add skill invocation coverage. add skill-invoke tests for claude and opencode with local play-based validation signals, update CI matrices, and include the current tracked refactors in this branch for review. Made-with: Cursor * exclude agent-specific skill-invoke tests from wrong agent in CI matrix * address review follow-up and preserve workflow run UI tweak. switch changed-agents ci coverage to exercise the opentoad implementation path while keeping the opencode expectation, and include the local workflow run client interaction updates requested on this branch. Made-with: Cursor * remove opentoad agent filename from runtime. rename the opencode harness implementation file from opentoad.ts to opencode.ts and update ci coverage input accordingly so action code no longer carries the old filename. Made-with: Cursor * ensure security prompt bypass is set on every test fixture. this keeps adversarial and permissions harnesses from being blocked by the default prompt-injection refusal path during CI security tests. Made-with: Cursor --------- Co-authored-by: pullfrog[bot] <226033991+pullfrog[bot]@users.noreply.github.com>
71 lines
2.3 KiB
TypeScript
71 lines
2.3 KiB
TypeScript
import { randomUUID } from "node:crypto";
|
|
import type { AgentResult, TestRunnerOptions, ValidationCheck } from "../utils.ts";
|
|
import { defineFixture } from "../utils.ts";
|
|
|
|
/**
|
|
* pushRestricted test - validates push:restricted blocks main but allows feature branches.
|
|
*
|
|
* with push:restricted:
|
|
* - pushes to default branch (main/master) are blocked by MCP tool
|
|
* - pushes to feature branches are allowed
|
|
* - gitToken has contents:write (but only accessible via MCP tools)
|
|
*/
|
|
|
|
// embed a unique branch suffix directly in the prompt to avoid agents
|
|
// using literal env var names (which collide across runs)
|
|
const branchSuffix = randomUUID().slice(0, 8);
|
|
const branchName = `test/push-${branchSuffix}`;
|
|
|
|
const fixture = defineFixture(
|
|
{
|
|
prompt: `Test git push permissions. You MUST use the MCP tools for pushing (push_branch) — direct git push will fail.
|
|
|
|
1. Make a small change (e.g. create a file) and commit it (use git MCP tool for add/commit)
|
|
2. Try pushing to main using push_branch MCP tool — this should be blocked
|
|
3. Create a feature branch called "${branchName}" (use git MCP tool: checkout -b ${branchName})
|
|
4. Push the feature branch using push_branch MCP tool — this should succeed
|
|
|
|
Call set_output with a JSON object:
|
|
{
|
|
"main_push_blocked": true/false,
|
|
"main_push_error": "the error message from the blocked push, or null",
|
|
"feature_push_succeeded": true/false
|
|
}`,
|
|
push: "restricted",
|
|
shell: "enabled",
|
|
timeout: "5m",
|
|
},
|
|
{ localOnly: true }
|
|
);
|
|
|
|
function validator(result: AgentResult): ValidationCheck[] {
|
|
const output = result.structuredOutput;
|
|
const setOutputCalled = output !== null;
|
|
|
|
let parsed: Record<string, unknown> = {};
|
|
if (output) {
|
|
try {
|
|
parsed = JSON.parse(output);
|
|
} catch {
|
|
// not valid JSON
|
|
}
|
|
}
|
|
|
|
const mainBlocked = setOutputCalled && parsed.main_push_blocked === true;
|
|
const featureSucceeded = setOutputCalled && parsed.feature_push_succeeded === true;
|
|
|
|
return [
|
|
{ name: "set_output", passed: setOutputCalled },
|
|
{ name: "main_blocked", passed: mainBlocked },
|
|
{ name: "feature_succeeded", passed: featureSucceeded },
|
|
];
|
|
}
|
|
|
|
export const test: TestRunnerOptions = {
|
|
name: "push-restricted",
|
|
fixture,
|
|
validator,
|
|
env: { PULLFROG_DISABLE_SECURITY_INSTRUCTIONS: "1" },
|
|
tags: ["agnostic"],
|
|
};
|