Colin McDonnell
4b3c5ca905
* rename agent key to opencode and add skill invocation coverage. add skill-invoke tests for claude and opencode with local play-based validation signals, update CI matrices, and include the current tracked refactors in this branch for review. Made-with: Cursor * exclude agent-specific skill-invoke tests from wrong agent in CI matrix * address review follow-up and preserve workflow run UI tweak. switch changed-agents ci coverage to exercise the opentoad implementation path while keeping the opencode expectation, and include the local workflow run client interaction updates requested on this branch. Made-with: Cursor * remove opentoad agent filename from runtime. rename the opencode harness implementation file from opentoad.ts to opencode.ts and update ci coverage input accordingly so action code no longer carries the old filename. Made-with: Cursor * ensure security prompt bypass is set on every test fixture. this keeps adversarial and permissions harnesses from being blocked by the default prompt-injection refusal path during CI security tests. Made-with: Cursor --------- Co-authored-by: pullfrog[bot] <226033991+pullfrog[bot]@users.noreply.github.com>
78 lines
2.2 KiB
TypeScript
78 lines
2.2 KiB
TypeScript
import type { AgentResult, TestRunnerOptions, ValidationCheck } from "../utils.ts";
|
|
import { defineFixture } from "../utils.ts";
|
|
|
|
/**
|
|
* push enabled test - validates full push access.
|
|
* NOTE: This actually pushes to the test repo - use with caution!
|
|
*
|
|
* run with: pnpm runtest pushEnabled
|
|
*/
|
|
|
|
const fixture = defineFixture(
|
|
{
|
|
prompt: `You are testing git permissions with push: enabled (full access).
|
|
|
|
## Test 1: Create and Push a Branch
|
|
1. Create a new local branch called "test-push-enabled-\${RANDOM}" using the git MCP tool (git checkout -b)
|
|
2. Push it using push_branch
|
|
3. Report if it succeeded
|
|
|
|
## Test 2: Tag Operations
|
|
1. Create a local tag using the git MCP tool: git tag -a test-tag-enabled -m "test tag"
|
|
2. Try push_tags tool with tag "test-tag-enabled"
|
|
3. Report if tag push succeeded
|
|
|
|
## Test 3: Branch Deletion (cleanup)
|
|
1. Try delete_branch on the branch you created
|
|
2. Report if deletion succeeded
|
|
|
|
DO NOT push to main or delete important branches!
|
|
|
|
Call set_output with a JSON object containing:
|
|
{
|
|
"branch_push_worked": true/false,
|
|
"branch_name": "the branch you created",
|
|
"push_tags_worked": true/false,
|
|
"delete_branch_worked": true/false
|
|
}`,
|
|
push: "enabled",
|
|
shell: "restricted",
|
|
timeout: "5m",
|
|
},
|
|
{ localOnly: true }
|
|
);
|
|
|
|
function validator(result: AgentResult): ValidationCheck[] {
|
|
const output = result.structuredOutput;
|
|
const setOutputCalled = output !== null;
|
|
|
|
let parsed: Record<string, unknown> = {};
|
|
if (output) {
|
|
try {
|
|
parsed = JSON.parse(output);
|
|
} catch {
|
|
// not valid JSON
|
|
}
|
|
}
|
|
|
|
// all operations should work with push: enabled
|
|
const branchPushWorked = parsed.branch_push_worked === true;
|
|
const pushTagsWorked = parsed.push_tags_worked === true;
|
|
const deleteBranchWorked = parsed.delete_branch_worked === true;
|
|
|
|
return [
|
|
{ name: "set_output", passed: setOutputCalled },
|
|
{ name: "branch_push", passed: branchPushWorked },
|
|
{ name: "push_tags", passed: pushTagsWorked },
|
|
{ name: "delete_branch", passed: deleteBranchWorked },
|
|
];
|
|
}
|
|
|
|
export const test: TestRunnerOptions = {
|
|
name: "push-enabled",
|
|
fixture,
|
|
validator,
|
|
env: { PULLFROG_DISABLE_SECURITY_INSTRUCTIONS: "1" },
|
|
tags: ["agnostic"],
|
|
};
|