improve runtest, optimize CI batching (#210)

This commit is contained in:
David Blass
2026-02-01 21:48:53 +00:00
committed by pullfrog[bot]
parent 2b3bd97b86
commit 18ba8e5fd0
19 changed files with 1534 additions and 514 deletions
+8 -12
View File
@@ -1,31 +1,27 @@
import type { AgentResult, TestRunnerOptions, ValidationCheck } from "../utils.ts";
import { defineFixture } from "../utils.ts";
import { defineFixture, getStructuredOutput } from "../utils.ts";
/**
* smoke test - validates agent can connect to API and call MCP tools.
* verifies select_mode tool is called with correct params.
* verifies set_output tool is called with correct value.
*/
const fixture = defineFixture(
{
prompt: `Call the select_mode tool with modeName "Build" and confirm you received the mode's prompt instructions.
Then say "SMOKE TEST PASSED".`,
prompt: `Call set_output with "SMOKE TEST PASSED".`,
effort: "mini",
},
{ localOnly: true }
);
function validator(result: AgentResult): ValidationCheck[] {
// verify MCP tool was called with correct params:
// → select_mode({"modeName":"Build"}) or → mcp__gh_pullfrog__select_mode({"modeName":"Build"})
const toolCallValid = /→.*select_mode\s*\([^)]*"modeName"\s*:\s*"Build"/i.test(result.output);
// verify agent confirmed success
const confirmationFound = /SMOKE TEST PASSED/i.test(result.output);
const output = getStructuredOutput(result);
const setOutputCalled = output !== null;
const correctValue = setOutputCalled && /SMOKE TEST PASSED/i.test(output);
return [
{ name: "tool_call", passed: toolCallValid },
{ name: "confirm", passed: confirmationFound },
{ name: "set_output", passed: setOutputCalled },
{ name: "correct_value", passed: correctValue },
];
}