improve runtest, optimize CI batching (#210)
This commit is contained in:
committed by
pullfrog[bot]
parent
2b3bd97b86
commit
18ba8e5fd0
@@ -1,31 +1,27 @@
|
||||
import type { AgentResult, TestRunnerOptions, ValidationCheck } from "../utils.ts";
|
||||
import { defineFixture } from "../utils.ts";
|
||||
import { defineFixture, getStructuredOutput } from "../utils.ts";
|
||||
|
||||
/**
|
||||
* smoke test - validates agent can connect to API and call MCP tools.
|
||||
* verifies select_mode tool is called with correct params.
|
||||
* verifies set_output tool is called with correct value.
|
||||
*/
|
||||
|
||||
const fixture = defineFixture(
|
||||
{
|
||||
prompt: `Call the select_mode tool with modeName "Build" and confirm you received the mode's prompt instructions.
|
||||
|
||||
Then say "SMOKE TEST PASSED".`,
|
||||
prompt: `Call set_output with "SMOKE TEST PASSED".`,
|
||||
effort: "mini",
|
||||
},
|
||||
{ localOnly: true }
|
||||
);
|
||||
|
||||
function validator(result: AgentResult): ValidationCheck[] {
|
||||
// verify MCP tool was called with correct params:
|
||||
// → select_mode({"modeName":"Build"}) or → mcp__gh_pullfrog__select_mode({"modeName":"Build"})
|
||||
const toolCallValid = /→.*select_mode\s*\([^)]*"modeName"\s*:\s*"Build"/i.test(result.output);
|
||||
// verify agent confirmed success
|
||||
const confirmationFound = /SMOKE TEST PASSED/i.test(result.output);
|
||||
const output = getStructuredOutput(result);
|
||||
const setOutputCalled = output !== null;
|
||||
const correctValue = setOutputCalled && /SMOKE TEST PASSED/i.test(output);
|
||||
|
||||
return [
|
||||
{ name: "tool_call", passed: toolCallValid },
|
||||
{ name: "confirm", passed: confirmationFound },
|
||||
{ name: "set_output", passed: setOutputCalled },
|
||||
{ name: "correct_value", passed: correctValue },
|
||||
];
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user