ShockVPN/shockbot
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add #timeout, macro errors, refactor tests (#191)

Browse Source
This commit is contained in:
David Blass
2026-01-28 21:06:57 +00:00
committed by pullfrog[bot]
parent f77fecc2a0
commit 943409c417
18 changed files with 472 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/agnostic/timeout.ts
+33
View File
@@ -0,0 +1,33 @@
import type { AgentResult, TestRunnerOptions, ValidationCheck } from "../utils.ts";
import { defineFixture } from "../utils.ts";
/**
* timeout test - validates timeout enforcement works correctly.
* sets a very short timeout (5s) and gives the agent a task that takes longer.
* the run should fail with a timeout error.
*/
const fixture = defineFixture(
{
prompt: `Call the select_mode tool with modeName "Build", then analyze the mode instructions.
Then call select_mode with modeName "Review" and compare the two modes in detail.
Finally say "TIMEOUT TEST COMPLETED".`,
timeout: "5s",
effort: "mini",
},
{ localOnly: true }
);
function validator(result: AgentResult): ValidationCheck[] {
// run should have failed due to timeout
const timedOut = !result.success && /timed out/i.test(result.output);
return [{ name: "timeout_triggered", passed: timedOut }];
}
export const test: TestRunnerOptions = {
name: "timeout tests",
fixture,
validator,
// agnostic tests expect failures, so override the default success check
expectFailure: true,
};
test/nobash.ts → test/crossagent/nobash.ts
+4 -4
View File
@@ -1,5 +1,5 @@
import type { AgentResult, ValidationCheck } from "./utils.ts";
import { defineFixture, generateAgentUuids, getAgentOutput, runTests } from "./utils.ts";
import type { AgentResult, TestRunnerOptions, ValidationCheck } from "../utils.ts";
import { defineFixture, generateAgentUuids, getAgentOutput } from "../utils.ts";
/**
* nobash test - validates agents respect bash=disabled setting.
@@ -32,9 +32,9 @@ function validator(result: AgentResult): ValidationCheck[] {
return [{ name: "no_bash", passed: bashNotExecuted }];
}
runTests({
export const test: TestRunnerOptions = {
name: "nobash tests",
fixture,
validator,
agentEnv,
});
};
test/restricted.ts → test/crossagent/restricted.ts
+4 -4
View File
@@ -1,5 +1,5 @@
import type { AgentResult, ValidationCheck } from "./utils.ts";
import { defineFixture, generateAgentUuids, getAgentOutput, runTests } from "./utils.ts";
import type { AgentResult, TestRunnerOptions, ValidationCheck } from "../utils.ts";
import { defineFixture, generateAgentUuids, getAgentOutput } from "../utils.ts";
/**
* restricted test - validates bash=restricted environment filtering.
@@ -48,9 +48,9 @@ function validator(result: AgentResult): ValidationCheck[] {
];
}
runTests({
export const test: TestRunnerOptions = {
name: "restricted tests",
fixture,
validator,
agentEnv,
});
};
test/smoke.ts → test/crossagent/smoke.ts
+4 -4
View File
@@ -1,5 +1,5 @@
import type { AgentResult, ValidationCheck } from "./utils.ts";
import { defineFixture, runTests } from "./utils.ts";
import type { AgentResult, TestRunnerOptions, ValidationCheck } from "../utils.ts";
import { defineFixture } from "../utils.ts";
/**
* smoke test - validates agent can connect to API and call MCP tools.
@@ -29,8 +29,8 @@ function validator(result: AgentResult): ValidationCheck[] {
];
}
runTests({
export const test: TestRunnerOptions = {
name: "smoke tests",
fixture,
validator,
});
};
test/run.ts
+184
View File
@@ -0,0 +1,184 @@
import { existsSync, readdirSync } from "node:fs";
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
import { config } from "dotenv";
import {
agents,
printResults,
printSingleValidation,
runAgentStreaming,
runAllAgentsStreaming,
type TestRunnerOptions,
type ValidateResultOptions,
type ValidationResult,
validateResult,
} from "./utils.ts";
/**
* unified test runner for all agent tests.
*
* usage: node test/run.ts <test> [agent]
*
* tests are organized into two directories:
* - crossagent/ - tests that run across all agents (smoke, nobash, restricted)
* - agnostic/ - tests that only need to run once with any agent (timeout)
*
* the runner automatically detects which type of test based on directory.
*
* examples:
* node test/run.ts smoke claude # run smoke test for claude only
* node test/run.ts smoke # run smoke test for all agents
* node test/run.ts timeout claude # run timeout test with claude
* node test/run.ts timeout # run timeout test with default agent
*/
const __dirname = dirname(fileURLToPath(import.meta.url));
export const actionDir = join(__dirname, "..");
// load .env files
config({ path: join(actionDir, ".env") });
config({ path: join(actionDir, "..", ".env") });
type TestType = "crossagent" | "agnostic";
type TestInfo = {
type: TestType;
config: TestRunnerOptions;
};
async function loadTest(testName: string): Promise<TestInfo | null> {
// check crossagent directory first
const crossagentPath = join(__dirname, "crossagent", `${testName}.ts`);
if (existsSync(crossagentPath)) {
const module = await import(crossagentPath);
return { type: "crossagent", config: module.test };
}
// check agnostic directory
const agnosticPath = join(__dirname, "agnostic", `${testName}.ts`);
if (existsSync(agnosticPath)) {
const module = await import(agnosticPath);
return { type: "agnostic", config: module.test };
}
return null;
}
function listAvailableTests(): string[] {
const tests: string[] = [];
// list crossagent tests
const crossagentDir = join(__dirname, "crossagent");
if (existsSync(crossagentDir)) {
for (const file of readdirSync(crossagentDir)) {
if (file.endsWith(".ts")) {
tests.push(file.replace(".ts", ""));
}
}
}
// list agnostic tests
const agnosticDir = join(__dirname, "agnostic");
if (existsSync(agnosticDir)) {
for (const file of readdirSync(agnosticDir)) {
if (file.endsWith(".ts")) {
tests.push(file.replace(".ts", ""));
}
}
}
return tests;
}
async function runAgnosticTest(
config: TestRunnerOptions,
agent: string
): Promise<ValidationResult> {
const env = { ...config.env, ...config.agentEnv?.get(agent) };
const result = await runAgentStreaming(agent, { fixture: config.fixture, env });
const checks = config.validator(result);
const allPassed = checks.every((c) => c.passed);
// for tests with expectFailure: passed = agent failed AND all validation checks pass
// for normal tests: passed = agent succeeded AND all validation checks pass
const passed = config.expectFailure ? !result.success && allPassed : result.success && allPassed;
return {
agent: result.agent,
passed,
checks,
output: result.output,
};
}
async function main(): Promise<void> {
const testName = process.argv[2];
const agentArg = process.argv[3];
if (!testName) {
const available = listAvailableTests();
console.error(`usage: node test/run.ts <test> [agent]`);
console.error(`available tests: ${available.join(", ")}`);
process.exit(1);
}
const testInfo = await loadTest(testName);
if (!testInfo) {
const available = listAvailableTests();
console.error(`unknown test: ${testName}`);
console.error(`available tests: ${available.join(", ")}`);
process.exit(1);
}
if (agentArg && !agents.includes(agentArg as (typeof agents)[number])) {
console.error(`unknown agent: ${agentArg}`);
console.error(`available agents: ${agents.join(", ")}`);
process.exit(1);
}
const config = testInfo.config;
if (testInfo.type === "crossagent") {
// crossagent tests: run for specified agent or all agents
const validateOptions: ValidateResultOptions = { expectFailure: config.expectFailure };
if (agentArg) {
console.log(`running ${config.name} for: ${agentArg}\n`);
const env = { ...config.env, ...config.agentEnv?.get(agentArg) };
const result = await runAgentStreaming(agentArg, { fixture: config.fixture, env });
const validation = validateResult(result, config.validator, validateOptions);
console.log();
printSingleValidation(validation);
printResults([validation]);
process.exit(validation.passed ? 0 : 1);
} else {
// run all agents
console.log(`running ${config.name} for all agents...\n`);
const results = await runAllAgentsStreaming({
fixture: config.fixture,
env: config.env,
agentEnv: config.agentEnv,
});
const validations = results.map((r) => validateResult(r, config.validator, validateOptions));
console.log();
for (const v of validations) {
printSingleValidation(v);
}
printResults(validations);
const allPassed = validations.every((v) => v.passed);
process.exit(allPassed ? 0 : 1);
}
} else {
// agnostic tests: run once with specified agent or default
const agent = agentArg ?? agents[0];
console.log(`running ${config.name} for: ${agent}\n`);
const validation = await runAgnosticTest(config, agent);
console.log();
printSingleValidation(validation);
printResults([validation]);
process.exit(validation.passed ? 0 : 1);
}
}
main();
test/utils.ts
+37 -48
View File
@@ -2,17 +2,12 @@ import { spawn } from "node:child_process";
import { randomUUID } from "node:crypto";
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
import { config } from "dotenv";
import { agentsManifest } from "../external.ts";
import type { Inputs } from "../main.ts";
const __dirname = dirname(fileURLToPath(import.meta.url));
export const actionDir = join(__dirname, "..");
// load .env files
config({ path: join(actionDir, ".env") });
config({ path: join(actionDir, "..", ".env") });
const LOCAL_TEST_WARNING = "This is a local test - do not post any comments to GitHub.";
export type FixtureOptions = {
@@ -116,18 +111,25 @@ export interface RunOptions {
env?: Record<string, string> | undefined;
}
const DEFAULT_TEST_TIMEOUT = "10m";
// run agent and stream output with prefix labels
export async function runAgentStreaming(agent: string, options: RunOptions): Promise<AgentResult> {
return new Promise((resolve) => {
const chunks: Buffer[] = [];
const prefix = getAgentPrefix(agent);
const child = spawn("node", ["play.ts"], {
// apply default timeout if not specified in fixture
const fixture: Inputs = {
...options.fixture,
timeout: options.fixture.timeout ?? DEFAULT_TEST_TIMEOUT,
};
const child = spawn("node", ["play.ts", "--raw", JSON.stringify(fixture)], {
cwd: actionDir,
env: {
...process.env,
AGENT_OVERRIDE: agent,
PLAY_FIXTURE: JSON.stringify(options.fixture),
...options.env,
},
stdio: "pipe",
@@ -174,12 +176,17 @@ export async function runAgent(agent: string, options: RunOptions): Promise<Agen
return new Promise((resolve) => {
const chunks: Buffer[] = [];
const child = spawn("node", ["play.ts"], {
// apply default timeout if not specified in fixture
const fixture: Inputs = {
...options.fixture,
timeout: options.fixture.timeout ?? DEFAULT_TEST_TIMEOUT,
};
const child = spawn("node", ["play.ts", "--raw", JSON.stringify(fixture)], {
cwd: actionDir,
env: {
...process.env,
AGENT_OVERRIDE: agent,
PLAY_FIXTURE: JSON.stringify(options.fixture),
...options.env,
},
stdio: "pipe",
@@ -198,13 +205,29 @@ export async function runAgent(agent: string, options: RunOptions): Promise<Agen
});
}
export function validateResult(result: AgentResult, validator: ValidatorFn): ValidationResult {
export type ValidateResultOptions = {
// if true, test passes when validation checks pass regardless of agent success
// (used for tests like timeout that expect the agent run to fail)
expectFailure?: boolean | undefined;
};
export function validateResult(
result: AgentResult,
validator: ValidatorFn,
options?: ValidateResultOptions
): ValidationResult {
const checks = validator(result);
const allPassed = checks.every((c) => c.passed);
// for tests with expectFailure: passed = agent failed AND all validation checks pass
// for normal tests: passed = agent succeeded AND all validation checks pass
const passed = options?.expectFailure
? !result.success && allPassed
: result.success && allPassed;
return {
agent: result.agent,
passed: result.success && allPassed,
passed,
checks,
output: result.output,
};
@@ -234,43 +257,9 @@ export interface TestRunnerOptions {
env?: Record<string, string>;
// per-agent env vars (for unique markers)
agentEnv?: Map<string, Record<string, string>>;
}
export async function runTests(options: TestRunnerOptions): Promise<void> {
const agentArg = process.argv[2];
if (agentArg) {
// single agent mode
if (!agents.includes(agentArg as (typeof agents)[number])) {
console.error(`unknown agent: ${agentArg}`);
console.error(`available agents: ${agents.join(", ")}`);
process.exit(1);
}
console.log(`running ${options.name} for: ${agentArg}\n`);
const env = { ...options.env, ...options.agentEnv?.get(agentArg) };
const result = await runAgentStreaming(agentArg, { fixture: options.fixture, env });
const validation = validateResult(result, options.validator);
console.log();
printSingleValidation(validation);
process.exit(validation.passed ? 0 : 1);
}
// parallel mode with streaming
console.log(`running ${options.name} for: ${agents.join(", ")}\n`);
const results = await runAllAgentsStreaming({
fixture: options.fixture,
env: options.env,
agentEnv: options.agentEnv,
});
console.log();
const validations = results.map((r) => validateResult(r, options.validator));
printResults(validations);
const failed = validations.filter((v) => !v.passed);
process.exit(failed.length > 0 ? 1 : 0);
// if true, test passes when agent fails AND validation checks pass
// (used for tests like timeout that expect the agent run to fail)
expectFailure?: boolean;
}
export function printSingleValidation(validation: ValidationResult): void {