improve agent CI matrix, token permissions, and waitlist follower backfill (#313)
* add workflows permission to git token and waitlist improvements - add `workflows` to `InstallationTokenPermissions` type in both action and API token routes - include `workflows: write` in the git token so agents can push workflow file changes - add `githubFollowers` field to WaitlistSignup schema with migration - add script to populate waitlist followers from GitHub API - add frog-green-square-border logo asset Co-authored-by: Cursor <cursoragent@cursor.com> * improve CI, agent logging, token permissions, and delegation guardrails - add format check and build step to root CI job - standardize agent model/effort log lines across all agents - fix GitHub App permissions types to match OpenAPI schema (workflows is write-only) - improve delegation error message to prevent subagent recursion - demote noisy OpenCode stderr to debug level - add subagent delegation rules to resolved instructions Co-authored-by: Cursor <cursoragent@cursor.com> * fix graphql partial error handling, update delegation message, add workflow_run fixtures Co-authored-by: Cursor <cursoragent@cursor.com> * remove module-level env var throws that break CI build Co-authored-by: Cursor <cursoragent@cursor.com> * fix logging bug and type hole from PR review - use batch-local notFound counter so per-batch log doesn't undercount - add workflows to WorkflowTokenPermissions so wire type matches what action sends Co-authored-by: Cursor <cursoragent@cursor.com> * lazy-init appOctokit to fix next build without env vars Co-authored-by: Cursor <cursoragent@cursor.com> * drop pnpm build from CI test workflow Co-authored-by: Cursor <cursoragent@cursor.com> * fix delegate-effort test regex to match actual log format, disable fail-fast for agnostic tests the test was matching `running \w+ with effort=auto` but the actual log line from shared.ts is `» effort: auto`. also temporarily set fail-fast: false on action-agnostic so all failures surface at once. Co-authored-by: Cursor <cursoragent@cursor.com> * disable fail-fast in action workflow too, relax ci.test.ts to match both workflow files now use fail-fast: false for agnostic tests so all matrix jobs run to completion. the ci consistency test now checks that the two workflows agree rather than requiring true. Co-authored-by: Cursor <cursoragent@cursor.com> * restore fail-fast: true now that all agnostic tests pass Co-authored-by: Cursor <cursoragent@cursor.com> * skip agent tests in CI when agent harness file didn't change adds action/test/changed-agents.sh which reads the PR diff (via dorny/paths-filter) and outputs only agents whose harness file was modified. the action-agents matrix now uses this dynamic list instead of a hardcoded array, so e.g. a PR touching only cursor.ts runs 6 jobs instead of 30. Co-authored-by: Cursor <cursoragent@cursor.com> * update ci.test.ts to validate dynamic agent matrix the test now checks that the matrix references the changes job output and that changed-agents.sh correctly discovers all agents. Co-authored-by: Cursor <cursoragent@cursor.com> * parallelize action jobs and use claude canary fallback for shared changes runs action-agents in parallel with action-agnostic after root/changes, and updates changed-agents logic so shared or non-harness action runtime changes run only claude while harness-specific edits run only those changed agents. Co-authored-by: Cursor <cursoragent@cursor.com> --------- Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
committed by
pullfrog[bot]
parent
37dcea86b9
commit
b6658ddbc1
@@ -30,11 +30,11 @@ function validator(result: AgentResult): ValidationCheck[] {
|
||||
const setOutputCalled = output !== null;
|
||||
const correctValue = setOutputCalled && /EFFORT_TEST_PASSED/i.test(output);
|
||||
|
||||
// the orchestrator runs at auto (effort=auto in its log line).
|
||||
// the delegate tool should spawn the subagent at mini (effort=mini in its log line).
|
||||
// if effort override works, we should see BOTH effort=auto AND effort=mini in the output.
|
||||
const orchestratorEffort = /running \w+ with effort=auto/i.test(agentOutput);
|
||||
const subagentEffort = /running \w+ with effort=mini/i.test(agentOutput);
|
||||
// the orchestrator runs at auto (» effort: auto in its log line).
|
||||
// the delegate tool should spawn the subagent at mini (» effort: mini in its log line).
|
||||
// if effort override works, we should see BOTH effort values in the output.
|
||||
const orchestratorEffort = /» effort:\s+auto/i.test(agentOutput);
|
||||
const subagentEffort = /» effort:\s+mini/i.test(agentOutput);
|
||||
|
||||
return [
|
||||
{ name: "set_output", passed: setOutputCalled },
|
||||
|
||||
Executable
+45
@@ -0,0 +1,45 @@
|
||||
#!/usr/bin/env bash
|
||||
# determines which agents need testing based on changed files.
|
||||
# reads changed file paths from stdin (JSON array or newline-delimited).
|
||||
# outputs a JSON array of agent names to stdout.
|
||||
#
|
||||
# only agents whose harness file changed are included.
|
||||
# shared.ts/index.ts and other non-harness action changes fall back to claude as a canary.
|
||||
set -euo pipefail
|
||||
|
||||
# read stdin - auto-detect JSON array vs newline-delimited
|
||||
input=$(cat)
|
||||
if echo "$input" | jq -e 'type == "array"' > /dev/null 2>&1; then
|
||||
files=$(echo "$input" | jq -r '.[]')
|
||||
else
|
||||
files="$input"
|
||||
fi
|
||||
|
||||
# find which agent harness files changed
|
||||
changed_agents=()
|
||||
has_non_agent_change=false
|
||||
|
||||
while IFS= read -r file; do
|
||||
[[ -z "$file" ]] && continue
|
||||
case "$file" in
|
||||
action/agents/shared.ts|action/agents/index.ts)
|
||||
has_non_agent_change=true
|
||||
;;
|
||||
action/agents/*.ts)
|
||||
changed_agents+=("$(basename "$file" .ts)")
|
||||
;;
|
||||
action/*)
|
||||
has_non_agent_change=true
|
||||
;;
|
||||
esac
|
||||
done <<< "$files"
|
||||
|
||||
# output agents based on change type.
|
||||
# non-agent action changes run claude as a canary.
|
||||
if [[ ${#changed_agents[@]} -gt 0 ]]; then
|
||||
printf '%s\n' "${changed_agents[@]}" | sort -u | jq -R . | jq -sc .
|
||||
elif $has_non_agent_change; then
|
||||
echo '["claude"]'
|
||||
else
|
||||
echo '[]'
|
||||
fi
|
||||
+21
-2
@@ -1,3 +1,4 @@
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { readdirSync, readFileSync } from "node:fs";
|
||||
import { dirname, join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
@@ -57,6 +58,7 @@ const expectedAgents = Object.keys(agentsManifest).sort();
|
||||
const crossagentTests = getTestNamesFromDir("crossagent");
|
||||
const agnosticTests = getTestNamesFromDir("agnostic");
|
||||
const adhocTests = getTestNamesFromDir("adhoc");
|
||||
const dynamicAgentsExpression = "$" + "{{ fromJSON(needs.changes.outputs.agents) }}";
|
||||
|
||||
// all API key names from all agents + GITHUB_TOKEN + model overrides
|
||||
const expectedAgentEnvVars = [
|
||||
@@ -84,8 +86,25 @@ describe("ci workflow consistency", () => {
|
||||
const rootJob = rootWorkflow.jobs["action-agents"];
|
||||
const actionJob = actionWorkflow.jobs.agents;
|
||||
|
||||
it("root agent matrix matches agentsManifest", () => {
|
||||
expect([...rootJob.strategy!.matrix.agent].sort()).toEqual(expectedAgents);
|
||||
it("root agent matrix uses dynamic output from changes job", () => {
|
||||
expect(rootJob.strategy!.matrix.agent).toBe(dynamicAgentsExpression);
|
||||
});
|
||||
|
||||
it("changed-agents.sh falls back to claude when shared agent code changed", () => {
|
||||
const input = JSON.stringify(["action/agents/shared.ts"]);
|
||||
const output = execFileSync("bash", [join(__dirname, "changed-agents.sh")], {
|
||||
input,
|
||||
encoding: "utf-8",
|
||||
});
|
||||
expect(JSON.parse(output)).toEqual(["claude"]);
|
||||
});
|
||||
|
||||
it("changed-agents.sh falls back to claude for non-agent action changes", () => {
|
||||
const output = execFileSync("bash", [join(__dirname, "changed-agents.sh")], {
|
||||
input: JSON.stringify(["action/mcp/delegate.ts"]),
|
||||
encoding: "utf-8",
|
||||
});
|
||||
expect(JSON.parse(output)).toEqual(["claude"]);
|
||||
});
|
||||
|
||||
it("action agent matrix matches agentsManifest", () => {
|
||||
|
||||
Reference in New Issue
Block a user