From d6de1c369a97677616e367306d2a60f256e45791 Mon Sep 17 00:00:00 2001 From: Colin McDonnell Date: Fri, 8 May 2026 22:45:26 +0000 Subject: [PATCH] learnings: edit-in-place tmpfile (drop `update_learnings` tool) (#635) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * learnings: edit-in-place tmpfile (drop update_learnings tool) learnings now follow the PR-summary file pattern: server seeds `pullfrog-learnings.md` from `Repo.learnings` at startup, agent reads it as part of context, may edit in place during the post-run reflection turn, server reads back at end-of-run and PATCHes if changed. motivation: `update_learnings` required the agent to pass the FULL merged list as a string parameter — an output-token tax that grew linearly with the learnings size, and a constant prompt-context expansion since the contents were also inlined into the LEARNINGS section. for repos with mature learnings the prompt was getting visibly noisy in CI logs. key changes: - new `action/utils/learnings.ts` (seed/read helpers + 10k cap) - `main.ts`: always seed; `persistLearnings` mirrors `persistSummary` (success path, error path, exit-signal handler, idempotent guard, byte-trim equality skip); forwards `model` for `LearningsRevision.model` - `LEARNINGS` prompt section now contains only the file path + a one-line "read it" instruction (no contents inlined) - `update_learnings` MCP tool deleted; `action/mcp/learnings.ts` removed - reflection turn (`buildLearningsReflectionPrompt`) reframed around file editing with explicit prune-stale + leave-alone-if-nothing-new framing - `learningsStep` removed from every mode checklist — surface lives only in the LEARNINGS prompt section + the reflection turn now * learnings: harden seed step + refresh stale docs (review feedback) Three findings from PR review, all implemented: 1. wrap learnings seed in best-effort try/catch (action/main.ts) — the always-on seed block ran unconditionally and an unwrapped `seedLearningsFile` (mkdir + writeFile) failure (ENOSPC, EACCES, hostile sandbox) would unwind into the outer main() catch and flip an otherwise-successful run to "❌ Pullfrog failed" before the agent even started. asymmetric with `persistLearnings`'s own best-effort contract. wrap and log on failure; downstream consumers (`persistLearnings`, agent harnesses, `resolveInstructions`) already handle `learningsFilePath: undefined` cleanly. 2. refresh wiki/main.md — `resolveInstructions` parameter renamed from `learnings` to `learningsFilePath` in this PR; the data-flow diagram and the resolver dependency table both still showed the pre-refactor signature. 3. drop deleted `learnings.ts` from ROADMAP.md + RESEARCH.md "missing MCP tool tests" bullets — `action/mcp/learnings.ts` was removed in this PR; the bullets are otherwise still accurate. --- agents/claude.ts | 4 +- agents/opencode.ts | 4 +- agents/postRun.test.ts | 16 +++--- agents/postRun.ts | 35 +++++++------ agents/shared.ts | 7 +++ main.ts | 110 +++++++++++++++++++++++++++++++++++++++- mcp/learnings.ts | 41 --------------- mcp/server.ts | 21 ++++++-- modes.ts | 22 ++------ utils/instructions.ts | 23 ++++++--- utils/learnings.test.ts | 70 +++++++++++++++++++++++++ utils/learnings.ts | 64 +++++++++++++++++++++++ 12 files changed, 320 insertions(+), 97 deletions(-) delete mode 100644 mcp/learnings.ts create mode 100644 utils/learnings.test.ts create mode 100644 utils/learnings.ts diff --git a/agents/claude.ts b/agents/claude.ts index 45d3a0e..b8f0679 100644 --- a/agents/claude.ts +++ b/agents/claude.ts @@ -727,7 +727,9 @@ export const claude = agent({ stopScript: ctx.stopScript, summaryFilePath: ctx.summaryFilePath, summarySeed: ctx.summarySeed, - reflectionPrompt: buildLearningsReflectionPrompt("claude"), + reflectionPrompt: ctx.learningsFilePath + ? buildLearningsReflectionPrompt(ctx.learningsFilePath) + : undefined, canResume: (r) => Boolean(r.sessionId), resume: async (c) => { const sessionId = c.previousResult.sessionId; diff --git a/agents/opencode.ts b/agents/opencode.ts index 9779414..4ffb378 100644 --- a/agents/opencode.ts +++ b/agents/opencode.ts @@ -1005,7 +1005,9 @@ export const opencode = agent({ stopScript: ctx.stopScript, summaryFilePath: ctx.summaryFilePath, summarySeed: ctx.summarySeed, - reflectionPrompt: buildLearningsReflectionPrompt("opencode"), + reflectionPrompt: ctx.learningsFilePath + ? buildLearningsReflectionPrompt(ctx.learningsFilePath) + : undefined, resume: async (c) => runOpenCode({ ...runParams, diff --git a/agents/postRun.test.ts b/agents/postRun.test.ts index f81df09..8e9c813 100644 --- a/agents/postRun.test.ts +++ b/agents/postRun.test.ts @@ -42,10 +42,10 @@ describe("runPostRunRetryLoop — reflection turn", () => { }); it("does not flip a successful run to failed when reflection returns success:false", async () => { - // the reflection turn is a best-effort nudge (update_learnings). if it - // fails — e.g. the model API errors mid-turn — the underlying task has - // already completed and been gated cleanly, so the run as a whole must - // still be reported as successful. + // the reflection turn is a best-effort nudge (edit the learnings + // tmpfile). if it fails — e.g. the model API errors mid-turn — the + // underlying task has already completed and been gated cleanly, so the + // run as a whole must still be reported as successful. const initial = successResult({ output: "task done" }); const resume = vi .fn<(ctx: { prompt: string; previousResult: AgentResult }) => Promise>() @@ -56,7 +56,7 @@ describe("runPostRunRetryLoop — reflection turn", () => { initialUsage: undefined, stopScript: null, resume, - reflectionPrompt: "REFLECTION: call update_learnings if anything is worth saving", + reflectionPrompt: "REFLECTION: edit learnings file if anything is worth saving", }); expect(result.success).toBe(true); @@ -110,7 +110,7 @@ describe("runPostRunRetryLoop — reflection turn", () => { initialUsage: undefined, stopScript: null, resume, - reflectionPrompt: "REFLECTION: consider update_learnings", + reflectionPrompt: "REFLECTION: consider editing learnings", }); expect(result.success).toBe(true); @@ -135,7 +135,7 @@ describe("runPostRunRetryLoop — reflection turn", () => { initialUsage: undefined, stopScript: null, resume, - reflectionPrompt: "REFLECTION: consider update_learnings", + reflectionPrompt: "REFLECTION: consider editing learnings", }); expect(result.success).toBe(true); @@ -186,7 +186,7 @@ describe("runPostRunRetryLoop — reflection turn", () => { initialUsage: undefined, stopScript: null, resume, - reflectionPrompt: "REFLECTION: consider update_learnings", + reflectionPrompt: "REFLECTION: consider editing learnings", }); expect(result.success).toBe(true); diff --git a/agents/postRun.ts b/agents/postRun.ts index 9b9e504..57f0dc1 100644 --- a/agents/postRun.ts +++ b/agents/postRun.ts @@ -1,5 +1,4 @@ import { readFile } from "node:fs/promises"; -import { type AgentId, formatMcpToolRef } from "../external.ts"; import { LIFECYCLE_HOOK_TIMEOUT_MS } from "../lifecycle.ts"; import { log } from "../utils/cli.ts"; import { @@ -157,26 +156,30 @@ export function buildPostRunPrompt(issues: PostRunIssues): string { } /** - * prompt for a dedicated post-run reflection turn nudging the agent to call - * `update_learnings` if it discovered anything worth persisting. + * prompt for a dedicated post-run reflection turn nudging the agent to edit + * the rolling learnings file if it discovered anything worth persisting. * - * this exists because the learnings step baked into mode checklists is - * frequently ignored — the agent stays focused on the task and the meta-ask - * falls through. delivering it as its own resume turn, with nothing competing - * for attention, raises the fire rate substantially. + * this exists because passive "if you learned something, write it down" + * instructions baked into mode checklists are frequently ignored — the agent + * stays focused on the task and the meta-ask falls through. delivering it + * as its own resume turn, with nothing competing for attention, raises the + * fire rate substantially. + * + * the file is the single source of truth — there is no separate MCP tool + * call. the server reads the file at end-of-run and persists any edits to + * `Repo.learnings`. */ -export function buildLearningsReflectionPrompt(agentId: AgentId): string { - const t = (name: string) => formatMcpToolRef(agentId, name); +export function buildLearningsReflectionPrompt(filePath: string): string { return [ - `REFLECTION — before you finish, think back over this task: did you discover anything about this repo's setup, test commands, conventions, or patterns that you are confident is correct and would reliably help future runs?`, + `REFLECTION — before you finish, think back over this task: did you discover anything about this repo's setup, test commands, conventions, or patterns that is high-confidence and would reliably help future runs?`, "", - `if so, call \`${t("update_learnings")}\` to persist it.`, + `the rolling learnings file is at \`${filePath}\`. read it first if you haven't already, then edit it in place using your native file tools. the server reads this file at end-of-run and persists any changes — there is no tool to call.`, "", - `rules:`, - `- only call \`${t("update_learnings")}\` when the finding is high-confidence and broadly useful. skip if unsure, speculative, or one-off.`, - `- pass the FULL merged list: existing learnings from the original prompt + your new discoveries. one fact per bullet, lines starting with \`- \`.`, - `- deduplicate, and drop bullets that are clearly wrong or no longer relevant to the current codebase.`, - `- if you already called \`${t("update_learnings")}\` earlier in this run, or nothing new is worth capturing, just reply "done" and stop — do not edit the repo for this reflection.`, + `keep the file healthy:`, + `- only add bullets when the finding is high-confidence AND broadly useful. skip speculative, one-off, or "maybe" findings.`, + `- prune bullets that are clearly wrong, no longer relevant, or low-signal (rarely useful). a focused, accurate file beats a long stale one.`, + `- format: flat bullet list, one fact per line starting with \`- \`. deduplicate against existing entries — if a bullet covers the same fact, update it in place instead of adding a duplicate.`, + `- leave the file alone if you have nothing substantively new to add and the existing entries still look healthy. silence is a valid outcome — just reply "done" and stop.`, ].join("\n"); } diff --git a/agents/shared.ts b/agents/shared.ts index a8169a0..66be562 100644 --- a/agents/shared.ts +++ b/agents/shared.ts @@ -133,6 +133,13 @@ export interface AgentRunContext { * track of multi-step instructions. */ summarySeed?: string | undefined; + /** + * absolute path to the rolling repo-level learnings tmpfile. seeded for + * every run from `Repo.learnings`. used by the post-run reflection turn + * so the prompt can point the agent at a concrete path to edit; the + * file's content is read back and persisted by main.ts after the run. + */ + learningsFilePath?: string | undefined; /** * called synchronously when the agent subprocess is killed for inner * activity timeout. lets main.ts tear down shared resources (MCP HTTP diff --git a/main.ts b/main.ts index 782747b..3391b93 100644 --- a/main.ts +++ b/main.ts @@ -31,6 +31,7 @@ import { resolveGit, setGitAuthServer } from "./utils/gitAuth.ts"; import { startGitAuthServer } from "./utils/gitAuthServer.ts"; import { createOctokit, writeGitHubUsageSummaryToFile } from "./utils/github.ts"; import { resolveInstructions } from "./utils/instructions.ts"; +import { readLearningsFile, seedLearningsFile } from "./utils/learnings.ts"; import { executeLifecycleHook } from "./utils/lifecycle.ts"; import { normalizeEnv } from "./utils/normalizeEnv.ts"; import { aggregateUsage, patchWorkflowRunFields } from "./utils/patchWorkflowRunFields.ts"; @@ -395,6 +396,58 @@ async function fetchPreviousSnapshot(ctx: ToolContext, prNumber: number): Promis * (on incremental runs) or serialize the placeholder scaffold (on first * runs), neither of which is useful. */ +/** + * Read the agent-edited repo-level learnings tmpfile and PATCH it to + * `Repo.learnings`. + * + * Best-effort: any failure is logged and does not affect the run's success + * status. Skips the PATCH when the file is byte-trim-identical to its seed — + * the agent didn't touch it, so writing the same content back would just + * burn a `LearningsRevision` row and an API round-trip. + * + * `model` is forwarded so `LearningsRevision.model` keeps populating; it + * powers the per-revision attribution badge in the UI history view. + */ +async function persistLearnings(ctx: ToolContext): Promise { + const filePath = ctx.toolState.learningsFilePath; + if (!filePath) return; + if (ctx.toolState.learningsPersistAttempted) return; + ctx.toolState.learningsPersistAttempted = true; + const current = await readLearningsFile(filePath); + if (current === null) { + log.debug(`learnings tmpfile missing or unreadable at ${filePath} — skipping persist`); + return; + } + const seed = ctx.toolState.learningsSeed?.trim() ?? ""; + if (current === seed) { + log.debug("learnings tmpfile unchanged from seed — skipping persist"); + return; + } + try { + const response = await apiFetch({ + path: `/api/repo/${ctx.repo.owner}/${ctx.repo.name}/learnings`, + method: "PATCH", + headers: { + authorization: `Bearer ${ctx.apiToken}`, + "content-type": "application/json", + }, + body: JSON.stringify({ + learnings: current, + model: ctx.toolState.model, + }), + signal: AbortSignal.timeout(10_000), + }); + if (!response.ok) { + const error = await response.text().catch(() => "(no body)"); + log.debug(`learnings persist failed (${response.status}): ${error}`); + return; + } + log.info("» learnings updated"); + } catch (err) { + log.debug(`learnings persist failed: ${err instanceof Error ? err.message : String(err)}`); + } +} + async function persistSummary(ctx: ToolContext): Promise { const filePath = ctx.toolState.summaryFilePath; if (!filePath) return; @@ -647,6 +700,47 @@ export async function main(): Promise { log.info(`» MCP server started at ${mcpHttpServer.url}`); timer.checkpoint("mcpServer"); + // seed the rolling repo-level learnings tmpfile for every run. the + // agent reads the file at startup (path is surfaced in the LEARNINGS + // section of the prompt) and may edit it during the post-run + // reflection turn. persistLearnings reads it back at end-of-run and + // PATCHes any changes to Repo.learnings, byte-trim equality against + // the seed gates the API call. always-seed (vs gated): learnings are + // universal — any run can produce them, and gating just hides the + // affordance. + // + // wrapped in best-effort try/catch: this block runs unconditionally, + // and an unwrapped filesystem failure (ENOSPC, EACCES, hostile sandbox) + // would unwind into the outer main() catch and flip an otherwise- + // successful run to "❌ Pullfrog failed" before the agent even starts. + // matches `persistLearnings`'s own best-effort contract — learnings + // are a peripheral artifact, not a load-bearing capability. on failure + // toolState.learningsFilePath stays unset, and downstream consumers + // (`persistLearnings`, agent harnesses, `resolveInstructions`) all + // treat undefined as "no learnings affordance this run". + try { + const learningsPath = await seedLearningsFile({ + tmpdir, + current: runContext.repoSettings.learnings, + }); + toolState.learningsFilePath = learningsPath; + try { + toolState.learningsSeed = await readFile(learningsPath, "utf8"); + } catch { + // intentionally empty — learningsSeed stays undefined, persistLearnings + // will treat seed as "" and persist any non-empty content + } + log.info( + `» learnings seeded at ${learningsPath} (existing=${runContext.repoSettings.learnings ? "yes" : "no"})` + ); + const ctxForExit = toolContext; + onExitSignal(() => persistLearnings(ctxForExit)); + } catch (err) { + log.warning( + `» learnings seed failed: ${err instanceof Error ? err.message : String(err)} — continuing without learnings file` + ); + } + // seed the rolling PR summary tmpfile when the dispatcher requested it. // gated on event being a PR — issue/workflow_dispatch runs have no // summarySnapshot to maintain. file path is exposed to the agent via @@ -698,7 +792,7 @@ export async function main(): Promise { modes, agentId, outputSchema, - learnings: runContext.repoSettings.learnings, + learningsFilePath: toolState.learningsFilePath ?? null, }); const logParts = [ instructions.eventInstructions @@ -795,6 +889,7 @@ export async function main(): Promise { stopScript: runContext.repoSettings.stopScript, summaryFilePath: toolState.summaryFilePath, summarySeed: toolState.summarySeed, + learningsFilePath: toolState.learningsFilePath, onActivityTimeout: onInnerActivityTimeout, onToolUse: (event) => { const wasTracked = recordDiffReadFromToolUse({ @@ -882,6 +977,13 @@ export async function main(): Promise { await persistSummary(toolContext); } + // same for the rolling repo-level learnings tmpfile. always seeded, so + // always read back; persistLearnings short-circuits when the file is + // unchanged from its seed. + if (toolContext) { + await persistLearnings(toolContext); + } + // clean up stranded progress comments. the comment is stale unless // report_progress wrote a final summary to it — three sub-cases all reduce // to !finalSummaryWritten: @@ -966,6 +1068,12 @@ export async function main(): Promise { await persistSummary(toolContext); } + // same rationale for learnings: a partial edit before a crash is still + // worth keeping. persistLearnings is idempotent via learningsPersistAttempted. + if (toolContext) { + await persistLearnings(toolContext); + } + return { success: false, error: errorMessage, diff --git a/mcp/learnings.ts b/mcp/learnings.ts deleted file mode 100644 index d509df4..0000000 --- a/mcp/learnings.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { type } from "arktype"; -import { apiFetch } from "../utils/apiFetch.ts"; -import type { ToolContext } from "./server.ts"; -import { execute, tool } from "./shared.ts"; - -const UpdateLearningsParams = type({ - learnings: type.string.describe( - "the FULL merged learnings as a flat bullet list. each line starts with `- `. one discrete, actionable fact per bullet. combine existing bullets from the prompt with your new discoveries. deduplicate — if an existing bullet covers the same fact, update it in place rather than adding a new one. drop bullets that are clearly wrong or no longer relevant to the current codebase. keep the list focused and concise." - ), -}); - -export function UpdateLearningsTool(ctx: ToolContext) { - return tool({ - name: "update_learnings", - description: - "persist operational learnings about this repository (setup steps, test commands, key conventions, patterns). ONLY call this when you have high confidence the information is correct and broadly useful for future runs — not for one-off findings or uncertain observations. format: flat bullet list (`- ` per line, one fact per bullet). pass the FULL merged list — combine existing learnings from the prompt with new discoveries. deduplicate, and drop bullets that are clearly wrong or no longer relevant to the current codebase.", - parameters: UpdateLearningsParams, - execute: execute(async (params) => { - const response = await apiFetch({ - path: `/api/repo/${ctx.repo.owner}/${ctx.repo.name}/learnings`, - method: "PATCH", - headers: { - authorization: `Bearer ${ctx.apiToken}`, - "content-type": "application/json", - }, - body: JSON.stringify({ - learnings: params.learnings, - model: ctx.toolState.model, - }), - signal: AbortSignal.timeout(10_000), - }); - - if (!response.ok) { - const error = await response.text(); - throw new Error(`failed to update learnings: ${error}`); - } - - return { success: true }; - }), - }); -} diff --git a/mcp/server.ts b/mcp/server.ts index 0d45d9f..a1005c1 100644 --- a/mcp/server.ts +++ b/mcp/server.ts @@ -39,7 +39,6 @@ import { GetIssueCommentsTool } from "./issueComments.ts"; import { GetIssueEventsTool } from "./issueEvents.ts"; import { IssueInfoTool } from "./issueInfo.ts"; import { AddLabelsTool } from "./labels.ts"; -import { UpdateLearningsTool } from "./learnings.ts"; import { SetOutputTool } from "./output.ts"; import { CreatePullRequestTool, UpdatePullRequestBodyTool } from "./pr.ts"; import { PullRequestInfoTool } from "./prInfo.ts"; @@ -143,6 +142,21 @@ export interface ToolState { // persisted) from redundantly re-running the DB PATCH on the // success-then-late-throw path. summaryPersistAttempted?: boolean; + // absolute path to the rolling repo-level learnings markdown file the + // agent reads at startup and may edit at end-of-run. seeded by main.ts + // for every run from `Repo.learnings` (empty file when no learnings + // exist yet); read back at end-of-run to persist any edits. + learningsFilePath?: string; + // exact bytes of the seeded learnings file at run start. compared + // against the file content at end-of-run to detect "agent never touched + // it" — in that case persistLearnings skips the DB PATCH (saving the + // identical content would be a no-op write that wastes a LearningsRevision + // row and the API round-trip). + learningsSeed?: string; + // mirror of `summaryPersistAttempted` for the learnings tmpfile — guards + // the error-path / exit-signal callers from a redundant second PATCH + // after the success path already persisted. + learningsPersistAttempted?: boolean; output?: string; usageEntries: AgentUsage[]; model?: string | undefined; @@ -189,8 +203,8 @@ export interface ToolContext { tmpdir: string; // repo-level OSS flag + account-level billing plan. together they decide // whether pullfrog is paying for marginal infra — see isInfraCovered in - // utils/runContext.ts. plan gating for things like update_learnings is - // enforced server-side via 402, so we pass plan along mostly for future + // utils/runContext.ts. plan gating for endpoints like the learnings PATCH + // is enforced server-side via 402, so we pass plan along mostly for future // use / observability. see wiki/pricing.md. oss: boolean; plan: AccountPlan; @@ -288,7 +302,6 @@ function buildOrchestratorTools(ctx: ToolContext, outputSchema?: JsonSchema): To DeleteBranchTool(ctx), CreatePullRequestTool(ctx), UpdatePullRequestBodyTool(ctx), - UpdateLearningsTool(ctx), ]; } diff --git a/modes.ts b/modes.ts index 810cbf0..b70b26c 100644 --- a/modes.ts +++ b/modes.ts @@ -65,10 +65,6 @@ Rules: - Focus on *intent*, not *what* — the diff already shows what changed - Get the file count and commit count from the checkout_pr metadata, not by counting manually`; -function learningsStep(t: (toolName: string) => string, n: number): string { - return `${n}. **learnings** (only if high confidence): if you discovered something about repo setup, test commands, conventions, or patterns that you are confident is correct and would reliably help future runs, call \`${t("update_learnings")}\` to persist it. skip this step if you are unsure or the finding is speculative/one-off. format as a flat bullet list (\`- \` per line, one fact per bullet). merge with existing learnings from the prompt — pass the FULL merged list. deduplicate, and drop bullets that are clearly wrong or no longer relevant to the current codebase.`; -} - export function computeModes(agentId: AgentId): Mode[] { const t = (toolName: string) => formatMcpToolRef(agentId, toolName); return [ @@ -125,8 +121,6 @@ export function computeModes(agentId: AgentId): Mode[] { - create a PR via \`${t("create_pull_request")}\` - call \`${t("report_progress")}\` with the PR link or the exact error if push/PR failed -${learningsStep(t, 6)} - ### Notes For simple, well-defined tasks, skip the plan phase and go straight to build.`, @@ -155,9 +149,7 @@ For simple, well-defined tasks, skip the plan phase and go straight to build.`, - confirm a clean working tree, then push via \`${t("push_branch")}\` (same push/prepush guidance as Build mode in *SYSTEM*) - reply to each comment using \`${t("reply_to_review_comment")}\` - resolve addressed threads via \`${t("resolve_review_thread")}\` - - call \`${t("report_progress")}\` with a brief summary (or the exact push error if push failed) - -${learningsStep(t, 6)}`, + - call \`${t("report_progress")}\` with a brief summary (or the exact push error if push failed)`, }, // Review and IncrementalReview use the multi-lens orchestrator pattern // (canonical source: .claude/commands/anneal.md). The orchestrator does @@ -331,9 +323,7 @@ ${PR_SUMMARY_FORMAT}`, 2. Produce a structured, actionable plan with clear milestones. -3. Call \`${t("report_progress")}\` with the plan. - -${learningsStep(t, 4)}`, +3. Call \`${t("report_progress")}\` with the plan.`, }, { name: "Fix", @@ -356,9 +346,7 @@ ${learningsStep(t, 4)}`, 5. Finalize: - confirm a clean working tree, then push via \`${t("push_branch")}\` (same push/prepush guidance as Build mode in *SYSTEM*) - - call \`${t("report_progress")}\` with the diagnosis and fix summary (or the exact push error if push failed) - -${learningsStep(t, 6)}`, + - call \`${t("report_progress")}\` with the diagnosis and fix summary (or the exact push error if push failed)`, }, { name: "ResolveConflicts", @@ -403,9 +391,7 @@ ${learningsStep(t, 6)}`, 3. Finalize: - if code changes were made, push to a pull request (new or existing) using \`${t("push_branch")}\` and \`${t("create_pull_request")}\` as needed. \`git status\` must be clean before you finish (see *SYSTEM* Git rules if push fails). - call \`${t("report_progress")}\` once with results — include exact tool errors if push or PR creation failed - - if the task involved labeling, commenting, or other GitHub operations, perform those directly - -${learningsStep(t, 4)}`, + - if the task involved labeling, commenting, or other GitHub operations, perform those directly`, }, ]; } diff --git a/utils/instructions.ts b/utils/instructions.ts index 9aa89ec..3f14d42 100644 --- a/utils/instructions.ts +++ b/utils/instructions.ts @@ -12,7 +12,10 @@ interface InstructionsContext { modes: Mode[]; agentId: AgentId; outputSchema?: Record | undefined; - learnings: string | null; + /** absolute path to the seeded learnings tmpfile, or null when the file + * couldn't be seeded for some reason. main.ts always seeds, so in + * practice this is always set; the null case keeps the type honest. */ + learningsFilePath: string | null; } interface PromptContext extends InstructionsContext { @@ -350,11 +353,17 @@ function assembleFullPrompt(ctx: { procedure: string; eventContext: string; system: string; - learnings: string | null; + learningsFilePath: string | null; runtime: string; }): string { - const learningsSection = ctx.learnings - ? `************* LEARNINGS *************\n\n${ctx.learnings}` + // the LEARNINGS section is intentionally tiny — just the file path and a + // one-line "read it" instruction. embedding the contents would re-inflate + // the prompt every run (the previous design's failure mode) and clutter + // CI logs. the agent reads the file with its native file tool; the + // post-run reflection turn (action/agents/postRun.ts) is where editing + // is encouraged, with the prune-stale framing. + const learningsSection = ctx.learningsFilePath + ? `************* LEARNINGS *************\n\nRepo-level learnings accumulated by previous agent runs live at \`${ctx.learningsFilePath}\`. Read this file early and let the entries inform your approach (test commands, conventions, gotchas, etc.). The file may be empty if no learnings have been collected yet.` : ""; const runtimeSection = `************* RUNTIME *************\n\n${ctx.runtime}`; @@ -389,8 +398,8 @@ export function resolveInstructions(ctx: InstructionsContext): ResolvedInstructi if (eventContext) tocEntries.push({ label: "EVENT CONTEXT", description: "related PR/issue data" }); tocEntries.push({ label: "SYSTEM", description: "persona, security, tools, workflow rules" }); - if (pctx.learnings) - tocEntries.push({ label: "LEARNINGS", description: "repo-specific knowledge" }); + if (pctx.learningsFilePath) + tocEntries.push({ label: "LEARNINGS", description: "repo-specific knowledge file path" }); tocEntries.push({ label: "RUNTIME", description: "environment metadata" }); const toc = buildToc(tocEntries); @@ -401,7 +410,7 @@ export function resolveInstructions(ctx: InstructionsContext): ResolvedInstructi procedure, eventContext, system, - learnings: pctx.learnings, + learningsFilePath: pctx.learningsFilePath, runtime: pctx.runtime, }); diff --git a/utils/learnings.test.ts b/utils/learnings.test.ts new file mode 100644 index 0000000..93b80a4 --- /dev/null +++ b/utils/learnings.test.ts @@ -0,0 +1,70 @@ +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { + LEARNINGS_FILE_NAME, + learningsFilePath, + readLearningsFile, + seedLearningsFile, +} from "./learnings.ts"; + +describe("learnings tmpfile round-trip", () => { + let dir: string; + + beforeEach(async () => { + dir = await mkdtemp(join(tmpdir(), "pullfrog-learnings-test-")); + }); + + afterEach(async () => { + await rm(dir, { recursive: true, force: true }); + }); + + it("seeds with existing learnings and reads them back verbatim", async () => { + const current = "- run tests with `pnpm -r test`\n- default branch is `main`"; + const path = await seedLearningsFile({ tmpdir: dir, current }); + expect(path).toBe(learningsFilePath(dir)); + expect(path.endsWith(LEARNINGS_FILE_NAME)).toBe(true); + const read = await readLearningsFile(path); + expect(read).toBe(current); + }); + + it("seeds an empty file when the repo has no learnings yet", async () => { + // empty seed (vs scaffold-with-comment) keeps the byte-trim equality + // gate clean: an untouched first run reads back as "" and persistLearnings + // skips the API round-trip rather than writing a placeholder string into + // Repo.learnings. + const path = await seedLearningsFile({ tmpdir: dir, current: null }); + const read = await readLearningsFile(path); + expect(read).toBe(""); + }); + + it("returns null when the file is missing (treated as no-change by persist)", async () => { + const path = learningsFilePath(dir); + const read = await readLearningsFile(path); + expect(read).toBeNull(); + }); + + it("trims whitespace so trailing newlines never trigger a spurious PATCH", async () => { + // editors commonly add a trailing newline on save. without trimming, a + // round-trip "read seed → save unchanged" would fail byte-equality and + // burn a LearningsRevision row on every run. + const current = "- one fact"; + const path = await seedLearningsFile({ tmpdir: dir, current }); + await writeFile(path, `${current}\n\n `, "utf8"); + const read = await readLearningsFile(path); + expect(read).toBe(current); + }); + + it("truncates content over the 10k server-side cap", async () => { + // server enforces MAX_LEARNINGS_LENGTH = 10_000. truncating client-side + // avoids a 400 round-trip and keeps the bytes the agent will see in the + // next run aligned with what the server actually stored. + const oversized = "x".repeat(11_000); + const path = await seedLearningsFile({ tmpdir: dir, current: null }); + await writeFile(path, oversized, "utf8"); + const read = await readLearningsFile(path); + expect(read).toBeTruthy(); + expect(read?.length).toBe(10_000); + }); +}); diff --git a/utils/learnings.ts b/utils/learnings.ts new file mode 100644 index 0000000..17b45ec --- /dev/null +++ b/utils/learnings.ts @@ -0,0 +1,64 @@ +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { dirname, join } from "node:path"; + +/** + * Repo-level learnings — operational facts about a repo (setup steps, test + * commands, conventions, gotchas) that accumulate across agent runs and feed + * back into future runs as durable context. Modeled on the PR-summary tmpfile + * pattern (see action/utils/prSummary.ts): + * + * 1. server seeds `pullfrog-learnings.md` from `Repo.learnings` (or empty + * when the repo has none yet) + * 2. the agent reads the file at startup as part of its context, and may + * edit it in place at end-of-run when prompted by the reflection turn + * 3. main.ts reads the file back at end-of-run and PATCHes + * `/api/repo/[owner]/[repo]/learnings` if it changed (byte-trim equality + * against the seed determines change detection) + * + * Edit-in-place avoids stuffing the entire learnings list into both the + * prompt context and an `update_learnings` MCP tool call (which previously + * required passing the FULL merged list as a string parameter — an + * output-token tax that grew linearly with the learnings size). + */ + +export const LEARNINGS_FILE_NAME = "pullfrog-learnings.md"; + +/** server-side cap mirrors `MAX_LEARNINGS_LENGTH` in + * `app/api/repo/[owner]/[repo]/learnings/route.ts`. truncating client-side + * keeps the PATCH from being rejected with a 400. */ +const MAX_LEARNINGS_LENGTH = 10_000; + +export function learningsFilePath(tmpdir: string): string { + return join(tmpdir, LEARNINGS_FILE_NAME); +} + +/** seed the learnings file with the repo's current learnings, or an empty + * file when the repo has none yet. returns the absolute path. */ +export async function seedLearningsFile(params: { + tmpdir: string; + current: string | null; +}): Promise { + const path = learningsFilePath(params.tmpdir); + await mkdir(dirname(path), { recursive: true }); + // empty file when no learnings exist yet — the agent reads it, sees + // nothing, and the LEARNINGS prompt section explains what the file is for. + // a header comment would risk being persisted as part of the first real + // edit, polluting the DB row with placeholder text. + await writeFile(path, params.current ?? "", "utf8"); + return path; +} + +/** read the agent-edited learnings file. returns null when the file is + * missing or unreadable (treated as "no change"). caps content at the + * server's max length to avoid a 400 round-trip. */ +export async function readLearningsFile(path: string): Promise { + let raw: string; + try { + raw = await readFile(path, "utf8"); + } catch { + return null; + } + const trimmed = raw.trim(); + if (trimmed.length > MAX_LEARNINGS_LENGTH) return trimmed.slice(0, MAX_LEARNINGS_LENGTH); + return trimmed; +}