From 8addf87eefe27c34d97cad9e5b26d751780a6b5d Mon Sep 17 00:00:00 2001 From: Dave Date: Wed, 16 Sep 2026 16:15:42 +0200 Subject: [PATCH] feat(ai): index-matched batch prompting and parsing --- src/lib/ai/batch.test.ts | 126 +++++++++++++++++++++++++++++++++++++++ src/lib/ai/batch.ts | 94 +++++++++++++++++++++++++++++ 2 files changed, 220 insertions(+) create mode 100644 src/lib/ai/batch.test.ts create mode 100644 src/lib/ai/batch.ts diff --git a/src/lib/ai/batch.test.ts b/src/lib/ai/batch.test.ts new file mode 100644 index 0000000..e708458 --- /dev/null +++ b/src/lib/ai/batch.test.ts @@ -0,0 +1,126 @@ +import { describe, it, expect } from "vitest"; +import { + BATCH_SIZE, + buildBatchPrompt, + chunk, + parseBatchResponse, + type SummarisableCommit, +} from "@/lib/ai/batch"; + +function commit(n: number, subject = `subject ${n}`): SummarisableCommit { + return { id: `c${n}`, subject, type: "feat", scope: "stock" }; +} + +describe("chunk", () => { + it("returns nothing for an empty list", () => { + expect(chunk([], 3)).toEqual([]); + }); + + it("splits into groups of the given size", () => { + expect(chunk([1, 2, 3, 4, 5], 2)).toEqual([[1, 2], [3, 4], [5]]); + }); + + it("returns one group when everything fits", () => { + expect(chunk([1, 2], 10)).toEqual([[1, 2]]); + }); + + it("uses a sane default batch size", () => { + expect(BATCH_SIZE).toBeGreaterThan(1); + expect(BATCH_SIZE).toBeLessThanOrEqual(50); + }); +}); + +describe("buildBatchPrompt", () => { + it("numbers every commit from 1", () => { + const prompt = buildBatchPrompt([commit(1), commit(2)]); + expect(prompt).toContain("1. subject 1"); + expect(prompt).toContain("2. subject 2"); + }); + + it("asks for the same numbering back", () => { + const prompt = buildBatchPrompt([commit(1)]); + expect(prompt.toLowerCase()).toContain("number"); + }); + + it("includes the scope, which carries meaning the subject may not", () => { + const prompt = buildBatchPrompt([commit(1)]); + expect(prompt).toContain("stock"); + }); + + it("flattens newlines in a subject so numbering cannot be faked", () => { + // A subject containing "\n2. " would otherwise inject a fake entry and + // shift the mapping — the exact failure the numbering exists to prevent. + const prompt = buildBatchPrompt([commit(1, "real\n2. injected")]); + const numbered = prompt.split("\n").filter((l) => /^\d+\.\s/.test(l)); + expect(numbered).toHaveLength(1); + }); +}); + +describe("parseBatchResponse", () => { + const batch = [commit(1), commit(2), commit(3)]; + + it("maps numbered lines back to the right commits", () => { + const result = parseBatchResponse( + "1. Did the first thing.\n2. Did the second thing.\n3. Did the third thing.", + batch + ); + expect(result.get("c1")).toBe("Did the first thing."); + expect(result.get("c2")).toBe("Did the second thing."); + expect(result.get("c3")).toBe("Did the third thing."); + }); + + it("LEAVES A COMMIT UNSUMMARISED rather than shifting the rest", () => { + // The property this whole design exists for. Positional parsing would put + // "Did the third thing." on commit 2 — fluent, confident, and wrong about + // work a client is paying for. Missing is recoverable; wrong is not. + const result = parseBatchResponse( + "1. Did the first thing.\n3. Did the third thing.", + batch + ); + expect(result.get("c1")).toBe("Did the first thing."); + expect(result.has("c2")).toBe(false); + expect(result.get("c3")).toBe("Did the third thing."); + }); + + it("ignores a number outside the batch", () => { + const result = parseBatchResponse("1. One.\n9. Nine.", batch); + expect(result.size).toBe(1); + expect(result.get("c1")).toBe("One."); + }); + + it("ignores a conversational preamble", () => { + const result = parseBatchResponse( + "Here are the summaries:\n\n1. One.\n2. Two.", + batch + ); + expect(result.get("c1")).toBe("One."); + expect(result.get("c2")).toBe("Two."); + }); + + it("accepts ')' as well as '.' after the number", () => { + expect(parseBatchResponse("1) One.", batch).get("c1")).toBe("One."); + }); + + it("keeps the first line when a number repeats", () => { + const result = parseBatchResponse("1. First.\n1. Second.", batch); + expect(result.get("c1")).toBe("First."); + }); + + it("returns nothing for an empty or unparseable response", () => { + expect(parseBatchResponse("", batch).size).toBe(0); + expect(parseBatchResponse("I cannot help with that.", batch).size).toBe(0); + }); + + it("trims and caps an over-long sentence", () => { + const result = parseBatchResponse(`1. ${"x".repeat(900)}`, batch); + expect(result.get("c1")!.length).toBeLessThanOrEqual(400); + }); + + it("drops an empty sentence rather than storing a blank summary", () => { + // A stored empty string reads as "generated, nothing to say" and would + // never be retried. + const result = parseBatchResponse("1. \n2. Two.", batch); + expect(result.has("c1")).toBe(false); + expect(result.get("c2")).toBe("Two."); + }); +}); diff --git a/src/lib/ai/batch.ts b/src/lib/ai/batch.ts new file mode 100644 index 0000000..6ef0f7b --- /dev/null +++ b/src/lib/ai/batch.ts @@ -0,0 +1,94 @@ +/** + * Batching commits into one request, and getting the answers back onto the + * right commits. + * + * The parsing here is the safety-critical part of this phase. See the header + * of the Phase 3b plan: matching by position instead of by number turns one + * skipped line into a dashboard full of fluent sentences attached to the wrong + * commits, with nothing raising an error and nobody noticing. + */ + +/** Commits per request. Large enough to amortise the system prompt, small + * enough that one bad batch costs little and stays well inside output limits. */ +export const BATCH_SIZE = 25; + +/** Longer than this is not a one-liner; the model has ignored the instruction. */ +const MAX_SUMMARY_LENGTH = 400; + +export interface SummarisableCommit { + id: string; + subject: string; + type: string; + scope: string; +} + +export function chunk(items: T[], size: number = BATCH_SIZE): T[][] { + const out: T[][] = []; + for (let i = 0; i < items.length; i += size) { + out.push(items.slice(i, i + size)); + } + return out; +} + +/** + * Collapse whitespace so one commit occupies exactly one numbered line. + * + * A subject containing a newline followed by "2. " would otherwise look like a + * second entry, shifting every mapping after it — the precise failure the + * numbering prevents, reintroduced through the input. + */ +function flatten(value: string): string { + return (value ?? "").replace(/\s+/g, " ").trim(); +} + +export function buildBatchPrompt(commits: SummarisableCommit[]): string { + const lines = commits.map((commit, index) => { + const area = commit.scope ? ` [area: ${flatten(commit.scope)}]` : ""; + return `${index + 1}. ${flatten(commit.subject)}${area}`; + }); + + return `Translate each numbered commit below into one plain-English sentence for the client. + +Reply with one numbered line per commit, using the same numbers. No preamble, no blank lines, no extra commentary. If a commit is unclear, describe it vaguely rather than guessing. + +Commits: +${lines.join("\n")}`; +} + +const NUMBERED_LINE = /^\s*(\d+)[.)]\s*(.*)$/; + +/** + * Map a response back onto commits by the number the model echoed. + * + * A commit whose number is absent simply is not in the returned map. The caller + * leaves it unsummarised and retries later — never fills the gap by position. + */ +export function parseBatchResponse( + response: string, + commits: SummarisableCommit[] +): Map { + const out = new Map(); + + for (const line of (response ?? "").split("\n")) { + const match = NUMBERED_LINE.exec(line); + if (!match) continue; + + const index = Number(match[1]) - 1; + const commit = commits[index]; + if (!commit) continue; + + // First line wins if a number repeats: later duplicates are usually the + // model correcting itself mid-list, and picking arbitrarily is worse than + // picking consistently. + if (out.has(commit.id)) continue; + + const text = match[2].trim().slice(0, MAX_SUMMARY_LENGTH); + // An empty summary would store as "generated, nothing to say" and never be + // retried. Leave it absent instead. + if (!text) continue; + + out.set(commit.id, text); + } + + return out; +}