From 6c41f4c662400cdda44db08e2b16d4de501c5be4 Mon Sep 17 00:00:00 2001 From: Dave Date: Thu, 17 Sep 2026 09:19:34 +0200 Subject: [PATCH] =?UTF-8?q?fix(ai):=20raise=20max=5Ftokens=20=E2=80=94=20i?= =?UTF-8?q?nline=20reasoning=20was=20eating=20the=20whole=20budget?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MiniMax M2.x reasons inside tags in the normal content field, and that reasoning is billed against max_tokens. At 4096 the heavier batches spent the entire budget thinking, were truncated before writing a single answer, and stripThinking correctly removed the unclosed block — leaving an empty response. The signature was distinctive: batches returned 25/25 or 0/25, never anything partial. 193 of 779 commits went unsummarised with no error anywhere. At 16384 the same 193 completed 8/8 batches with zero failures. The interview's question call had the same bug at 1024, where it would have returned an empty question and silently ended the interview on the first turn. Now 8192. Co-Authored-By: Claude Opus 5 --- runsum.mts | 119 ------------------------------------ src/lib/ai/narrate.ts | 4 +- src/lib/ai/summarise.ts | 9 ++- src/lib/intake/interview.ts | 8 ++- 4 files changed, 17 insertions(+), 123 deletions(-) delete mode 100644 runsum.mts diff --git a/runsum.mts b/runsum.mts deleted file mode 100644 index 5eaf338..0000000 --- a/runsum.mts +++ /dev/null @@ -1,119 +0,0 @@ -/** - * One-off: summarise a project's commits from the command line. - * - * Imports the same PURE modules the app uses — batch.ts, preamble.ts, - * prompt.ts — so the prompts, the batching and the index-matched parsing are - * byte-identical to what the "Summarise" button does. Only the surrounding - * plumbing (which carries `server-only`) is reimplemented here. - */ -import PocketBase from "pocketbase"; - -import { BATCH_SIZE, buildBatchPrompt, chunk, parseBatchResponse } from "./src/lib/ai/batch"; -import { PROMPT_VERSION } from "./src/lib/ai/preamble"; -import { buildSystemPrompt } from "./src/lib/ai/prompt"; -import { stripThinking } from "./src/lib/ai/think-strip"; - -async function main() { - const pb = new PocketBase(process.env.NEXT_PUBLIC_PB_URL!); - pb.autoCancellation(false); - await pb - .collection("_superusers") - .authWithPassword(process.env.PB_SUPERUSER_EMAIL!, process.env.PB_SUPERUSER_PASSWORD!); - - const settings = await pb.collection("settings").getFirstListItem("id != ''"); - const key: string = settings.ai_api_key ?? ""; - const model: string = settings.ai_model || "MiniMax-M2"; - if (!key) throw new Error("No API key configured — paste it in Settings first."); - - const system = buildSystemPrompt(settings.ai_system_prompt_extra ?? ""); - - const pending = await pb.collection("commits").getFullList({ - filter: [ - "is_merge = false", - `(summary = "" || summary_version < ${PROMPT_VERSION})`, - "summary_attempts < 3", - ].join(" && "), - sort: "-authored_at", - fields: "id,subject,type,scope,summary_attempts", - }); - - console.log(`provider=${settings.ai_provider} model=${model}`); - console.log(`pending commits: ${pending.length}`); - if (pending.length === 0) return; - - const batches = chunk( - pending.map((c) => ({ - id: c.id, - subject: (c.subject as string) || "", - type: (c.type as string) || "other", - scope: (c.scope as string) || "", - })), - BATCH_SIZE - ); - - let summarised = 0; - let missing = 0; - let inTok = 0; - let outTok = 0; - - for (const [index, batch] of batches.entries()) { - process.stdout.write(` batch ${index + 1}/${batches.length} … `); - - let text = ""; - try { - const res = await fetch("https://api.minimax.io/v1/chat/completions", { - method: "POST", - headers: { "content-type": "application/json", authorization: `Bearer ${key}` }, - body: JSON.stringify({ - model, - max_tokens: 4096, - n: 1, - messages: [ - { role: "system", content: system }, - { role: "user", content: buildBatchPrompt(batch) }, - ], - }), - }); - - if (!res.ok) { - console.log(`HTTP ${res.status} — ${(await res.text()).slice(0, 120)}`); - continue; - } - - const json = await res.json(); - text = stripThinking(json.choices?.[0]?.message?.content ?? ""); - inTok += json.usage?.prompt_tokens ?? 0; - outTok += json.usage?.completion_tokens ?? 0; - } catch (error) { - console.log(`failed — ${(error as Error).message}`); - continue; - } - - const parsed = parseBatchResponse(text, batch); - - for (const commit of batch) { - const summary = parsed.get(commit.id); - if (!summary) { - missing += 1; - continue; - } - await pb.collection("commits").update(commit.id, { - summary, - summary_version: PROMPT_VERSION, - summary_attempts: 0, - }); - summarised += 1; - } - - console.log(`${parsed.size}/${batch.length}`); - } - - console.log(`\nsummarised: ${summarised}`); - console.log(`not returned by the model: ${missing}`); - console.log(`tokens: ${inTok} in, ${outTok} out`); -} - -main().catch((e) => { - console.error("FAILED:", e?.message ?? e); - process.exit(1); -}); diff --git a/src/lib/ai/narrate.ts b/src/lib/ai/narrate.ts index dc53afb..5484a6f 100644 --- a/src/lib/ai/narrate.ts +++ b/src/lib/ai/narrate.ts @@ -103,7 +103,9 @@ export async function narrateProject(projectId: string): Promise { title: card.title, description: card.description ?? "" }, items ), - maxTokens: 4096, + // 16k for the same reason as summarise.ts: inline reasoning is billed + // against max_tokens, and a truncated response yields nothing at all. + maxTokens: 16384, }); if (!result.text) { diff --git a/src/lib/ai/summarise.ts b/src/lib/ai/summarise.ts index dc6e20c..dbd1c83 100644 --- a/src/lib/ai/summarise.ts +++ b/src/lib/ai/summarise.ts @@ -101,7 +101,14 @@ export async function summariseProject(projectId: string): Promise tags, and that + // reasoning is billed against max_tokens. At 4096 the heavier batches + // spent the whole budget thinking, were truncated before writing a + // single answer, and stripThinking correctly removed the unclosed + // block — leaving an empty response and zero parsed summaries. The + // symptom was whole batches returning 0/25 while others returned + // 25/25, never anything partial. + maxTokens: 16384, }); text = result.text; } catch (error) { diff --git a/src/lib/intake/interview.ts b/src/lib/intake/interview.ts index 6a91946..2079daf 100644 --- a/src/lib/intake/interview.ts +++ b/src/lib/intake/interview.ts @@ -94,7 +94,10 @@ export async function nextStep(input: { const result = await provider.complete({ system, user: buildQuestionPrompt(ctx.context, turns), - maxTokens: 1024, + // A question is one sentence, but the reasoning that produces it is + // billed against this too — see summarise.ts. At 1024 a reasoning + // model returns an empty question and the interview silently ends. + maxTokens: 8192, }); const question = result.text.trim(); @@ -113,7 +116,8 @@ export async function nextStep(input: { const result = await provider.complete({ system, user: buildBriefPrompt(ctx.context, turns), - maxTokens: 4096, + // 16k — see the note in summarise.ts about inline reasoning. + maxTokens: 16384, }); raw = result.text; } catch {