From 79d54ce3fd73cd360196d5426c7056655a4b4c13 Mon Sep 17 00:00:00 2001 From: Dave Date: Sat, 17 Oct 2026 16:07:26 +0200 Subject: [PATCH] fix(collection-ai): re-read sideways invoice photos turned upright A sideways Bestobell invoice was read as doc 10045064 / a bank-details line; turned 270 deg the same photo reads correctly (10245084, PO202610010MRE, rubber insertion @ R365). When no printed PO number resolves, retry the photo at 270 and 90 deg and keep the first read whose PO resolves (within the tick's time budget). Co-Authored-By: Claude Opus 5.5 --- src/lib/server/collection-ai-service.test.ts | 51 +++++++++++++++++++- src/lib/server/collection-ai-service.ts | 51 +++++++++++++++++--- 2 files changed, 95 insertions(+), 7 deletions(-) diff --git a/src/lib/server/collection-ai-service.test.ts b/src/lib/server/collection-ai-service.test.ts index ab6afa1..5230109 100644 --- a/src/lib/server/collection-ai-service.test.ts +++ b/src/lib/server/collection-ai-service.test.ts @@ -435,10 +435,16 @@ function makeDeps(over: Partial = {}) { const bytes = Buffer.from(contents[filename] ?? `bytes of ${filename}`); return { base64: bytes.toString("base64"), contentType: "image/jpeg", bytes }; }); + // Rotation stub: marks the bytes so a readDocument mock can tell which turn it got. + const rotateImage = vi.fn(async (bytes: Buffer, angle: number) => { + const rotated = Buffer.from(`rot${angle}:${bytes.toString()}`); + return { base64: rotated.toString("base64"), contentType: "image/jpeg" }; + }); const deps = { readDocument, pairLines, fetchFile, + rotateImage, now: () => NOW, ...over, } as CollectionAiDeps & { @@ -875,6 +881,48 @@ describe("checkCollectionCard", () => { }); }); + it("re-reads a sideways photo turned upright when no printed PO number resolves", async () => { + seedNss(); + const readDocument = vi.fn(async (img: { base64: string }) => + Buffer.from(img.base64, "base64").toString().startsWith("rot270:") + ? nssRead({ document_no: "10245084", po_refs: [PO_CEM] }) + : nssRead({ document_no: "10045064", po_refs: ["PO202610010019CEM/207"] }), + ); + const { deps } = makeDeps({ readDocument }); + + const r = await checkCollectionCard(PO_CEM, { deps }); + + expect(readDocument).toHaveBeenCalledTimes(2); + const read = reads()[0]; + expect(read.document_no).toBe("10245084"); + expect(read.po_refs).toEqual([PO_CEM]); + expect(read.notes as string[]).toContain("Read with the photo turned 270°."); + expect(r.files_read).toBe(1); + }); + + it("reads an upright photo once when its PO number resolves", async () => { + seedNss(); + const { deps } = makeDeps(); + + await checkCollectionCard(PO_CEM, { deps }); + + expect(deps.readDocument).toHaveBeenCalledTimes(1); + expect(deps.rotateImage).not.toHaveBeenCalled(); + }); + + it("keeps the original read when no turn resolves a PO number either", async () => { + seedNss(); + const readDocument = vi.fn(async () => nssRead({ po_refs: ["ORDER 99887"] })); + const { deps } = makeDeps({ readDocument }); + + await checkCollectionCard(PO_CEM, { deps }); + + expect(readDocument).toHaveBeenCalledTimes(3); + const read = reads()[0]; + expect(read.po_refs).toEqual(["ORDER 99887"]); + expect((read.notes as string[]).some((n) => n.startsWith("Read with the photo turned"))).toBe(false); + }); + it("falls back to the card's PO when no printed PO reference resolves, and says so", async () => { seedNss(); const { deps } = makeDeps({ @@ -1042,7 +1090,8 @@ describe("runCollectionAiTick", () => { const r = await runCollectionAiTick(deps); - expect(deps.readDocument).toHaveBeenCalledTimes(TICK_MAX_FILES); + // Files, not AI calls: a photo whose PO doesn't resolve is also re-read turned. + expect(deps.fetchFile).toHaveBeenCalledTimes(TICK_MAX_FILES); expect(r.summary?.files_read).toBe(TICK_MAX_FILES); expect(deps.fetchFile.mock.calls[0][1]).toBe("p12.jpg"); expect(deps.fetchFile.mock.calls.map((c) => c[1])).not.toContain("p1.jpg"); diff --git a/src/lib/server/collection-ai-service.ts b/src/lib/server/collection-ai-service.ts index 32cd6fe..92e54ae 100644 --- a/src/lib/server/collection-ai-service.ts +++ b/src/lib/server/collection-ai-service.ts @@ -51,6 +51,8 @@ export const TICK_LEASE_MS = 10 * 60_000; export interface CollectionAiDeps { readDocument?: typeof readDocumentImage; + /** Turns an image by `angle` degrees clockwise (default: sharp). */ + rotateImage?: (bytes: Buffer, angle: number) => Promise<{ base64: string; contentType: string }>; pairLines?: typeof pairDocumentLines; fetchFile?: ( rec: { collection: string; id: string }, @@ -386,6 +388,23 @@ async function defaultFetchFile( return { base64: bytes.toString("base64"), contentType, bytes }; } +/** + * Drivers often photograph landscape invoices sideways, and the AI misreads + * sideways text badly (wrong numbers, bank details taken for line items). + * When a read names no PO we know, the photo is re-read at these turns and + * the first read whose PO number resolves wins. + */ +const RETRY_ROTATIONS = [270, 90] as const; + +async function defaultRotateImage( + bytes: Buffer, + angle: number, +): Promise<{ base64: string; contentType: string }> { + const { default: sharp } = await import("sharp"); + const out = await sharp(bytes).rotate(angle).jpeg({ quality: 90 }).toBuffer(); + return { base64: out.toString("base64"), contentType: "image/jpeg" }; +} + type ReadFields = Omit; function emptyRead(f: FileRef, now: Date): ReadFields { @@ -503,6 +522,7 @@ async function checkCard( const readDocument = deps.readDocument ?? readDocumentImage; const pairLines = deps.pairLines ?? pairDocumentLines; const fetchFile = deps.fetchFile ?? defaultFetchFile; + const rotateImage = deps.rotateImage ?? defaultRotateImage; const nowFn = deps.now ?? (() => new Date()); const maxFiles = opts.maxFiles ?? TICK_MAX_FILES; const actorName = opts.actorName ? `${ACTOR_NAME} (run by ${opts.actorName})` : ACTOR_NAME; @@ -548,6 +568,29 @@ async function checkCard( /** Hashes whose AI read failed in THIS run: copies aren't re-sent. */ const failedByHash = new Map(); const queued: Array<{ file: FileRef; fields: ReadFields; replaceId: string | null }> = []; + let poIndex: PoIndex | null = null; + const getPoIndex = async () => (poIndex ??= await loadPoIndex(pb)); + + /** Read a photo; if no PO number it names resolves, retry it turned. */ + const readOriented = async (fetched: { base64: string; contentType: string; bytes: Buffer }) => { + const first = await readDocument({ base64: fetched.base64, contentType: fetched.contentType }); + if (first.doc_type === "other") return first; + if (resolveRefs(await getPoIndex(), first.po_refs).length > 0) return first; + for (const angle of RETRY_ROTATIONS) { + if (opts.deadline !== undefined && nowFn().getTime() >= opts.deadline) break; + let turned: DocumentReadResult; + try { + turned = await readDocument(await rotateImage(fetched.bytes, angle)); + } catch (e) { + if (e instanceof AiDisabledError) throw e; + continue; + } + if (resolveRefs(await getPoIndex(), turned.po_refs).length > 0) { + return { ...turned, notes: [...turned.notes, `Read with the photo turned ${angle}°.`] }; + } + } + return first; + }; const readOneFile = async (file: FileRef): Promise => { const existing = readsByFile.get(fileKey(file)); @@ -599,7 +642,7 @@ async function checkCard( let fields: ReadFields; try { - const read = await readDocument({ base64: fetched.base64, contentType: fetched.contentType }); + const read = await readOriented(fetched); fields = fromResult(file, hash, read, now); result.files_read += 1; } catch (e) { @@ -658,7 +701,6 @@ async function checkCard( if (docs.length === 0) return result; // 4. Pair each document with the lines of every PO it names. - let poIndex: PoIndex | null = null; const candidates = new Map(); const lineDocs = new Map(); const linesById = new Map(cardLines.map((l) => [l.id, l])); @@ -668,10 +710,7 @@ async function checkCard( for (const doc of docs) { try { let fromRefs: PoRef[] = []; - if (doc.po_refs.length > 0) { - poIndex ??= await loadPoIndex(pb); - fromRefs = resolveRefs(poIndex, doc.po_refs); - } + if (doc.po_refs.length > 0) fromRefs = resolveRefs(await getPoIndex(), doc.po_refs); // The findings speak about the POs the document names (or the card's // PO when none resolved) ... const fellBackToPo: string | null = fromRefs.length === 0 ? poNumber : null;