David/WFDASH-WEBAPP
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(collection-ai): re-read sideways invoice photos turned upright

Browse source 
A sideways Bestobell invoice was read as doc 10045064 / a bank-details
line; turned 270 deg the same photo reads correctly (10245084,
PO202610010MRE, rubber insertion @ R365). When no printed PO number
resolves, retry the photo at 270 and 90 deg and keep the first read whose
PO resolves (within the tick's time budget).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Dave 2026-10-17 16:07:26 +02:00
parent f8294b4e1d
commit 79d54ce3fd
2 changed files with 95 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

51
src/lib/server/collection-ai-service.test.ts
View file

@ -435,10 +435,16 @@ function makeDeps(over: Partial<CollectionAiDeps> = {}) {
const bytes = Buffer.from(contents[filename] ?? `bytes of ${filename}`);
return { base64: bytes.toString("base64"), contentType: "image/jpeg", bytes };
});
// Rotation stub: marks the bytes so a readDocument mock can tell which turn it got.
const rotateImage = vi.fn(async (bytes: Buffer, angle: number) => {
const rotated = Buffer.from(`rot${angle}:${bytes.toString()}`);
return { base64: rotated.toString("base64"), contentType: "image/jpeg" };
});
const deps = {
readDocument,
pairLines,
fetchFile,
rotateImage,
now: () => NOW,
...over,
} as CollectionAiDeps & {
@ -875,6 +881,48 @@ describe("checkCollectionCard", () => {
});
});
it("re-reads a sideways photo turned upright when no printed PO number resolves", async () => {
seedNss();
const readDocument = vi.fn(async (img: { base64: string }) =>
Buffer.from(img.base64, "base64").toString().startsWith("rot270:")
? nssRead({ document_no: "10245084", po_refs: [PO_CEM] })
: nssRead({ document_no: "10045064", po_refs: ["PO202610010019CEM/207"] }),
);
const { deps } = makeDeps({ readDocument });
const r = await checkCollectionCard(PO_CEM, { deps });
expect(readDocument).toHaveBeenCalledTimes(2);
const read = reads()[0];
expect(read.document_no).toBe("10245084");
expect(read.po_refs).toEqual([PO_CEM]);
expect(read.notes as string[]).toContain("Read with the photo turned 270°.");
expect(r.files_read).toBe(1);
});
it("reads an upright photo once when its PO number resolves", async () => {
seedNss();
const { deps } = makeDeps();
await checkCollectionCard(PO_CEM, { deps });
expect(deps.readDocument).toHaveBeenCalledTimes(1);
expect(deps.rotateImage).not.toHaveBeenCalled();
});
it("keeps the original read when no turn resolves a PO number either", async () => {
seedNss();
const readDocument = vi.fn(async () => nssRead({ po_refs: ["ORDER 99887"] }));
const { deps } = makeDeps({ readDocument });
await checkCollectionCard(PO_CEM, { deps });
expect(readDocument).toHaveBeenCalledTimes(3);
const read = reads()[0];
expect(read.po_refs).toEqual(["ORDER 99887"]);
expect((read.notes as string[]).some((n) => n.startsWith("Read with the photo turned"))).toBe(false);
});
it("falls back to the card's PO when no printed PO reference resolves, and says so", async () => {
seedNss();
const { deps } = makeDeps({
@ -1042,7 +1090,8 @@ describe("runCollectionAiTick", () => {
const r = await runCollectionAiTick(deps);
expect(deps.readDocument).toHaveBeenCalledTimes(TICK_MAX_FILES);
// Files, not AI calls: a photo whose PO doesn't resolve is also re-read turned.
expect(deps.fetchFile).toHaveBeenCalledTimes(TICK_MAX_FILES);
expect(r.summary?.files_read).toBe(TICK_MAX_FILES);
expect(deps.fetchFile.mock.calls[0][1]).toBe("p12.jpg");
expect(deps.fetchFile.mock.calls.map((c) => c[1])).not.toContain("p1.jpg");

51
src/lib/server/collection-ai-service.ts
View file

@ -51,6 +51,8 @@ export const TICK_LEASE_MS = 10 * 60_000;
export interface CollectionAiDeps {
readDocument?: typeof readDocumentImage;
/** Turns an image by `angle` degrees clockwise (default: sharp). */
rotateImage?: (bytes: Buffer, angle: number) => Promise<{ base64: string; contentType: string }>;
pairLines?: typeof pairDocumentLines;
fetchFile?: (
rec: { collection: string; id: string },
@ -386,6 +388,23 @@ async function defaultFetchFile(
return { base64: bytes.toString("base64"), contentType, bytes };
}
/**
* Drivers often photograph landscape invoices sideways, and the AI misreads
* sideways text badly (wrong numbers, bank details taken for line items).
* When a read names no PO we know, the photo is re-read at these turns and
* the first read whose PO number resolves wins.
*/
const RETRY_ROTATIONS = [270, 90] as const;
async function defaultRotateImage(
bytes: Buffer,
angle: number,
): Promise<{ base64: string; contentType: string }> {
const { default: sharp } = await import("sharp");
const out = await sharp(bytes).rotate(angle).jpeg({ quality: 90 }).toBuffer();
return { base64: out.toString("base64"), contentType: "image/jpeg" };
}
type ReadFields = Omit<CollectionDocRead, "id">;
function emptyRead(f: FileRef, now: Date): ReadFields {
@ -503,6 +522,7 @@ async function checkCard(
const readDocument = deps.readDocument ?? readDocumentImage;
const pairLines = deps.pairLines ?? pairDocumentLines;
const fetchFile = deps.fetchFile ?? defaultFetchFile;
const rotateImage = deps.rotateImage ?? defaultRotateImage;
const nowFn = deps.now ?? (() => new Date());
const maxFiles = opts.maxFiles ?? TICK_MAX_FILES;
const actorName = opts.actorName ? `${ACTOR_NAME} (run by ${opts.actorName})` : ACTOR_NAME;
@ -548,6 +568,29 @@ async function checkCard(
/** Hashes whose AI read failed in THIS run: copies aren't re-sent. */
const failedByHash = new Map<string, string>();
const queued: Array<{ file: FileRef; fields: ReadFields; replaceId: string | null }> = [];
let poIndex: PoIndex | null = null;
const getPoIndex = async () => (poIndex ??= await loadPoIndex(pb));
/** Read a photo; if no PO number it names resolves, retry it turned. */
const readOriented = async (fetched: { base64: string; contentType: string; bytes: Buffer }) => {
const first = await readDocument({ base64: fetched.base64, contentType: fetched.contentType });
if (first.doc_type === "other") return first;
if (resolveRefs(await getPoIndex(), first.po_refs).length > 0) return first;
for (const angle of RETRY_ROTATIONS) {
if (opts.deadline !== undefined && nowFn().getTime() >= opts.deadline) break;
let turned: DocumentReadResult;
try {
turned = await readDocument(await rotateImage(fetched.bytes, angle));
} catch (e) {
if (e instanceof AiDisabledError) throw e;
continue;
}
if (resolveRefs(await getPoIndex(), turned.po_refs).length > 0) {
return { ...turned, notes: [...turned.notes, `Read with the photo turned ${angle}°.`] };
}
}
return first;
};
const readOneFile = async (file: FileRef): Promise<void> => {
const existing = readsByFile.get(fileKey(file));
@ -599,7 +642,7 @@ async function checkCard(
let fields: ReadFields;
try {
const read = await readDocument({ base64: fetched.base64, contentType: fetched.contentType });
const read = await readOriented(fetched);
fields = fromResult(file, hash, read, now);
result.files_read += 1;
} catch (e) {
@ -658,7 +701,6 @@ async function checkCard(
if (docs.length === 0) return result;
// 4. Pair each document with the lines of every PO it names.
let poIndex: PoIndex | null = null;
const candidates = new Map<string, Candidate[]>();
const lineDocs = new Map<string, CollectionDocRead[]>();
const linesById = new Map(cardLines.map((l) => [l.id, l]));
@ -668,10 +710,7 @@ async function checkCard(
for (const doc of docs) {
try {
let fromRefs: PoRef[] = [];
if (doc.po_refs.length > 0) {
poIndex ??= await loadPoIndex(pb);
fromRefs = resolveRefs(poIndex, doc.po_refs);
}
if (doc.po_refs.length > 0) fromRefs = resolveRefs(await getPoIndex(), doc.po_refs);
// The findings speak about the POs the document names (or the card's
// PO when none resolved) ...
const fellBackToPo: string | null = fromRefs.length === 0 ? poNumber : null;