From 8fc4930de778d42433252b199d373f9c1817f033 Mon Sep 17 00:00:00 2001 From: Dave Date: Sat, 17 Oct 2026 01:40:33 +0200 Subject: [PATCH] feat(collection-ai): vision read of driver documents Co-Authored-By: Claude Sonnet 5.5 --- src/lib/server/collection-ai-prompt.ts | 14 ++ src/lib/server/collection-ai-read.test.ts | 130 +++++++++++++++++ src/lib/server/collection-ai-read.ts | 170 ++++++++++++++++++++++ 3 files changed, 314 insertions(+) create mode 100644 src/lib/server/collection-ai-prompt.ts create mode 100644 src/lib/server/collection-ai-read.test.ts create mode 100644 src/lib/server/collection-ai-read.ts diff --git a/src/lib/server/collection-ai-prompt.ts b/src/lib/server/collection-ai-prompt.ts new file mode 100644 index 0000000..6e80162 --- /dev/null +++ b/src/lib/server/collection-ai-prompt.ts @@ -0,0 +1,14 @@ +/** + * Default system prompt for reading a driver's photo of a supplier document. + * Operators can override it via the `collection_ai_prompt` AI setting. + */ +export const DEFAULT_COLLECTION_AI_PROMPT = `You read photos of South African supplier documents (tax invoices and delivery notes) that a driver took when collecting goods. Return exactly what is printed via the submit_document_read tool. + +Rules: +1. doc_type: "tax_invoice", "delivery_note", or "other" (a photo of goods, not paperwork). +2. document_no: the invoice or delivery-note number printed as the document's own number (not account or VAT numbers). date: the document date as YYYY-MM-DD. +3. po_refs: every customer order / PO number printed (often under "Customer PO", "Customer Order No" or "Your Ref"). Split values like "PO202610003CEM/PO202610009CAS" into separate entries. +4. lines: one entry per item line. code = the supplier's stock code; description as printed; quantity = quantity supplied; unit as printed. +5. unit_price must be the price for ONE unit. If the document prices per 100 (or per 1000, per box of N), divide accordingly and say so in price_note (e.g. "printed 33.00 per 100"). +6. to_follow: the "to follow" / back-order quantity if printed, else null. +7. Never guess a number you cannot read clearly: use null. The photo may be rotated or partly covered.`; diff --git a/src/lib/server/collection-ai-read.test.ts b/src/lib/server/collection-ai-read.test.ts new file mode 100644 index 0000000..197f72b --- /dev/null +++ b/src/lib/server/collection-ai-read.test.ts @@ -0,0 +1,130 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const getAiSettings = vi.fn(); +vi.mock("@/lib/server/ai-settings-service", () => ({ + getActiveAiConfig: vi.fn(async () => ({ + modelId: "MiniMax-M3", + maxTokens: 2000, + apiKey: "k", + baseUrl: "u", + provider: "minimax", + })), + getAiSettings: (...a: unknown[]) => getAiSettings(...a), +})); + +import { + parseDocumentRead, + readDocumentImage, + SUBMIT_DOCUMENT_READ_TOOL, +} from "./collection-ai-read"; +import { DEFAULT_COLLECTION_AI_PROMPT } from "./collection-ai-prompt"; + +const nssInput = { + doc_type: "tax_invoice", + document_no: "INV12345", + doc_date: "2026-10-14", + supplier_name: "NSS Fasteners", + po_refs: ["PO202610003CEM", "PO202610009CAS"], + lines: [ + { code: "A1", description: "Bolt", quantity: 100, unit: "ea", unit_price: 0.95, line_total: 95, to_follow: null, price_note: "" }, + { code: "A2", description: "Nut", quantity: 100, unit: "ea", unit_price: 0.33, line_total: 33, to_follow: null, price_note: "printed 33.00 per 100" }, + { code: "A3", description: "Washer", quantity: 50, unit: "ea", unit_price: 0.2, line_total: 10, to_follow: 5, price_note: "" }, + { code: "A4", description: "Anchor", quantity: 2, unit: "ea", unit_price: 14.51, line_total: 29.02, to_follow: null, price_note: "" }, + ], + notes: ["Rotated photo"], +}; + +function fakeClient(content: unknown[], stop_reason = "tool_use") { + const create = vi.fn<(args: unknown) => Promise<{ content: unknown[]; stop_reason: string }>>( + async () => ({ content, stop_reason }), + ); + return { client: { messages: { create } } as never, create }; +} +const toolUse = (input: unknown) => ({ type: "tool_use", id: "t1", name: "submit_document_read", input }); +const img = { base64: "AAAA", contentType: "image/jpeg" }; + +type Args = { + tool_choice: unknown; + tools: unknown; + system: string; + max_tokens: number; + messages: Array<{ content: Array<{ type: string; source?: unknown }> }>; +}; + +beforeEach(() => { + getAiSettings.mockReset(); + getAiSettings.mockResolvedValue({ collection_ai_prompt: "" }); +}); + +describe("parseDocumentRead", () => { + it("returns the NSS fixture unchanged and numbers lines 1..n", () => { + const r = parseDocumentRead(nssInput); + expect(r.po_refs).toEqual(["PO202610003CEM", "PO202610009CAS"]); + expect(r.lines.map((l) => l.unit_price)).toEqual([0.95, 0.33, 0.2, 14.51]); + expect(r.lines.map((l) => l.line_no)).toEqual([1, 2, 3, 4]); + expect(r.lines[2].to_follow).toBe(5); + expect(r.doc_type).toBe("tax_invoice"); + expect(r.document_no).toBe("INV12345"); + expect(r.notes).toEqual(["Rotated photo"]); + }); + + it("is defensive about bad input", () => { + const r = parseDocumentRead({ + doc_type: "receipt", + document_no: " X1 ", + doc_date: "14/10/2026", + lines: [{ code: " c ", description: " d ", quantity: "lots", unit_price: "abc", unit: " ea " }], + }); + expect(r.doc_type).toBe("other"); + expect(r.document_no).toBe("X1"); + expect(r.doc_date).toBe(""); + expect(r.lines[0]).toMatchObject({ code: "c", description: "d", unit: "ea", quantity: null, unit_price: null, line_no: 1 }); + expect(parseDocumentRead({}).lines).toEqual([]); + expect(parseDocumentRead(null).doc_type).toBe("other"); + }); +}); + +describe("readDocumentImage", () => { + it("sends one image block plus text, forces the tool, uses the default prompt", async () => { + const { client, create } = fakeClient([toolUse(nssInput)]); + const res = await readDocumentImage(img, { client }); + const args = create.mock.calls[0][0] as Args; + expect(args.tool_choice).toEqual({ type: "tool", name: "submit_document_read" }); + expect(args.tools).toEqual([SUBMIT_DOCUMENT_READ_TOOL]); + expect(args.system).toBe(DEFAULT_COLLECTION_AI_PROMPT); + expect(args.max_tokens).toBe(4096); + const content = args.messages[0].content; + expect(content.filter((b) => b.type === "image")).toHaveLength(1); + expect(content[0].source).toEqual({ type: "base64", media_type: "image/jpeg", data: "AAAA" }); + expect(content.some((b) => b.type === "text")).toBe(true); + expect(res.model).toBe("MiniMax-M3"); + expect(res.lines).toHaveLength(4); + }); + + it("uses a non-blank override prompt", async () => { + getAiSettings.mockResolvedValue({ collection_ai_prompt: "Custom" }); + const { client, create } = fakeClient([toolUse(nssInput)]); + await readDocumentImage(img, { client }); + expect((create.mock.calls[0][0] as { system: string }).system).toBe("Custom"); + getAiSettings.mockResolvedValue({ collection_ai_prompt: " \n" }); + await readDocumentImage(img, { client }); + expect((create.mock.calls[1][0] as { system: string }).system).toBe(DEFAULT_COLLECTION_AI_PROMPT); + }); + + it("rejects unsupported image types before calling the AI", async () => { + const { client, create } = fakeClient([toolUse(nssInput)]); + await expect(readDocumentImage({ base64: "x", contentType: "application/pdf" }, { client })).rejects.toThrow( + "Unsupported image type: application/pdf", + ); + expect(create).not.toHaveBeenCalled(); + }); + + it("throws on max_tokens and on a missing tool call", async () => { + await expect( + readDocumentImage(img, { client: fakeClient([toolUse(nssInput)], "max_tokens").client }), + ).rejects.toThrow("AI response was cut off (max_tokens)"); + await expect( + readDocumentImage(img, { client: fakeClient([{ type: "text", text: "no" }]).client }), + ).rejects.toThrow("AI returned no submit_document_read tool call"); + }); +}); diff --git a/src/lib/server/collection-ai-read.ts b/src/lib/server/collection-ai-read.ts new file mode 100644 index 0000000..b7b821a --- /dev/null +++ b/src/lib/server/collection-ai-read.ts @@ -0,0 +1,170 @@ +/** + * collection-ai-read — vision read of one driver-photographed supplier + * document (tax invoice / delivery note) into structured data. + * + * Same wiring as quote-assistant-ai: built-in AI (MiniMax via the Anthropic + * SDK), forced tool call, injectable client for tests. + */ +import Anthropic from "@anthropic-ai/sdk"; +import type { CollectionDocLine, CollectionDocRead, CollectionDocType } from "@/lib/types"; +import { getActiveAiConfig, getAiSettings } from "./ai-settings-service"; +import { DEFAULT_COLLECTION_AI_PROMPT } from "./collection-ai-prompt"; + +const TOOL_NAME = "submit_document_read" as const; + +export const SUBMIT_DOCUMENT_READ_TOOL = { + name: TOOL_NAME, + description: + "Submit what is printed on the supplier document photo. Call this tool exactly once. Use null for any number you cannot read clearly.", + input_schema: { + type: "object" as const, + properties: { + doc_type: { + type: "string", + enum: ["tax_invoice", "delivery_note", "other"], + description: "Kind of document; \"other\" for a photo of goods rather than paperwork.", + }, + document_no: { type: "string", description: "The document's own invoice / delivery-note number." }, + doc_date: { type: "string", description: "Document date as YYYY-MM-DD, or empty if unreadable." }, + supplier_name: { type: "string", description: "Supplier name printed on the document." }, + po_refs: { + type: "array", + items: { type: "string" }, + description: "Every customer order / PO number printed, one per entry.", + }, + lines: { + type: "array", + description: "One entry per item line.", + items: { + type: "object", + properties: { + line_no: { type: "number", description: "1-based position on the document." }, + code: { type: "string", description: "Supplier stock code." }, + description: { type: "string" }, + quantity: { type: ["number", "null"], description: "Quantity supplied." }, + unit: { type: "string" }, + unit_price: { type: ["number", "null"], description: "Price for ONE unit." }, + line_total: { type: ["number", "null"] }, + to_follow: { type: ["number", "null"], description: "Back-order quantity, else null." }, + price_note: { type: "string", description: "e.g. \"printed 33.00 per 100\"." }, + }, + required: ["code", "description", "quantity", "unit", "unit_price"], + }, + }, + notes: { + type: "array", + items: { type: "string" }, + description: "Anything worth flagging (rotated, partly covered, handwritten changes).", + }, + }, + required: ["doc_type", "document_no", "doc_date", "supplier_name", "po_refs", "lines"], + }, +}; + +export type DocumentReadResult = Pick< + CollectionDocRead, + "doc_type" | "document_no" | "doc_date" | "supplier_name" | "po_refs" | "lines" | "notes" +> & { model: string }; + +const IMAGE_TYPES = ["image/jpeg", "image/png", "image/gif", "image/webp"] as const; +type ImageType = (typeof IMAGE_TYPES)[number]; + +const DOC_TYPES: readonly string[] = ["tax_invoice", "delivery_note", "other"]; + +function str(v: unknown): string { + return typeof v === "string" ? v.trim() : ""; +} + +function numOrNull(v: unknown): number | null { + if (typeof v === "number") return Number.isFinite(v) ? v : null; + return null; +} + +function strList(v: unknown): string[] { + if (!Array.isArray(v)) return []; + return v.map(str).filter((s) => s !== ""); +} + +/** Defensive parse of the tool input; never throws. */ +export function parseDocumentRead(raw: unknown): Omit { + const r = (raw && typeof raw === "object" ? raw : {}) as Record; + const docType = typeof r.doc_type === "string" && DOC_TYPES.includes(r.doc_type) ? r.doc_type : "other"; + const date = str(r.doc_date); + const rawLines = Array.isArray(r.lines) ? r.lines : []; + const lines: CollectionDocLine[] = rawLines.map((item, i) => { + const l = (item && typeof item === "object" ? item : {}) as Record; + const no = typeof l.line_no === "number" && Number.isInteger(l.line_no) && l.line_no > 0 ? l.line_no : i + 1; + return { + line_no: no, + code: str(l.code), + description: str(l.description), + quantity: numOrNull(l.quantity), + unit: str(l.unit), + unit_price: numOrNull(l.unit_price), + line_total: numOrNull(l.line_total), + to_follow: numOrNull(l.to_follow), + price_note: str(l.price_note), + }; + }); + return { + doc_type: docType as CollectionDocType, + document_no: str(r.document_no), + doc_date: /^\d{4}-\d{2}-\d{2}$/.test(date) ? date : "", + supplier_name: str(r.supplier_name), + po_refs: strList(r.po_refs), + lines, + notes: strList(r.notes), + }; +} + +function newAnthropic(aiConfig: { apiKey: string; baseUrl: string }): Anthropic { + return new Anthropic({ + apiKey: aiConfig.apiKey, + // Same base-URL handling as quote-assistant-ai / quotes-ai-processor. + ...(aiConfig.baseUrl ? { baseURL: aiConfig.baseUrl } : {}), + }); +} + +export async function readDocumentImage( + image: { base64: string; contentType: string }, + deps: { client?: Pick } = {}, +): Promise { + if (!(IMAGE_TYPES as readonly string[]).includes(image.contentType)) { + throw new Error(`Unsupported image type: ${image.contentType}`); + } + const aiConfig = await getActiveAiConfig(); + const client = deps.client ?? newAnthropic(aiConfig); + const settings = await getAiSettings(); + const override = (settings.collection_ai_prompt || "").trim(); + const system = override ? settings.collection_ai_prompt : DEFAULT_COLLECTION_AI_PROMPT; + const maxTokens = Math.max(aiConfig.maxTokens, 4096); + + const response = await client.messages.create({ + model: aiConfig.modelId, + max_tokens: maxTokens, + tools: [SUBMIT_DOCUMENT_READ_TOOL], + tool_choice: { type: "tool", name: TOOL_NAME }, + system, + messages: [ + { + role: "user", + content: [ + { + type: "image", + source: { type: "base64", media_type: image.contentType as ImageType, data: image.base64 }, + }, + { type: "text", text: "Read this document and call submit_document_read." }, + ], + }, + ], + }); + + if (response.stop_reason === "max_tokens") { + throw new Error("AI response was cut off (max_tokens)"); + } + const toolBlock = response.content.find((b) => b.type === "tool_use" && b.name === TOOL_NAME); + if (!toolBlock || toolBlock.type !== "tool_use") { + throw new Error("AI returned no submit_document_read tool call"); + } + return { ...parseDocumentRead(toolBlock.input), model: aiConfig.modelId }; +}