diff --git a/docs/n8n/po-email-intake.md b/docs/n8n/po-email-intake.md index 12c83af..1c286a4 100644 --- a/docs/n8n/po-email-intake.md +++ b/docs/n8n/po-email-intake.md @@ -169,7 +169,15 @@ nodes and set the mailbox on each. No rebuild is needed — the workflow logic i This is the one design decision worth understanding. -MEGA's finalisation step deliberately **never re-maps the site after OCR** — +**Update 2026-11-02:** after OCR, `process-batch` now checks the site against the PO's own line +cost codes (`src/lib/purchase-order-site.ts`). Every line starts with the project code +(`C26/09, MC-SEW` is The Hills). If every line names one other site in the app and nothing is +booked against the PO, it is moved there; lines naming two projects, or a code that is no site in +the app, are recorded as `raw_ocr_data.site_check.result = "mismatch"` and the filed site stays. +OCR's header project reference is still never used. Nellmapius Ext 25's code was corrected from +`001` to `C26/27` (the code on its POs) the same day. The history below is kept for context. + +Before that, MEGA's finalisation step deliberately **never re-mapped the site after OCR** — `src/app/api/purchase-orders/process-batch/route.ts` states it outright: *"`site` was set at upload by the person who uploaded the … and is NEVER touched here"*, because OCR only matches a real project code on roughly 8% of documents. The old @@ -180,7 +188,7 @@ real project code on roughly 8% of documents. The old So the site has to be known when the row is created, and the workflow reads it out of the email itself, in this order: -1. exact project code — `C26/09`, `C26/13`, `C26/14`, `001`, `HEAD` +1. exact project code — `C26/09`, `C26/13`, `C26/14`, `C26/27`, `HEAD` 2. project code appearing anywhere in subject + body preview + sender 3. site name — *The Hills*, *Cedar Gate*, *Munyaka 2*, *Nellmapius Ext 25*, *Mega Head Office* diff --git a/src/app/api/purchase-orders/process-batch/route.test.ts b/src/app/api/purchase-orders/process-batch/route.test.ts index 7ecf286..dca0431 100644 --- a/src/app/api/purchase-orders/process-batch/route.test.ts +++ b/src/app/api/purchase-orders/process-batch/route.test.ts @@ -7,6 +7,8 @@ const createMock = vi.fn(); const updateMock = vi.fn(); const deleteMock = vi.fn(); const supplierCreateMock = vi.fn(); +const sitesListMock = vi.fn(); +const deliveryNotesListMock = vi.fn(); const fakePb = { authStore: { token: "test-admin-token" }, @@ -26,6 +28,8 @@ const fakePb = { if (name === "suppliers") { return { getFullList: getFullListMock, create: supplierCreateMock }; } + if (name === "sites") return { getFullList: sitesListMock }; + if (name === "delivery_notes") return { getList: deliveryNotesListMock }; throw new Error(`unexpected collection in test: ${name}`); }, }; @@ -111,6 +115,74 @@ describe("POST /api/purchase-orders/process-batch", () => { deleteMock.mockReset(); supplierCreateMock.mockReset(); supplierCreateMock.mockResolvedValue({ id: "sup0000000001" }); + sitesListMock.mockReset(); + sitesListMock.mockResolvedValue([]); + deliveryNotesListMock.mockReset(); + deliveryNotesListMock.mockResolvedValue({ totalItems: 0, items: [] }); + }); + + describe("site from the PO's own cost codes", () => { + const SITES = [ + { id: "hills0000000001", project_code: "C26/09", name: "The Hills" }, + { id: "cedar0000000001", project_code: "C26/13", name: "Cedar Gate" }, + ]; + const coded = (...codes: string[]) => validExtraction("MEGCPO9482", { + items: codes.map((cost_code, i) => ({ cost_code, description: `Line ${i}`, qty: 1, rate: 10 })), + }); + beforeEach(() => { + getFullListMock.mockResolvedValue([]); + sitesListMock.mockResolvedValue(SITES); + }); + + it("moves the PO to the site every line's cost code names, and schedules deliveries there", async () => { + const previous = process.env.DN_STAGED_INGESTION_ENABLED; + process.env.DN_STAGED_INGESTION_ENABLED = "true"; + try { + getOneMock.mockResolvedValue({ id: ID, site: "hills0000000001", processing_status: "processing" }); + const res = await POST(postWithValidAuth({ purchase_order_id: ID, extracted: coded("C26/13, 0001", "C26/13, 0002") })); + expect(res.status).toBe(200); + const update = updateMock.mock.calls[0][1]; + expect(update.site).toBe("cedar0000000001"); + expect(update.raw_ocr_data.site_check).toEqual(expect.objectContaining({ result: "moved", from: "hills0000000001", site: "cedar0000000001", codes: ["C26/13"] })); + expect(queueWaitingDeliveries).toHaveBeenCalledWith(fakePb, expect.objectContaining({ site: "cedar0000000001" })); + } finally { if (previous === undefined) delete process.env.DN_STAGED_INGESTION_ENABLED; else process.env.DN_STAGED_INGESTION_ENABLED = previous; } + }); + + it("records a confirmed site without touching it", async () => { + getOneMock.mockResolvedValue({ id: ID, site: "hills0000000001", processing_status: "processing" }); + await POST(postWithValidAuth({ purchase_order_id: ID, extracted: coded("C26/09, MC-SEW") })); + const update = updateMock.mock.calls[0][1]; + expect(update.site).toBeUndefined(); + expect(update.raw_ocr_data.site_check.result).toBe("confirmed"); + }); + + it("flags lines naming two projects and keeps the filed site", async () => { + getOneMock.mockResolvedValue({ id: ID, site: "cedar0000000001", processing_status: "processing" }); + await POST(postWithValidAuth({ purchase_order_id: ID, extracted: coded("C26/01, 0001", "C26/09, 0001") })); + const update = updateMock.mock.calls[0][1]; + expect(update.site).toBeUndefined(); + expect(update.raw_ocr_data.site_check).toEqual(expect.objectContaining({ result: "mismatch", codes: ["C26/01", "C26/09"] })); + }); + + it("does not move a PO that already has delivery notes booked against it", async () => { + getOneMock.mockResolvedValue({ id: ID, site: "hills0000000001", processing_status: "processing" }); + deliveryNotesListMock.mockResolvedValue({ totalItems: 2, items: [{ id: "dn0000000000001" }] }); + await POST(postWithValidAuth({ purchase_order_id: ID, extracted: coded("C26/13, 0001") })); + const update = updateMock.mock.calls[0][1]; + expect(update.site).toBeUndefined(); + expect(update.raw_ocr_data.site_check).toEqual(expect.objectContaining({ result: "mismatch", reason: expect.stringContaining("delivery notes") })); + }); + + it("still fills the PO in when the site list cannot be read", async () => { + getOneMock.mockResolvedValue({ id: ID, site: "hills0000000001", processing_status: "processing" }); + sitesListMock.mockRejectedValue(new Error("pb down")); + const res = await POST(postWithValidAuth({ purchase_order_id: ID, extracted: coded("C26/13, 0001") })); + expect(res.status).toBe(200); + const update = updateMock.mock.calls[0][1]; + expect(update.site).toBeUndefined(); + expect(update.processing_status).toBe("processed"); + expect(update.raw_ocr_data.site_check).toEqual(expect.objectContaining({ result: "unchecked", reason: "site list unavailable" })); + }); }); it("schedules waiting deliveries after saving all PO items, without failing the upload if scheduling is interrupted", async () => { @@ -265,7 +337,7 @@ describe("POST /api/purchase-orders/process-batch", () => { expect(updateMock.mock.calls[0][0]).toBe("po0000000000001"); expect(update.purchase_order_number).toBe("MEGCPO9199"); expect(update.processing_status).toBe("processed"); - // Critical: the uploader's site choice must never be touched here. + // The cost code "C-1" names no project, so the uploader's site stands. expect(update.site).toBeUndefined(); expect(update.raw_ocr_data.extracted_project_ref).toBe("PRJ-A"); diff --git a/src/app/api/purchase-orders/process-batch/route.ts b/src/app/api/purchase-orders/process-batch/route.ts index 067eb1f..96737d0 100644 --- a/src/app/api/purchase-orders/process-batch/route.ts +++ b/src/app/api/purchase-orders/process-batch/route.ts @@ -8,6 +8,7 @@ import { purchaseOrderImportRefusal } from "@/lib/purchase-order-document-kind"; import { fetchPurchaseOrderDocument } from "@/lib/purchase-order-document"; import { readPoOrderDate } from "@/lib/purchase-order-date"; import { findPurchaseOrderNumberCandidates, findPurchaseOrderFileMatches, type PurchaseOrderNumberRecord } from "@/lib/purchase-order-duplicates"; +import { checkPurchaseOrderSite, type SiteCheckResult } from "@/lib/purchase-order-site"; export const dynamic = "force-dynamic"; export const runtime = "nodejs"; @@ -58,8 +59,9 @@ type ValidationResult = * * 1. Fill in: the extracted number is genuinely new. Update the * placeholder in place with the real number, supplier, dates, and - * line items. `site` is NEVER touched here — the uploader's choice - * is far more reliable than OCR's guess at a project reference. + * line items. `site` is moved only when every line's cost code names + * one other site in the app (src/lib/purchase-order-site.ts); OCR's + * header project reference is never used for it. * 2. Merge on dedup: the extracted number already belongs to a * different (already-processed) purchase_orders row. This upload * was a genuine duplicate — delete the placeholder (leaving it @@ -290,9 +292,14 @@ export async function POST(request: Request) { }, { status: 409 }); } - // 6. Fill in. `site` was set at upload by the person who uploaded the - // file and is intentionally absent below — it is NOT overwritten from - // the extraction. + // 6. Fill in. `site` was set at upload; it changes only when the PO's own + // line cost codes all name one other site and nothing is booked against it + // yet. Never fatal: if the check cannot run, the uploader's site stands. + const siteCheck = await checkSiteFromCostCodes(pb, purchaseOrderId, po.site ?? "", extracted.items); + const site = siteCheck.result === "moved" ? siteCheck.site : po.site; + if (siteCheck.result === "moved" || siteCheck.result === "mismatch") { + console.warn("[po-process] site_check", { purchase_order_id: purchaseOrderId, ...siteCheck }); + } // Reuse an exact normalized supplier match, or create the named supplier // automatically. An unreadable supplier remains unset for review. let supplierMatch; @@ -318,6 +325,7 @@ export async function POST(request: Request) { try { await pb.collection("purchase_orders").update(purchaseOrderId, { purchase_order_number: extracted.purchase_order_number, + ...(siteCheck.result === "moved" ? { site: siteCheck.site } : {}), ...(orderDate ? { order_date: `${orderDate} 00:00:00.000Z` } : {}), supplier: extracted.supplier ?? "", ...(supplierMatch.supplierId ? { supplier_ref: supplierMatch.supplierId } : {}), @@ -334,6 +342,7 @@ export async function POST(request: Request) { // lives with the other extraction output, not in a schema column // named project_code. extracted_project_ref: extracted.project_code ?? "", + site_check: { ...siteCheck, checked_at: new Date().toISOString() }, }, }); } catch (err) { @@ -379,7 +388,7 @@ export async function POST(request: Request) { } if (process.env.DN_STAGED_INGESTION_ENABLED === "true") { - try { await queueWaitingDeliveries(pb, {id:purchaseOrderId,site:po.site,purchase_order_number:extracted.purchase_order_number,processing_status:"processed",items_count:extracted.items.length}); } + try { await queueWaitingDeliveries(pb, {id:purchaseOrderId,site,purchase_order_number:extracted.purchase_order_number,processing_status:"processed",items_count:extracted.items.length}); } catch (error) { console.warn("[po-upload] delivery retry scheduling deferred to tick", {error:String(error)}); } } @@ -399,6 +408,40 @@ export async function POST(request: Request) { }); } +type SiteCheck = SiteCheckResult | { result: "unchecked"; reason: string }; + +async function checkSiteFromCostCodes( + pb: Awaited>, + purchaseOrderId: string, + filedSite: string, + items: ExtractedItem[], +): Promise { + let sites: { id: string; project_code?: string }[]; + try { + sites = await pb.collection("sites").getFullList({ fields: "id,project_code" }); + } catch (err) { + console.warn("[po-process] site list unavailable", { purchase_order_id: purchaseOrderId, err: String(err) }); + return { result: "unchecked", reason: "site list unavailable" }; + } + const check = checkPurchaseOrderSite(items, filedSite, sites); + if (check.result !== "moved") return check; + // A retried PO can already carry receipts on its current site; moving the + // order would leave them on the wrong one. A person decides those. + try { + const booked = await pb.collection("delivery_notes").getList(1, 1, { + filter: pb.filter("purchase_order = {:id}", { id: purchaseOrderId }), + fields: "id", + }); + if (booked.totalItems > 0) { + return { result: "mismatch", codes: check.codes, site: filedSite, reason: `cost codes name another site but ${booked.totalItems} delivery notes are booked here` }; + } + } catch (err) { + console.warn("[po-process] booked-delivery lookup failed", { purchase_order_id: purchaseOrderId, err: String(err) }); + return { result: "mismatch", codes: check.codes, site: filedSite, reason: "cost codes name another site; booked deliveries could not be checked" }; + } + return check; +} + function validateExtracted(input: unknown): ValidationResult { if (!input || typeof input !== "object") { return { ok: false, details: { _root: "must be an object" } }; diff --git a/src/lib/purchase-order-site.test.ts b/src/lib/purchase-order-site.test.ts new file mode 100644 index 0000000..95292e2 --- /dev/null +++ b/src/lib/purchase-order-site.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from "vitest"; +import { checkPurchaseOrderSite, projectCodesInCostCodes } from "./purchase-order-site"; + +const SITES = [ + { id: "hills0000000001", project_code: "C26/09", name: "The Hills" }, + { id: "cedar0000000001", project_code: "C26/13", name: "Cedar Gate" }, + { id: "nell00000000001", project_code: "C26/27", name: "Nellmapius Ext 25" }, + { id: "head00000000001", project_code: "HEAD", name: "Mega Head Office" }, +]; +const items = (...codes: string[]) => codes.map((cost_code) => ({ cost_code })); + +describe("projectCodesInCostCodes", () => { + it("reads the project code at the front of each cost code, once each", () => { + expect(projectCodesInCostCodes(items("C26/09, MC-SEW", "c26/09,0001", "C26/13 MC-ASP"))).toEqual(["C26/09", "C26/13"]); + }); + it("ignores lines with no project code", () => { + expect(projectCodesInCostCodes(items("", "19P900100D", "MC-SEW"))).toEqual([]); + }); +}); + +describe("checkPurchaseOrderSite", () => { + it("confirms the filed site when every line names it", () => { + expect(checkPurchaseOrderSite(items("C26/09, MC-SEW", "C26/09, MC-ASP"), "hills0000000001", SITES)) + .toEqual({ result: "confirmed", codes: ["C26/09"], site: "hills0000000001" }); + }); + it("moves the PO when every line names one other known site", () => { + expect(checkPurchaseOrderSite(items("C26/13, 0001", "C26/13, 0001"), "nell00000000001", SITES)) + .toEqual({ result: "moved", codes: ["C26/13"], site: "cedar0000000001", from: "nell00000000001" }); + }); + it("matches project codes regardless of case or spacing on the site", () => { + const sites = [{ id: "cedar0000000001", project_code: " c26/13 ", name: "Cedar Gate" }]; + expect(checkPurchaseOrderSite(items("C26/13, 0001"), "x00000000000001", sites).result).toBe("moved"); + }); + it("flags, and never moves, when the lines name more than one project", () => { + expect(checkPurchaseOrderSite(items("C26/01, 0001", "C26/09, 0001"), "cedar0000000001", SITES)) + .toEqual({ result: "mismatch", codes: ["C26/01", "C26/09"], site: "cedar0000000001", reason: "lines name more than one project: C26/01, C26/09" }); + }); + it("flags, and never moves, when a line's project code is not a site in the app", () => { + expect(checkPurchaseOrderSite(items("C25/42, MC-ASP"), "hills0000000001", SITES)) + .toEqual({ result: "mismatch", codes: ["C25/42"], site: "hills0000000001", reason: "no site has project code C25/42" }); + }); + it("leaves the site alone when no line carries a project code", () => { + expect(checkPurchaseOrderSite(items("", "19SALE750250"), "hills0000000001", SITES)) + .toEqual({ result: "no_codes", codes: [], site: "hills0000000001" }); + }); +}); diff --git a/src/lib/purchase-order-site.ts b/src/lib/purchase-order-site.ts new file mode 100644 index 0000000..458f630 --- /dev/null +++ b/src/lib/purchase-order-site.ts @@ -0,0 +1,48 @@ +/** + * Which site a purchase order belongs to, read from its own line cost codes. + * + * MEGA's POs carry the project code at the front of every line's cost code + * ("C26/09, MC-SEW" is The Hills). Across the live POs that code agreed with + * the filed site on every order whose code is a site in the app, which makes + * it far stronger evidence than the email subject or the uploader's pick. + * + * Only an unambiguous answer moves a PO: every line names the same project and + * that project is a site in the app. Anything else is flagged for a person and + * the filed site stays. + */ + +export type SiteCheckResult = + | { result: "confirmed"; codes: string[]; site: string } + | { result: "moved"; codes: string[]; site: string; from: string } + | { result: "mismatch"; codes: string[]; site: string; reason: string } + | { result: "no_codes"; codes: []; site: string }; + +type SiteRef = { id: string; project_code?: string | null }; + +const PROJECT_CODE = /\b([A-Z]\d{2}\/\d{2})\b/i; +const normalise = (code: string) => code.trim().toUpperCase(); + +export function projectCodesInCostCodes(items: ReadonlyArray<{ cost_code?: string | null }>): string[] { + const codes = new Set(); + for (const item of items) { + const match = PROJECT_CODE.exec(item.cost_code ?? ""); + if (match) codes.add(normalise(match[1])); + } + return [...codes]; +} + +export function checkPurchaseOrderSite( + items: ReadonlyArray<{ cost_code?: string | null }>, + filedSite: string, + sites: ReadonlyArray, +): SiteCheckResult { + const codes = projectCodesInCostCodes(items); + if (codes.length === 0) return { result: "no_codes", codes: [], site: filedSite }; + if (codes.length > 1) { + return { result: "mismatch", codes, site: filedSite, reason: `lines name more than one project: ${codes.join(", ")}` }; + } + const named = sites.find((site) => normalise(site.project_code ?? "") === codes[0]); + if (!named) return { result: "mismatch", codes, site: filedSite, reason: `no site has project code ${codes[0]}` }; + if (named.id === filedSite) return { result: "confirmed", codes, site: filedSite }; + return { result: "moved", codes, site: named.id, from: filedSite }; +}