diff --git a/scripts/pb-add-dn-staged-ingestion.py b/scripts/pb-add-dn-staged-ingestion.py index a3fc1ff..be14276 100644 --- a/scripts/pb-add-dn-staged-ingestion.py +++ b/scripts/pb-add-dn-staged-ingestion.py @@ -140,7 +140,18 @@ def system_fields(): ] +def supplier_identity_fields(): + """Optional supplier identifiers used by document entity resolution.""" + return [field("registration_number"), field("vat_number")] + + def self_test(): + identity_fields = supplier_identity_fields() + assert [item["name"] for item in identity_fields] == [ + "registration_number", "vat_number", + ] + assert all(item["required"] is False for item in identity_fields) + old = [ "CREATE INDEX `unrelated` ON `x` (`value`)", "CREATE UNIQUE INDEX `idx_delivery_notes_batch_note` ON `delivery_notes` (`batch_note`)", @@ -194,6 +205,10 @@ def main(): supplier_id = suppliers["id"] delivery_notes_id = delivery_notes["id"] + # Supplier identity evidence is optional: existing rows remain valid and + # operators can backfill identifiers as documents make them available. + add_fields(token, "suppliers", supplier_identity_fields()) + source_id = create_collection( token, "dn_batch_sources", system_fields() + [ diff --git a/src/lib/dn-pipeline/resolve-entities.test.ts b/src/lib/dn-pipeline/resolve-entities.test.ts index f4bb8c6..edfc353 100644 --- a/src/lib/dn-pipeline/resolve-entities.test.ts +++ b/src/lib/dn-pipeline/resolve-entities.test.ts @@ -117,4 +117,24 @@ describe("resolveEntities", () => { expect(result.candidates).toHaveLength(2); expect(result.candidates[0].evidence.margin).toBe(0); }); + + it("keeps below-threshold fuzzy candidates as review evidence", () => { + const result = resolveEntities(input({ + poNumber: "MEGCPO", + purchaseOrders: [ + { ...purchaseOrders[0], purchase_order_number: "MEGCPO9999" }, + ], + })); + + expect(result.purchaseOrderId).toBeNull(); + expect(result.autoResolved).toBe(false); + expect(result.method).toBe("po_fuzzy"); + expect(result.candidates).toHaveLength(1); + expect(result.candidates[0].evidence).toMatchObject({ + method: "po_fuzzy", + threshold: 0.75, + rank: 1, + }); + expect(result.candidates[0].evidence.score).toBeLessThan(0.75); + }); }); diff --git a/src/lib/dn-pipeline/resolve-entities.ts b/src/lib/dn-pipeline/resolve-entities.ts index 6ceec8f..8ee682d 100644 --- a/src/lib/dn-pipeline/resolve-entities.ts +++ b/src/lib/dn-pipeline/resolve-entities.ts @@ -1,5 +1,5 @@ import { normalizeSupplierName } from "../purchase-order-schema"; -import { normalisePoNumber, type PoCandidate, type PoMatch } from "../po-matching"; +import { normalisePoNumber, rankPoMatches, type PoCandidate, type PoMatch } from "../po-matching"; export interface EntityPurchaseOrder { id: string; @@ -20,9 +20,10 @@ export interface EntitySupplier { id: string; name: string; registration?: string | null; + registration_number?: string | null; + vat_number?: string | null; registration_no?: string | null; company_registration?: string | null; - vat_number?: string | null; vat_no?: string | null; tax_number?: string | null; [key: string]: unknown; @@ -331,8 +332,8 @@ function rankedSuggestions( purchase_order_number: po.purchase_order_number, supplier: po.supplier ?? undefined, })); - const ranked = localRankPoMatches(target, poCandidates); - const relevant = ranked.filter((match) => match.score >= 0.75); + const ranked = rankPoMatches(target, poCandidates); + const relevant = ranked.slice(0, 20); const margin = relevant.length > 1 ? relevant[0].score - relevant[1].score : relevant[0]?.score ?? 0; return relevant.map((match, index) => ({ match, @@ -350,34 +351,3 @@ function rankedSuggestions( }, })); } - -function localRankPoMatches(target: unknown, candidates: PoCandidate[]): PoMatch[] { - const normalisedTarget = normalisePoNumber(target); - if (normalisedTarget.length < 4) return []; - return candidates - .map((candidate) => ({ - ...candidate, - score: similarity(normalisedTarget, normalisePoNumber(candidate.purchase_order_number)), - })) - .filter((candidate) => normalisePoNumber(candidate.purchase_order_number).length > 0) - .sort((a, b) => b.score - a.score || a.id.localeCompare(b.id)); -} - -function similarity(a: string, b: string): number { - if (a === b) return 1; - if (!a.length || !b.length) return 0; - const previous = Array.from({ length: b.length + 1 }, (_, index) => index); - let prior = previous; - for (let i = 1; i <= a.length; i++) { - const current = [i]; - for (let j = 1; j <= b.length; j++) { - current[j] = Math.min( - current[j - 1] + 1, - prior[j] + 1, - prior[j - 1] + (a[i - 1] === b[j - 1] ? 0 : 1), - ); - } - prior = current; - } - return 1 - prior[b.length] / Math.max(a.length, b.length); -} diff --git a/src/lib/dn-pipeline/run.test.ts b/src/lib/dn-pipeline/run.test.ts index bf65a7b..c723066 100644 --- a/src/lib/dn-pipeline/run.test.ts +++ b/src/lib/dn-pipeline/run.test.ts @@ -32,12 +32,16 @@ vi.mock("./checks", async (importOriginal) => { }; }); vi.mock("./segment", () => ({ segmentPages: vi.fn() })); -vi.mock("../po-matching", () => ({ - findBestPoMatch: vi.fn(), - normalisePoNumber: vi.fn((s: unknown) => - typeof s === "string" ? s.replace(/[^a-z0-9]/gi, "").toLowerCase() : "", - ), -})); +vi.mock("../po-matching", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + findBestPoMatch: vi.fn(), + normalisePoNumber: vi.fn((s: unknown) => + typeof s === "string" ? s.replace(/[^a-z0-9]/gi, "").toLowerCase() : "", + ), + }; +}); // import AFTER mocks so run.ts wires through the fakes above. import { extractPage } from "./extract"; @@ -103,6 +107,10 @@ function makePb() { return { getFullList: dnItemAliasesGetFullList, }; + case "suppliers": + return { getFullList: suppliersGetFullList }; + case "supplier_document_aliases": + return { getFullList: supplierDocumentAliasesGetFullList }; // qty_plausible needs what the ledger already holds per PO line. case "delivery_note_items": return { getFullList: deliveryNoteItemsGetFullList }; @@ -112,6 +120,7 @@ function makePb() { }); return { collection, + filter: pbFilter, authStore: { token: "test-token" }, } as unknown as PocketBase; } @@ -126,7 +135,10 @@ const dnBatchNotesGetFullList = vi.fn(); const dnBatchNotesCreate = vi.fn(); const dnBatchNotesUpdate = vi.fn(); const dnItemAliasesGetFullList = vi.fn(); +const suppliersGetFullList = vi.fn(); +const supplierDocumentAliasesGetFullList = vi.fn(); const deliveryNoteItemsGetFullList = vi.fn(); +const pbFilter = vi.fn((raw: string, params: Record) => `${raw}|${JSON.stringify(params)}`); beforeEach(() => { vi.stubEnv("NEXT_PUBLIC_POCKETBASE_URL", "http://pb.test"); @@ -143,7 +155,10 @@ beforeEach(() => { dnBatchNotesCreate, dnBatchNotesUpdate, dnItemAliasesGetFullList, + suppliersGetFullList, + supplierDocumentAliasesGetFullList, deliveryNoteItemsGetFullList, + pbFilter, mExtractPage, mSplitPages, mExtractPageRange, @@ -216,6 +231,8 @@ beforeEach(() => { { id: "poi1", description: "G6 Sub Base", unit: "ton" }, ]); dnItemAliasesGetFullList.mockResolvedValue([]); + suppliersGetFullList.mockResolvedValue([]); + supplierDocumentAliasesGetFullList.mockResolvedValue([]); deliveryNoteItemsGetFullList.mockResolvedValue([]); dnBatchNotesGetFullList.mockResolvedValue([]); // No existing → create. dnBatchNotesCreate.mockResolvedValue({ id: "bn1" }); @@ -224,6 +241,7 @@ beforeEach(() => { mExtractPageRange.mockResolvedValue(Buffer.from("%PDF-mock-range")); mPromoteNote.mockResolvedValue({ deliveryNoteId: "dn1", itemsCreated: 1 }); mFindBestPoMatch.mockReturnValue(null); + pbFilter.mockImplementation((raw: string, params: Record) => `${raw}|${JSON.stringify(params)}`); }); afterEach(() => { @@ -432,6 +450,47 @@ describe("runBatch — happy path", () => { }); describe("runBatch — per-note checks", () => { + it("loads POs through a bound confirmed-site filter and real PO fields", async () => { + await runBatch(makePb(), "batch1"); + + expect(pbFilter).toHaveBeenCalledWith("site = {:site}", { site: "site1" }); + const options = purchaseOrdersGetFullList.mock.calls[0][0] as { filter: string; fields: string }; + expect(options.filter).toBe('site = {:site}|{"site":"site1"}'); + expect(options.fields.split(",")).not.toContain("status"); + expect(options.fields.split(",")).toContain("processing_status"); + }); + + it("resolves supplier evidence for a note with no printed PO", async () => { + const richNote = { + ...makeNote({ po_number_raw: null }), + supplier_name_raw: "R-Mix Haulage", + supplier_registration_raw: null, + }; + mSegmentPages.mockReturnValueOnce({ + notes: [richNote as NoteSegment], + unassigned: [], + needsSubsplit: [], + }); + suppliersGetFullList.mockResolvedValueOnce([{ id: "sup-rmix", name: "RAUMIX (PTY) LTD" }]); + supplierDocumentAliasesGetFullList.mockResolvedValueOnce([ + { + id: "alias-rmix", + supplier: "sup-rmix", + raw_value: "R-Mix Haulage", + normalised_key: "R-MIX HAULAGE", + approved_by: "operator-1", + }, + ]); + mRunNoteChecks.mockReturnValueOnce([{ id: "po_resolved", passed: false, reason: "no PO" }]); + + await runBatch(makePb(), "batch1"); + + expect(suppliersGetFullList).toHaveBeenCalledWith(expect.objectContaining({ fields: "id,name,registration_number,vat_number" })); + expect(supplierDocumentAliasesGetFullList).toHaveBeenCalledTimes(1); + const form = await readForm(dnBatchNotesCreate.mock.calls[0][0]); + expect(form.supplier).toBe("sup-rmix"); + expect(form.match_method).toBe("supplier_alias"); + }); it("uses the unique site-scoped exact normalised match", async () => { purchaseOrdersGetFullList.mockResolvedValueOnce([ { id: "po-exact", purchase_order_number: "MEGCPO9165" }, diff --git a/src/lib/dn-pipeline/run.ts b/src/lib/dn-pipeline/run.ts index 28411af..4ccde6f 100644 --- a/src/lib/dn-pipeline/run.ts +++ b/src/lib/dn-pipeline/run.ts @@ -74,7 +74,11 @@ export async function runBatch(pb: PocketBase, batchId: string): Promise [])) as unknown as AliasRow[]; - const needsEntityResolution = seg.notes.some((note) => Boolean(normalisePoNumber(note.po_number_raw ?? ""))); + const needsEntityResolution = seg.notes.some((note) => + Boolean(normalisePoNumber(note.po_number_raw ?? "")) || + Boolean(note.supplier_name_raw?.trim()) || + Boolean(note.supplier_registration_raw?.trim()), + ); const purchaseOrders = needsEntityResolution ? await loadSitePurchaseOrders(pb, batch.site) : []; const suppliers = needsEntityResolution ? await loadSuppliers(pb) : []; const supplierAliases = needsEntityResolution ? await loadSupplierAliases(pb) : []; @@ -83,15 +87,11 @@ export async function runBatch(pb: PocketBase, batchId: string): Promise { try { return (await pb.collection("suppliers").getFullList({ - fields: "id,name", + fields: "id,name,registration_number,vat_number", })) as unknown as EntitySupplier[]; } catch (err) { // Supplier records were introduced after the legacy runner. POs still diff --git a/src/lib/dn-pipeline/segment.test.ts b/src/lib/dn-pipeline/segment.test.ts index b8dfc5f..389c25b 100644 --- a/src/lib/dn-pipeline/segment.test.ts +++ b/src/lib/dn-pipeline/segment.test.ts @@ -50,6 +50,39 @@ function assertTotalCoverage(pages: PageRead[], seg: ReturnType { + it("preserves supplier identity evidence from the note header", () => { + const result = segmentPages([ + page(1, { + doc_number: "DN-SUPPLIER", + supplier_name_raw: "R-Mix Haulage", + supplier_registration_raw: "2010/123456/07", + }), + ]); + + expect(result.notes[0]).toMatchObject({ + supplier_name_raw: "R-Mix Haulage", + supplier_registration_raw: "2010/123456/07", + }); + }); + + it("fills missing supplier identity evidence from a continuation page", () => { + const result = segmentPages([ + page(1, { doc_number: "DN-SUPPLIER", supplier_name_raw: null, supplier_registration_raw: null }), + page(2, { + doc_number: "", + has_header: false, + is_continuation: true, + supplier_name_raw: "R-Mix Haulage", + supplier_registration_raw: "2010/123456/07", + }), + ]); + + expect(result.notes[0]).toMatchObject({ + supplier_name_raw: "R-Mix Haulage", + supplier_registration_raw: "2010/123456/07", + }); + }); + it("treats four single-page notes as four notes", () => { const pages = [ page(1, { doc_number: "DQ0010107", quantity: 29960, unit: "kg" }), diff --git a/src/lib/dn-pipeline/segment.ts b/src/lib/dn-pipeline/segment.ts index 317f330..d6e5500 100644 --- a/src/lib/dn-pipeline/segment.ts +++ b/src/lib/dn-pipeline/segment.ts @@ -75,6 +75,8 @@ function startNote(index: number, number: string, read: PageRead): NoteSegment { // overwritten by a later one — a continuation page repeating a // half-legible header must not clobber a clean read. po_number_raw: read.po_number_raw, + supplier_name_raw: read.supplier_name_raw, + supplier_registration_raw: read.supplier_registration_raw, customer_vat_no: read.customer_vat_no, date: read.date, received_by_name: read.received_by_name, @@ -94,6 +96,8 @@ function appendPage(note: NoteSegment, read: PageRead): void { note.items.push(...read.items); // Fill header fields only where the first page left a gap. note.po_number_raw ??= read.po_number_raw; + note.supplier_name_raw ??= read.supplier_name_raw; + note.supplier_registration_raw ??= read.supplier_registration_raw; note.customer_vat_no ??= read.customer_vat_no; note.date ??= read.date; note.received_by_name ??= read.received_by_name; diff --git a/src/lib/dn-pipeline/types.ts b/src/lib/dn-pipeline/types.ts index 991d6f4..e36f159 100644 --- a/src/lib/dn-pipeline/types.ts +++ b/src/lib/dn-pipeline/types.ts @@ -85,6 +85,9 @@ export interface NoteSegment { doc_type: "delivery_note" | "invoice" | "collection_note" | "unknown"; doc_number_alts: string[]; po_number_raw: string | null; + /** Supplier identity evidence carried forward from the first page. */ + supplier_name_raw?: string | null; + supplier_registration_raw?: string | null; customer_vat_no: string | null; date: string | null; received_by_name: string | null; diff --git a/src/lib/pb-types.ts b/src/lib/pb-types.ts index 474225f..097714b 100644 --- a/src/lib/pb-types.ts +++ b/src/lib/pb-types.ts @@ -37,6 +37,9 @@ export interface SupplierRecord extends RecordModel { /** Stored pre-normalized (trim, collapse whitespace, uppercase) — see * normalizeSupplierName in purchase-order-schema.ts. */ name: string; + /** Optional identifiers printed on supplier documents. */ + registration_number?: string; + vat_number?: string; } /**