From 11635d9f8f055a4c7b5daa34a0c1837b0ae3d571 Mon Sep 17 00:00:00 2001 From: DEls Date: Thu, 17 Sep 2026 11:57:47 +0200 Subject: [PATCH 1/9] feat(units): add unit alias normalisation --- src/lib/units.test.ts | 57 +++++++++++++++++++++++++++++++++++++++++++ src/lib/units.ts | 47 +++++++++++++++++++++++++++++++++++ 2 files changed, 104 insertions(+) create mode 100644 src/lib/units.test.ts create mode 100644 src/lib/units.ts diff --git a/src/lib/units.test.ts b/src/lib/units.test.ts new file mode 100644 index 0000000..3cfe324 --- /dev/null +++ b/src/lib/units.test.ts @@ -0,0 +1,57 @@ +import { describe, it, expect } from "vitest"; +import { normaliseUnit } from "./units"; + +describe("normaliseUnit", () => { + it("returns null for absent input", () => { + expect(normaliseUnit(null)).toBeNull(); + expect(normaliseUnit(undefined)).toBeNull(); + expect(normaliseUnit("")).toBeNull(); + expect(normaliseUnit(" ")).toBeNull(); + }); + + it("maps mass aliases", () => { + for (const a of ["kg", "KG", " Kgs ", "kilo", "kilos", "kilogram", "kilograms"]) { + expect(normaliseUnit(a)).toBe("kg"); + } + for (const a of ["t", "T", "ton", "tons", "tonne", "tonnes", "mt", "metric ton"]) { + expect(normaliseUnit(a)).toBe("ton"); + } + }); + + it("maps volume and area aliases", () => { + for (const a of ["m3", "m³", "cube", "cubes", "cubic metre", "cubic meter"]) { + expect(normaliseUnit(a)).toBe("m3"); + } + for (const a of ["m2", "m²", "square metre", "square meter"]) { + expect(normaliseUnit(a)).toBe("m2"); + } + for (const a of ["l", "litre", "litres", "liter", "liters"]) { + expect(normaliseUnit(a)).toBe("l"); + } + }); + + it("maps count and length aliases", () => { + for (const a of ["ea", "each", "unit", "units", "no", "nr", "item", "items"]) { + expect(normaliseUnit(a)).toBe("ea"); + } + for (const a of ["m", "metre", "meter", "metres", "meters", "lm", "linear metre"]) { + expect(normaliseUnit(a)).toBe("m"); + } + for (const a of ["bag", "bags", "pocket", "pockets"]) { + expect(normaliseUnit(a)).toBe("bag"); + } + for (const a of ["load", "loads", "trip", "trips"]) { + expect(normaliseUnit(a)).toBe("load"); + } + }); + + it("strips trailing punctuation the OCR leaves behind", () => { + expect(normaliseUnit("kg.")).toBe("kg"); + expect(normaliseUnit("(ton)")).toBe("ton"); + }); + + it("returns null for an unrecognised unit rather than guessing", () => { + expect(normaliseUnit("furlong")).toBeNull(); + expect(normaliseUnit("??")).toBeNull(); + }); +}); diff --git a/src/lib/units.ts b/src/lib/units.ts new file mode 100644 index 0000000..e6c0f6c --- /dev/null +++ b/src/lib/units.ts @@ -0,0 +1,47 @@ +/** + * Unit normalisation and conversion for delivery note quantities. + * + * Delivery notes are weighbridge tickets quoting kg; the purchase order + * lines they belong to are priced per ton. Writing the slip's number + * against a ton-priced line inflates received value 1000x, which is + * exactly what happened before this module existed. + * + * The conversion table is deliberately closed. An unrecognised unit or + * an unlisted pair returns a failure — callers must surface that as + * work for a human, never fall back to "assume they match". + */ + +const ALIASES: Record = { + kg: ["kg", "kgs", "kilo", "kilos", "kilogram", "kilograms"], + ton: ["t", "ton", "tons", "tonne", "tonnes", "mt", "metric ton"], + m3: ["m3", "m³", "cube", "cubes", "cubic metre", "cubic meter"], + m2: ["m2", "m²", "square metre", "square meter"], + m: ["m", "metre", "meter", "metres", "meters", "lm", "linear metre"], + ea: ["ea", "each", "unit", "units", "no", "nr", "item", "items"], + bag: ["bag", "bags", "pocket", "pockets"], + l: ["l", "litre", "litres", "liter", "liters"], + load: ["load", "loads", "trip", "trips"], +}; + +const LOOKUP: Map = new Map(); +for (const [canonical, aliases] of Object.entries(ALIASES)) { + for (const alias of aliases) LOOKUP.set(alias, canonical); +} + +/** + * Canonical token for a unit as written on a slip or PO line, or null + * when we do not recognise it. Null is a real answer, not an error — + * callers must treat it as "cannot convert". + */ +export function normaliseUnit(raw: string | null | undefined): string | null { + if (typeof raw !== "string") return null; + // Keep the superscript characters in m² / m³; strip the punctuation + // OCR sprinkles around units ("kg.", "(ton)"). + const cleaned = raw + .toLowerCase() + .replace(/[^a-z0-9²³ ]/g, " ") + .replace(/\s+/g, " ") + .trim(); + if (!cleaned) return null; + return LOOKUP.get(cleaned) ?? null; +} From 2881db22d932c39c058ae3f5e9187b103df9dc7d Mon Sep 17 00:00:00 2001 From: DEls Date: Thu, 17 Sep 2026 12:04:19 +0200 Subject: [PATCH 2/9] feat(units): add closed-table quantity conversion --- src/lib/units.test.ts | 58 +++++++++++++++++++++++++++++++++++++++++++ src/lib/units.ts | 42 +++++++++++++++++++++++++++++++ 2 files changed, 100 insertions(+) diff --git a/src/lib/units.test.ts b/src/lib/units.test.ts index 3cfe324..71acdf1 100644 --- a/src/lib/units.test.ts +++ b/src/lib/units.test.ts @@ -55,3 +55,61 @@ describe("normaliseUnit", () => { expect(normaliseUnit("??")).toBeNull(); }); }); + +import { convertQty } from "./units"; + +describe("convertQty", () => { + it("passes through when both units are the same", () => { + expect(convertQty(29960, "kg", "kg")).toEqual({ + ok: true, + qty: 29960, + converted: false, + }); + }); + + it("converts kg to ton", () => { + expect(convertQty(29960, "kg", "ton")).toEqual({ + ok: true, + qty: 29.96, + converted: true, + }); + }); + + it("converts ton to kg", () => { + expect(convertQty(29.96, "ton", "kg")).toEqual({ + ok: true, + qty: 29960, + converted: true, + }); + }); + + it("converts litres and cubic metres both ways", () => { + expect(convertQty(2500, "l", "m3")).toEqual({ ok: true, qty: 2.5, converted: true }); + expect(convertQty(2.5, "m3", "l")).toEqual({ ok: true, qty: 2500, converted: true }); + }); + + it("rounds to four decimal places", () => { + expect(convertQty(1, "kg", "ton")).toEqual({ ok: true, qty: 0.001, converted: true }); + expect(convertQty(12345, "kg", "ton")).toEqual({ ok: true, qty: 12.345, converted: true }); + }); + + it("fails on an unknown unit rather than assuming a match", () => { + expect(convertQty(10, null, "ton")).toEqual({ ok: false, reason: "unknown_unit" }); + expect(convertQty(10, "kg", null)).toEqual({ ok: false, reason: "unknown_unit" }); + expect(convertQty(10, null, null)).toEqual({ ok: false, reason: "unknown_unit" }); + }); + + it("fails on a pair that is not in the table", () => { + expect(convertQty(10, "kg", "m3")).toEqual({ ok: false, reason: "incompatible_units" }); + expect(convertQty(10, "ea", "bag")).toEqual({ ok: false, reason: "incompatible_units" }); + expect(convertQty(10, "m2", "m")).toEqual({ ok: false, reason: "incompatible_units" }); + }); + + it("accepts raw aliases, not just canonical tokens", () => { + expect(convertQty(1000, "Kgs", "tonnes")).toEqual({ + ok: true, + qty: 1, + converted: true, + }); + }); +}); diff --git a/src/lib/units.ts b/src/lib/units.ts index e6c0f6c..e22c337 100644 --- a/src/lib/units.ts +++ b/src/lib/units.ts @@ -45,3 +45,45 @@ export function normaliseUnit(raw: string | null | undefined): string | null { if (!cleaned) return null; return LOOKUP.get(cleaned) ?? null; } + +export type UnitConversion = + | { ok: true; qty: number; converted: boolean } + | { ok: false; reason: "unknown_unit" | "incompatible_units" }; + +/** + * The complete set of conversions. Anything absent here fails. + * Keyed ">>" on canonical tokens. + */ +const FACTORS: Record = { + "kg>>>ton": 1 / 1000, + "ton>>>kg": 1000, + "l>>>m3": 1 / 1000, + "m3>>>l": 1000, +}; + +/** + * Convert `qty` from one unit into another. Accepts raw slip/PO text on + * either side — it normalises both before looking anything up. + * + * A null on either side is `unknown_unit`, never "assume they match": + * assuming is what made every received value 1000x too large. + */ +export function convertQty( + qty: number, + fromUnit: string | null, + toUnit: string | null, +): UnitConversion { + const from = normaliseUnit(fromUnit); + const to = normaliseUnit(toUnit); + if (from === null || to === null) return { ok: false, reason: "unknown_unit" }; + if (from === to) return { ok: true, qty, converted: false }; + + const factor = FACTORS[`${from}>>>${to}`]; + if (factor === undefined) return { ok: false, reason: "incompatible_units" }; + + return { + ok: true, + qty: Math.round(qty * factor * 10000) / 10000, + converted: true, + }; +} From 1a30f6a62126d3bf96223457808a4d551fd87099 Mon Sep 17 00:00:00 2001 From: DEls Date: Thu, 17 Sep 2026 12:07:59 +0200 Subject: [PATCH 3/9] feat(delivery-notes): add qty_raw/unit_raw/unit_converted to line items --- scripts/pb-add-dn-item-units.py | 66 +++++++++++++++++++++++++++++++++ src/lib/pb-types.ts | 7 ++++ 2 files changed, 73 insertions(+) create mode 100644 scripts/pb-add-dn-item-units.py diff --git a/scripts/pb-add-dn-item-units.py b/scripts/pb-add-dn-item-units.py new file mode 100644 index 0000000..b0be48a --- /dev/null +++ b/scripts/pb-add-dn-item-units.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +""" +Adds qty_raw / unit_raw / unit_converted to the delivery_note_items +collection. + +qty_received keeps its name but changes meaning: it becomes strictly +"in the purchase order line's unit". qty_raw and unit_raw preserve what +the slip actually said, so the audit trail survives the conversion and +the UI can show both. + +Idempotent — re-running on a collection that already has the fields is +a no-op. + +Credentials come from .env.local via pb_env. +""" +import sys + +from pb_env import PB, http, superuser_token + + +def get_collection(token: str, name: str): + code, resp = http("GET", f"/api/collections/{name}", token=token) + if code != 200: + sys.exit(f"GET collection {name} failed: {code} {resp}") + return resp + + +def main() -> None: + token = superuser_token() + print(f"\n→ delivery_note_items on {PB}") + coll = get_collection(token, "delivery_note_items") + fields = list(coll.get("fields", [])) + existing = {f.get("name") for f in fields} + + wanted = [ + {"name": "qty_raw", "type": "number", "required": False}, + {"name": "unit_raw", "type": "text", "required": False}, + {"name": "unit_converted", "type": "bool", "required": False}, + ] + + additions = [] + for want in wanted: + if want["name"] in existing: + print(f" • {want['name']:15s} already exists — skipping") + else: + additions.append(want) + + if not additions: + print(" • no changes needed") + return + + code, resp = http( + "PATCH", + f"/api/collections/{coll['id']}", + {"fields": fields + additions}, + token, + ) + if code != 200: + sys.exit(f"PATCH failed: {code} {resp}") + for a in additions: + print(f" ✓ added {a['name']}") + + +if __name__ == "__main__": + main() + print("\nDone.") diff --git a/src/lib/pb-types.ts b/src/lib/pb-types.ts index 7cd31ed..10619f8 100644 --- a/src/lib/pb-types.ts +++ b/src/lib/pb-types.ts @@ -321,7 +321,14 @@ export interface DeliveryNoteItemRecord extends RecordModel { delivery_note: string; /** Optional — empty when OCR could not match the line to a PO item. */ purchase_order_item: string; + /** In the matched PO line's unit. Zero when the line is unmatched or unconvertible. */ qty_received: number; + /** Verbatim quantity from the slip, before any conversion. */ + qty_raw: number; + /** Verbatim unit from the slip, e.g. "kg". Empty when the slip showed none. */ + unit_raw: string; + /** True when qty_received differs from qty_raw because a conversion was applied. */ + unit_converted: boolean; condition_notes: string; expand?: { delivery_note?: DeliveryNoteRecord; From 401fa51ff9b7a4a99354915be04360138089df4e Mon Sep 17 00:00:00 2001 From: DEls Date: Thu, 17 Sep 2026 12:11:40 +0200 Subject: [PATCH 4/9] feat(delivery-notes): build line item rows with unit conversion Filters once so matched PO item ids cannot drift when a zero-quantity line is dropped, and converts the slip quantity into the PO line's unit. An unconvertible line contributes zero rather than entering the maths at the wrong magnitude. --- src/lib/dn-item-row.test.ts | 134 ++++++++++++++++++++++++++++++ src/lib/dn-item-row.ts | 159 ++++++++++++++++++++++++++++++++++++ 2 files changed, 293 insertions(+) create mode 100644 src/lib/dn-item-row.test.ts create mode 100644 src/lib/dn-item-row.ts diff --git a/src/lib/dn-item-row.test.ts b/src/lib/dn-item-row.test.ts new file mode 100644 index 0000000..3e7fbc9 --- /dev/null +++ b/src/lib/dn-item-row.test.ts @@ -0,0 +1,134 @@ +import { describe, it, expect } from "vitest"; +import { buildDeliveryNoteItemRows } from "./dn-item-row"; + +const PO_ITEMS = [ + { id: "aaaaaaaaaaaaaaa", unit: "ton" }, + { id: "bbbbbbbbbbbbbbb", unit: "ton" }, + { id: "ccccccccccccccc", unit: "ea" }, +]; + +describe("buildDeliveryNoteItemRows", () => { + it("converts the slip quantity into the PO line's unit", () => { + const { rows } = buildDeliveryNoteItemRows({ + items: [ + { description: "G6 Sub Base", qty: 29960, unit: "kg", matched_po_item_id: "aaaaaaaaaaaaaaa" }, + ], + poItems: PO_ITEMS, + confidence: 0.95, + }); + expect(rows).toHaveLength(1); + expect(rows[0].qty_received).toBe(29.96); + expect(rows[0].qty_raw).toBe(29960); + expect(rows[0].unit_raw).toBe("kg"); + expect(rows[0].unit_converted).toBe(true); + expect(rows[0].purchase_order_item).toBe("aaaaaaaaaaaaaaa"); + }); + + it("does not shift matched ids when an item is dropped for zero quantity", () => { + const { rows } = buildDeliveryNoteItemRows({ + items: [ + { description: "first", qty: 5, unit: "ton", matched_po_item_id: "aaaaaaaaaaaaaaa" }, + { description: "dropped", qty: 0, unit: "ton", matched_po_item_id: "bbbbbbbbbbbbbbb" }, + { description: "third", qty: 7, unit: "ea", matched_po_item_id: "ccccccccccccccc" }, + ], + poItems: PO_ITEMS, + confidence: 0.9, + }); + expect(rows).toHaveLength(2); + expect(rows[0].purchase_order_item).toBe("aaaaaaaaaaaaaaa"); + // Before the fix this was "bbbbbbbbbbbbbbb" — the dropped item's id. + expect(rows[1].purchase_order_item).toBe("ccccccccccccccc"); + expect(rows[1].qty_received).toBe(7); + }); + + it("zeroes the quantity and drops the link when units cannot convert", () => { + const { rows } = buildDeliveryNoteItemRows({ + items: [ + { description: "Sand", qty: 12, unit: "m3", matched_po_item_id: "aaaaaaaaaaaaaaa" }, + ], + poItems: PO_ITEMS, + confidence: 0.9, + }); + expect(rows[0].qty_received).toBe(0); + expect(rows[0].qty_raw).toBe(12); + expect(rows[0].unit_raw).toBe("m3"); + expect(rows[0].purchase_order_item).toBeNull(); + expect(rows[0].condition_notes).toContain("incompatible_units"); + }); + + it("zeroes the quantity when the slip showed no unit", () => { + const { rows } = buildDeliveryNoteItemRows({ + items: [{ description: "Stone", qty: 3, matched_po_item_id: "aaaaaaaaaaaaaaa" }], + poItems: PO_ITEMS, + confidence: 0.9, + }); + expect(rows[0].qty_received).toBe(0); + expect(rows[0].qty_raw).toBe(3); + expect(rows[0].unit_raw).toBe(""); + expect(rows[0].purchase_order_item).toBeNull(); + expect(rows[0].condition_notes).toContain("unknown_unit"); + }); + + it("keeps an unmatched line at zero with no link", () => { + const { rows } = buildDeliveryNoteItemRows({ + items: [{ description: "Mystery", qty: 4, unit: "kg" }], + poItems: PO_ITEMS, + confidence: 0.6, + }); + expect(rows[0].purchase_order_item).toBeNull(); + expect(rows[0].qty_received).toBe(0); + expect(rows[0].qty_raw).toBe(4); + expect(rows[0].unit_raw).toBe("kg"); + }); + + it("drops a matched id that is not on the PO when the PO items are known", () => { + const { rows, unverifiable } = buildDeliveryNoteItemRows({ + items: [{ description: "X", qty: 1, unit: "ton", matched_po_item_id: "zzzzzzzzzzzzzzz" }], + poItems: PO_ITEMS, + confidence: 0.9, + }); + expect(rows[0].purchase_order_item).toBeNull(); + expect(unverifiable).toBe(false); + }); + + it("trusts a matched id when the PO items could not be fetched, and flags it", () => { + const { rows, unverifiable } = buildDeliveryNoteItemRows({ + items: [{ description: "X", qty: 1, unit: "ton", matched_po_item_id: "zzzzzzzzzzzzzzz" }], + poItems: [], + confidence: 0.9, + }); + expect(rows[0].purchase_order_item).toBe("zzzzzzzzzzzzzzz"); + // No local unit to convert into, so the quantity cannot be trusted. + expect(rows[0].qty_received).toBe(0); + expect(unverifiable).toBe(true); + }); + + it("skips non-objects and negative quantities", () => { + const { rows } = buildDeliveryNoteItemRows({ + items: [null, "nope", { description: "neg", qty: -3, unit: "kg" }], + poItems: PO_ITEMS, + confidence: 0.9, + }); + expect(rows).toHaveLength(0); + }); + + it("writes condition notes in the legacy format so the migration can parse them", () => { + const { rows } = buildDeliveryNoteItemRows({ + items: [ + { + description: "G6 Sub Base", + qty: 29960, + unit: "kg", + unit_price: 168.5, + match_confidence: 1, + matched_po_item_id: "aaaaaaaaaaaaaaa", + }, + ], + poItems: PO_ITEMS, + confidence: 0.95, + }); + expect(rows[0].condition_notes).toBe( + "OCR 95% · desc: G6 Sub Base · unit: kg · rate: 168.50 · match: 100%", + ); + }); +}); \ No newline at end of file diff --git a/src/lib/dn-item-row.ts b/src/lib/dn-item-row.ts new file mode 100644 index 0000000..a285d17 --- /dev/null +++ b/src/lib/dn-item-row.ts @@ -0,0 +1,159 @@ +/** + * Turns raw OCR line items into the exact rows written to + * delivery_note_items. + * + * Two bugs live here by construction rather than by care: + * + * 1. Matched-id drift. The route this replaced built its matched-id + * list from every item but consumed it only for items surviving the + * qty > 0 check, so one zero-quantity line shifted every later line + * onto the wrong PO item. Filtering happens once, here, and the rows + * carry their own ids — there is no second list to fall out of step. + * + * 2. Unit mismatch. Slips quote kg; PO lines are priced per ton. A line + * whose unit cannot be converted into its PO line's unit contributes + * ZERO and loses its link, exactly as an unmatched line already does. + * It surfaces as work for a human. It does not enter the maths at the + * wrong magnitude. + */ +import { convertQty, normaliseUnit } from "./units"; +import { validateMatchedPoItems } from "./validate-matched-po-items"; + +export interface PoItemLite { + id: string; + unit?: string; +} + +export interface DeliveryNoteItemRow { + /** Null when unmatched or unconvertible. */ + purchase_order_item: string | null; + /** In the PO line's unit. Zero whenever purchase_order_item is null. */ + qty_received: number; + /** Verbatim from the slip. */ + qty_raw: number; + /** Verbatim from the slip. Empty string when the slip showed none. */ + unit_raw: string; + unit_converted: boolean; + condition_notes: string; +} + +export interface BuildDeliveryNoteItemRowsInput { + items: readonly unknown[]; + poItems: readonly PoItemLite[]; + /** Overall OCR confidence, 0..1. Rendered into condition_notes. */ + confidence: number; +} + +export interface BuildDeliveryNoteItemRowsResult { + rows: DeliveryNoteItemRow[]; + /** True when we could not fetch the PO items but OCR supplied ids. */ + unverifiable: boolean; +} + +export function buildDeliveryNoteItemRows( + input: BuildDeliveryNoteItemRowsInput, +): BuildDeliveryNoteItemRowsResult { + const { items, poItems, confidence } = input; + + // Filter ONCE. Everything downstream indexes this list and only this + // list, so the matched ids cannot drift out of step with the rows. + const usable = items + .filter(isPlainObject) + .map((it) => ({ it, qty: asNumber(it.qty) })) + .filter((e): e is { it: Record; qty: number } => + e.qty !== null && e.qty > 0, + ); + + const { validIds, unverifiable } = validateMatchedPoItems({ + poItems, + matchedIds: usable.map((e) => asString(e.it.matched_po_item_id)), + }); + + const poItemById = new Map(poItems.map((p) => [p.id, p])); + + const rows = usable.map((entry, i) => { + const { it, qty } = entry; + const matchedId = validIds[i] ?? null; + const unitRaw = asString(it.unit) ?? ""; + + let poItemId: string | null = matchedId; + let qtyReceived = 0; + let converted = false; + let failure: string | null = null; + + if (matchedId === null) { + // Unmatched line: no PO rate, no target unit. Contributes nothing. + failure = null; + } else { + const poItem = poItemById.get(matchedId); + if (!poItem) { + // The id came from OCR and we could not cross-check it (empty + // local fetch). Keep the link for the audit trail but do not + // invent a quantity we cannot express in the right unit. + failure = "po_item_unavailable"; + } else { + const conv = convertQty(qty, normaliseUnit(unitRaw), normaliseUnit(poItem.unit)); + if (conv.ok) { + qtyReceived = conv.qty; + converted = conv.converted; + } else { + failure = conv.reason; + poItemId = null; + } + } + } + + return { + purchase_order_item: poItemId, + qty_received: qtyReceived, + qty_raw: qty, + unit_raw: unitRaw, + unit_converted: converted, + condition_notes: buildNotes(it, confidence, failure), + }; + }); + + return { rows, unverifiable }; +} + +/** + * The legacy format is preserved deliberately: scripts/migrate-dn-item-units.py + * recovers the slip's unit from the `unit: ` segment of rows + * written before unit_raw existed. + */ +function buildNotes( + it: Record, + confidence: number, + failure: string | null, +): string { + const parts = [ + `OCR ${Math.round(confidence * 100)}%`, + asString(it.description) ? `desc: ${asString(it.description)}` : "", + asString(it.unit) ? `unit: ${asString(it.unit)}` : "", + typeof it.unit_price === "number" ? `rate: ${it.unit_price.toFixed(2)}` : "", + typeof it.match_confidence === "number" + ? `match: ${Math.round(it.match_confidence * 100)}%` + : "", + failure ? `unconverted: ${failure}` : "", + ]; + return parts.filter(Boolean).join(" · "); +} + +function isPlainObject(v: unknown): v is Record { + return typeof v === "object" && v !== null && !Array.isArray(v); +} + +function asString(v: unknown): string | null { + if (typeof v !== "string") return null; + const s = v.trim(); + return s.length > 0 ? s : null; +} + +function asNumber(v: unknown): number | null { + if (typeof v === "number" && Number.isFinite(v)) return v; + if (typeof v === "string" && v.trim() !== "") { + const n = Number(v); + return Number.isFinite(n) ? n : null; + } + return null; +} \ No newline at end of file From 6a9c7403c499dd8ec6bfb981e0095162ad467f6d Mon Sep 17 00:00:00 2001 From: DEls Date: Thu, 17 Sep 2026 12:16:03 +0200 Subject: [PATCH 5/9] fix(delivery-notes): convert received quantity into the PO line's unit Fetches the PO item unit and delegates row construction to buildDeliveryNoteItemRows, which also removes the matched-id off-by-one that linked lines to the wrong PO item after a zero-quantity line. --- .../delivery-notes/[id]/ocr-result/route.ts | 87 +++++++------------ 1 file changed, 31 insertions(+), 56 deletions(-) diff --git a/src/app/api/delivery-notes/[id]/ocr-result/route.ts b/src/app/api/delivery-notes/[id]/ocr-result/route.ts index 5328a2f..492b55e 100644 --- a/src/app/api/delivery-notes/[id]/ocr-result/route.ts +++ b/src/app/api/delivery-notes/[id]/ocr-result/route.ts @@ -7,7 +7,7 @@ import { classifyPoMatch, type OcrMatchMethod, } from "@/lib/ocr-result-tier"; -import { validateMatchedPoItems } from "@/lib/validate-matched-po-items"; +import { buildDeliveryNoteItemRows } from "@/lib/dn-item-row"; /** * POST /api/delivery-notes/[id]/ocr-result @@ -205,21 +205,20 @@ export async function POST( .collection("purchase_order_items") .getFullList({ filter: `purchase_order='${poId}'`, - fields: "id,description,qty", + fields: "id,description,qty,unit", }); } catch (err) { console.warn("[dn-ocr-result] po_items fetch failed:", err); } } - const suppliedMatchedIds = items - .filter(isPlainObject) - .map((it) => - asString((it as Record).matched_po_item_id), - ); - const { validIds, unverifiable } = validateMatchedPoItems({ + // Rows are built before the tier decision because `unverifiable` + // feeds it. The builder is pure — see src/lib/dn-item-row.ts for why + // the filtering happens in exactly one place. + const { rows: itemRows, unverifiable } = buildDeliveryNoteItemRows({ + items, poItems, - matchedIds: suppliedMatchedIds, + confidence, }); if (unverifiable) { console.warn( @@ -316,69 +315,49 @@ export async function POST( } const createdItems: { id: string; po_item_id: string; qty: number }[] = []; - let itemIndex = 0; - for (const item of items) { - if (!isPlainObject(item)) continue; - const qty = asNumber(item.qty); - if (qty === null || qty <= 0) continue; - const validPoItemId = validIds[itemIndex++] ?? null; - - const notes = [ - `OCR ${Math.round(confidence * 100)}%`, - asString(item.description) ? `desc: ${asString(item.description)}` : "", - asString(item.unit) ? `unit: ${asString(item.unit)}` : "", - typeof item.unit_price === "number" - ? `rate: ${(item.unit_price as number).toFixed(2)}` - : "", - typeof item.match_confidence === "number" - ? `match: ${Math.round((item.match_confidence as number) * 100)}%` - : "", - ] - .filter(Boolean) - .join(" · "); - + for (const row of itemRows) { try { const created = await pb.collection("delivery_note_items").create({ delivery_note: dnId, - // Only link to a PO item if we got a verified match. Otherwise - // we still create the row (so the user sees what OCR found) but - // the link is left empty for manual completion. - purchase_order_item: validPoItemId, - qty_received: qty, - condition_notes: notes, + purchase_order_item: row.purchase_order_item, + qty_received: row.qty_received, + qty_raw: row.qty_raw, + unit_raw: row.unit_raw, + unit_converted: row.unit_converted, + condition_notes: row.condition_notes, }); createdItems.push({ id: created.id, - po_item_id: validPoItemId ?? "", - qty, + po_item_id: row.purchase_order_item ?? "", + qty: row.qty_received, }); } catch (err) { // FK constraint: PB rejected the link because the id no longer - // exists (race between our local fetch and the create — n8n - // had a stale id). Retry with the link dropped so the row - // itself still lands and the user can complete it manually. - // The PocketBase JS SDK surfaces this as a ClientResponseError - // with .status === 400 on the record create. + // exists (race between our local fetch and the create — n8n had a + // stale id). Retry with the link dropped so the row still lands + // and the user can complete it manually. The PocketBase JS SDK + // surfaces this as a ClientResponseError with .status === 400. const status = (err as { status?: number })?.status; const message = err instanceof Error ? err.message : String(err); - const isFkError = Boolean(validPoItemId) && status === 400; + const isFkError = Boolean(row.purchase_order_item) && status === 400; if (isFkError) { console.warn( - `[dn-ocr-result] delivery_note_item create failed for matched_po_item_id=${validPoItemId} (id no longer exists in PB) — retrying without link:`, + `[dn-ocr-result] delivery_note_item create failed for matched_po_item_id=${row.purchase_order_item} (id no longer exists in PB) — retrying without link:`, message, ); try { const created = await pb.collection("delivery_note_items").create({ delivery_note: dnId, purchase_order_item: null, - qty_received: qty, - condition_notes: `${notes} · link dropped: orphan id`, - }); - createdItems.push({ - id: created.id, - po_item_id: "", - qty, + // No PO line means no target unit, so no trustworthy + // converted quantity — same rule as an unmatched line. + qty_received: 0, + qty_raw: row.qty_raw, + unit_raw: row.unit_raw, + unit_converted: false, + condition_notes: `${row.condition_notes} · link dropped: orphan id`, }); + createdItems.push({ id: created.id, po_item_id: "", qty: 0 }); } catch (err2) { console.warn( "[dn-ocr-result] delivery_note_item create (no-link retry) failed:", @@ -438,10 +417,6 @@ function clamp01(n: number | null): number { return n; } -function isPlainObject(v: unknown): v is Record { - return typeof v === "object" && v !== null && !Array.isArray(v); -} - /* ------------------------------------------------------------------ */ /* Tiny PocketBase filter-string escaper */ /* ------------------------------------------------------------------ */ From e7882609b1bcb1365ffee6d16a015c3c1f622ea6 Mon Sep 17 00:00:00 2001 From: DEls Date: Thu, 17 Sep 2026 12:19:42 +0200 Subject: [PATCH 6/9] feat(purchasing): show received quantity in both PO and slip units --- .../financial/purchase-orders/[poId]/page.tsx | 7 ++++- .../purchase-orders/[poId]/po-detail-view.tsx | 6 +++-- src/lib/utils.test.ts | 26 +++++++++++++++++++ src/lib/utils.ts | 20 ++++++++++++++ 4 files changed, 56 insertions(+), 3 deletions(-) create mode 100644 src/lib/utils.test.ts diff --git a/src/app/dashboard/financial/purchase-orders/[poId]/page.tsx b/src/app/dashboard/financial/purchase-orders/[poId]/page.tsx index c4ff3c1..fdb5508 100644 --- a/src/app/dashboard/financial/purchase-orders/[poId]/page.tsx +++ b/src/app/dashboard/financial/purchase-orders/[poId]/page.tsx @@ -46,7 +46,7 @@ export default async function PurchaseOrderDetailPage({ expand: "site", }), pb.collection("delivery_note_items").getFullList({ - fields: "id,delivery_note,purchase_order_item,qty_received,condition_notes", + fields: "id,delivery_note,purchase_order_item,qty_received,qty_raw,unit_raw,condition_notes", }), // All suppliers, for the unknown-supplier picker on this page. pb.collection("suppliers").getFullList({ sort: "name" }), @@ -113,6 +113,11 @@ export default async function PurchaseOrderDetailPage({ id: r.id, purchaseOrderItemId: r.purchase_order_item, qtyReceived: r.qty_received, + // Rows written before the unit migration have no qty_raw; falling + // back to qty_received makes formatQtyDual render a single figure + // rather than "29.96 ton · 0 ". + qtyRaw: r.qty_raw ?? r.qty_received, + unitRaw: r.unit_raw ?? "", conditionNotes: r.condition_notes, })), })); diff --git a/src/app/dashboard/financial/purchase-orders/[poId]/po-detail-view.tsx b/src/app/dashboard/financial/purchase-orders/[poId]/po-detail-view.tsx index ab138d8..07bd5f2 100644 --- a/src/app/dashboard/financial/purchase-orders/[poId]/po-detail-view.tsx +++ b/src/app/dashboard/financial/purchase-orders/[poId]/po-detail-view.tsx @@ -32,7 +32,7 @@ import { DialogTitle, } from "@/components/ui/dialog"; import { SlipPreview } from "@/components/slip-preview"; -import { cn, formatCurrency, formatNumber } from "@/lib/utils"; +import { cn, formatCurrency, formatNumber, formatQtyDual } from "@/lib/utils"; import type { DeliveryNoteProcessingStatus } from "@/lib/pb-types"; export interface PoDetailItem { @@ -62,6 +62,8 @@ export interface PoDetailDn { id: string; purchaseOrderItemId: string; qtyReceived: number; + qtyRaw: number; + unitRaw: string; conditionNotes: string; }[]; } @@ -483,7 +485,7 @@ function DeliveryNoteCard({ )} - {formatNumber(r.qtyReceived)} {it?.unit ?? ""} + {formatQtyDual(r.qtyReceived, it?.unit ?? "", r.qtyRaw, r.unitRaw)} ); diff --git a/src/lib/utils.test.ts b/src/lib/utils.test.ts new file mode 100644 index 0000000..321bd1b --- /dev/null +++ b/src/lib/utils.test.ts @@ -0,0 +1,26 @@ +import { describe, it, expect } from "vitest"; +import { formatNumber, formatQtyDual } from "./utils"; + +describe("formatQtyDual", () => { + it("shows both figures when a conversion happened", () => { + expect(formatQtyDual(29.96, "ton", 29960, "kg")).toBe( + `${formatNumber(29.96)} ton · ${formatNumber(29960)} kg`, + ); + }); + + it("shows one figure when the units match", () => { + expect(formatQtyDual(5, "ton", 5, "ton")).toBe("5 ton"); + }); + + it("shows one figure when the slip recorded no unit", () => { + expect(formatQtyDual(5, "ton", 5, "")).toBe("5 ton"); + }); + + it("omits the unit entirely when the PO line has none", () => { + expect(formatQtyDual(5, "", 5, "")).toBe("5"); + }); + + it("falls back to the raw figure when nothing converted", () => { + expect(formatQtyDual(0, "ton", 12, "m3")).toBe("0 ton · 12 m3"); + }); +}); \ No newline at end of file diff --git a/src/lib/utils.ts b/src/lib/utils.ts index e0ab6df..a4b24d7 100644 --- a/src/lib/utils.ts +++ b/src/lib/utils.ts @@ -50,3 +50,23 @@ export function isUnknownSupplier(supplier: string | undefined | null): boolean if (!supplier) return true; return UNKNOWN_SUPPLIER_ALIASES.has(supplier.trim().toLowerCase()); } + +/** + * A received quantity, shown in the purchase order line's unit with the + * slip's own figure beside it. + * + * Site staff read the slip in kg; the PO is priced per ton. Showing only + * one of the two means somebody eventually reconciles the wrong number + * against the wrong document. + */ +export function formatQtyDual( + qtyReceived: number, + unit: string, + qtyRaw: number, + unitRaw: string, +): string { + const primary = unit ? `${formatNumber(qtyReceived)} ${unit}` : formatNumber(qtyReceived); + const sameUnit = !unitRaw || unitRaw.trim().toLowerCase() === unit.trim().toLowerCase(); + if (sameUnit) return primary; + return `${primary} · ${formatNumber(qtyRaw)} ${unitRaw}`; +} From 6e4ed417de170b6a4afa3a4340ac4a8245f089e3 Mon Sep 17 00:00:00 2001 From: DEls Date: Thu, 17 Sep 2026 12:22:58 +0200 Subject: [PATCH 7/9] chore(delivery-notes): backfill line item units and correct qty_received Rewrites qty_received into the purchase order line's unit, recovering the slip's own unit from the condition_notes format the OCR callback wrote. Rows with no recorded unit are left untouched and reported. --- scripts/migrate-dn-item-units.py | 176 +++++++++++++++++++++++++++++++ 1 file changed, 176 insertions(+) create mode 100644 scripts/migrate-dn-item-units.py diff --git a/scripts/migrate-dn-item-units.py b/scripts/migrate-dn-item-units.py new file mode 100644 index 0000000..b3696b6 --- /dev/null +++ b/scripts/migrate-dn-item-units.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +""" +Backfills qty_raw / unit_raw / unit_converted on delivery_note_items and +rewrites qty_received into the purchase order line's unit. + +Before this ran, qty_received held whatever the slip said — kg for +weighbridge tickets — while the PO line it points at is priced per ton. +Every fulfilment percentage and received value computed from those rows +was 1000x too large. + +The slip's unit survives only in condition_notes, which the OCR callback +wrote as: + + OCR 95% · desc: G6 Sub Base · unit: kg · match: 100% + +Rows with no `unit:` segment are left ALONE and reported. Guessing their +unit is how the original bug happened. + +Idempotent — rows that already have a non-empty unit_raw are skipped. + +Usage: + python3 scripts/migrate-dn-item-units.py # dry run + python3 scripts/migrate-dn-item-units.py --apply # writes +""" +import re +import sys + +from pb_env import PB, http, superuser_token + +UNIT_RE = re.compile(r"unit:\s*([^·]+)") + +ALIASES = { + "kg": ["kg", "kgs", "kilo", "kilos", "kilogram", "kilograms"], + "ton": ["t", "ton", "tons", "tonne", "tonnes", "mt", "metric ton"], + "m3": ["m3", "m³", "cube", "cubes", "cubic metre", "cubic meter"], + "m2": ["m2", "m²", "square metre", "square meter"], + "m": ["m", "metre", "meter", "metres", "meters", "lm", "linear metre"], + "ea": ["ea", "each", "unit", "units", "no", "nr", "item", "items"], + "bag": ["bag", "bags", "pocket", "pockets"], + "l": ["l", "litre", "litres", "liter", "liters"], + "load": ["load", "loads", "trip", "trips"], +} +LOOKUP = {a: c for c, al in ALIASES.items() for a in al} + +FACTORS = { + ("kg", "ton"): 1 / 1000, + ("ton", "kg"): 1000, + ("l", "m3"): 1 / 1000, + ("m3", "l"): 1000, +} + + +def normalise_unit(raw): + """Mirror of normaliseUnit in src/lib/units.ts. Keep the two in step.""" + if not isinstance(raw, str): + return None + cleaned = re.sub(r"\s+", " ", re.sub(r"[^a-z0-9²³ ]", " ", raw.lower())).strip() + return LOOKUP.get(cleaned) if cleaned else None + + +def convert(qty, from_unit, to_unit): + """Returns (qty, converted, failure_reason).""" + f, t = normalise_unit(from_unit), normalise_unit(to_unit) + if f is None or t is None: + return None, False, "unknown_unit" + if f == t: + return qty, False, None + factor = FACTORS.get((f, t)) + if factor is None: + return None, False, "incompatible_units" + return round(qty * factor, 4), True, None + + +def get_all(token, collection, fields=None, filt=None): + out, page = [], 1 + while True: + q = f"?page={page}&perPage=200" + if fields: + q += f"&fields={fields}" + if filt: + q += f"&filter={filt}" + code, resp = http("GET", f"/api/collections/{collection}/records{q}", token=token) + if code != 200: + sys.exit(f"GET {collection} failed: {code} {resp}") + out.extend(resp["items"]) + if page >= resp["totalPages"]: + return out + page += 1 + + +def main(apply: bool) -> None: + token = superuser_token() + print(f"\n→ delivery_note_items on {PB} ({'APPLY' if apply else 'DRY RUN'})") + + po_items = { + p["id"]: p + for p in get_all(token, "purchase_order_items", "id,unit,description") + } + rows = get_all( + token, + "delivery_note_items", + "id,delivery_note,purchase_order_item,qty_received,qty_raw,unit_raw,condition_notes", + ) + print(f" {len(rows)} rows, {len(po_items)} purchase order items\n") + + changed = skipped = no_unit = unconvertible = 0 + print(f" {'id':17s} {'from':>14s} {'to':>14s} note") + print(f" {'-' * 17} {'-' * 14} {'-' * 14} {'-' * 30}") + + for r in rows: + rid = r["id"] + if (r.get("unit_raw") or "").strip(): + skipped += 1 + continue + + qty = r.get("qty_received") or 0 + notes = r.get("condition_notes") or "" + m = UNIT_RE.search(notes) + if not m: + no_unit += 1 + print(f" {rid:17s} {qty:>14} {'—':>14} no unit: segment — LEFT ALONE") + continue + unit_raw = m.group(1).strip() + + poi = po_items.get(r.get("purchase_order_item") or "") + if not poi: + # Unlinked line: record what the slip said, leave qty_received. + patch = {"qty_raw": qty, "unit_raw": unit_raw, "unit_converted": False} + print(f" {rid:17s} {qty:>10} {unit_raw:<3} {'unlinked':>14} qty_raw only") + if apply: + code, resp = http("PATCH", f"/api/collections/delivery_note_items/records/{rid}", patch, token) + if code != 200: + sys.exit(f"PATCH {rid} failed: {code} {resp}") + changed += 1 + continue + + po_unit = poi.get("unit") or "" + new_qty, converted, failure = convert(qty, unit_raw, po_unit) + + if failure: + unconvertible += 1 + patch = { + "qty_raw": qty, + "unit_raw": unit_raw, + "unit_converted": False, + "qty_received": 0, + "purchase_order_item": None, + "condition_notes": f"{notes} · unconverted: {failure}", + } + print(f" {rid:17s} {qty:>10} {unit_raw:<3} {'0':>10} {po_unit:<3} {failure} — link dropped") + else: + patch = { + "qty_raw": qty, + "unit_raw": unit_raw, + "unit_converted": converted, + "qty_received": new_qty, + } + print(f" {rid:17s} {qty:>10} {unit_raw:<3} {new_qty:>10} {po_unit:<3} {poi.get('description', '')[:26]}") + + if apply: + code, resp = http("PATCH", f"/api/collections/delivery_note_items/records/{rid}", patch, token) + if code != 200: + sys.exit(f"PATCH {rid} failed: {code} {resp}") + changed += 1 + + print( + f"\n changed: {changed} already done: {skipped} " + f"no unit recorded: {no_unit} unconvertible: {unconvertible}" + ) + if not apply: + print("\n DRY RUN — nothing written. Re-run with --apply.") + + +if __name__ == "__main__": + main("--apply" in sys.argv) + print("\nDone.") \ No newline at end of file From 8d1b5a6db7b4eb3f09c9e16032f409a6b25e5529 Mon Sep 17 00:00:00 2001 From: DEls Date: Thu, 17 Sep 2026 12:42:44 +0200 Subject: [PATCH 8/9] docs(n8n): note the disconnected split branch pending the inbox pipeline --- docs/n8n/delivery-note-ocr.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/docs/n8n/delivery-note-ocr.md b/docs/n8n/delivery-note-ocr.md index c00482f..489987f 100644 --- a/docs/n8n/delivery-note-ocr.md +++ b/docs/n8n/delivery-note-ocr.md @@ -344,6 +344,35 @@ Watch the workflow's execution log in n8n and the DN's `processing_status` in Po ## Changelog +### 2026-09-17 — disconnected the split branch (workflow edit pending) + +The branch `Is first delivery note?` (false) → `Create split DN record` +in workflow `Delivery Note OCR` (`wVbLw4paISkbrIpE`) **needs to be +disconnected**. As of 2026-09-17 the automated disconnection via +`update_workflow` did not land — the MCP tool's `operations` array +parameter is currently being unwrapped to a single object server-side, +so the agent cannot apply the change. A human must remove the +connection in the n8n UI at https://wfdash.n8n.apphub.co.za. + +**Why.** The branch minted a new `delivery_notes` row per slip when a +single scan contained multiple slips, stamping `received_by='n8n-ocr-split'` +and `processing_status='ocr-complete'` with no line items. The dashboard +exception inbox does not list rows in either state, so the orphans +were invisible. Three of those rows existed before the planned +disconnection; they are surfaced and recovered by hand in Task 8 of +the unit-fix plan. + +**Effect once disconnected.** A multi-slip document will land as a +single note (the first item the LLM extracted) with +`processing_status='needs-review'` and a payload that covers all the +items. It shows up in the exception inbox. Phase 2 of the spec +replaces this with the inbox batch pipeline (§4) — at which point +the explicit split branch can be removed. + +**Tracking.** Once disconnected, the orphaned `received_by='n8n-ocr-split'` +row count must stay at 3 from then on. See Task 8 in `.superpowers/sdd/` +for the recovery flow. + ### 2026-08-26 — initial filter bug fix The first version of the workflow filter was: From 6cee861c3ec7a856ebd1055bcebb97d57a201703 Mon Sep 17 00:00:00 2001 From: DEls Date: Thu, 17 Sep 2026 12:49:37 +0200 Subject: [PATCH 9/9] chore: final review polish (trailing newlines, JSDoc, changelog tense) --- docs/n8n/delivery-note-ocr.md | 2 +- scripts/migrate-dn-item-units.py | 2 +- src/lib/dn-item-row.test.ts | 2 +- src/lib/dn-item-row.ts | 5 +++-- src/lib/utils.test.ts | 2 +- 5 files changed, 7 insertions(+), 6 deletions(-) diff --git a/docs/n8n/delivery-note-ocr.md b/docs/n8n/delivery-note-ocr.md index 489987f..0490dea 100644 --- a/docs/n8n/delivery-note-ocr.md +++ b/docs/n8n/delivery-note-ocr.md @@ -344,7 +344,7 @@ Watch the workflow's execution log in n8n and the DN's `processing_status` in Po ## Changelog -### 2026-09-17 — disconnected the split branch (workflow edit pending) +### 2026-09-17 — split branch pending disconnection (workflow edit pending) The branch `Is first delivery note?` (false) → `Create split DN record` in workflow `Delivery Note OCR` (`wVbLw4paISkbrIpE`) **needs to be diff --git a/scripts/migrate-dn-item-units.py b/scripts/migrate-dn-item-units.py index b3696b6..c8cecc2 100644 --- a/scripts/migrate-dn-item-units.py +++ b/scripts/migrate-dn-item-units.py @@ -173,4 +173,4 @@ def main(apply: bool) -> None: if __name__ == "__main__": main("--apply" in sys.argv) - print("\nDone.") \ No newline at end of file + print("\nDone.") diff --git a/src/lib/dn-item-row.test.ts b/src/lib/dn-item-row.test.ts index 3e7fbc9..97e949f 100644 --- a/src/lib/dn-item-row.test.ts +++ b/src/lib/dn-item-row.test.ts @@ -131,4 +131,4 @@ describe("buildDeliveryNoteItemRows", () => { "OCR 95% · desc: G6 Sub Base · unit: kg · rate: 168.50 · match: 100%", ); }); -}); \ No newline at end of file +}); diff --git a/src/lib/dn-item-row.ts b/src/lib/dn-item-row.ts index a285d17..3b06148 100644 --- a/src/lib/dn-item-row.ts +++ b/src/lib/dn-item-row.ts @@ -25,7 +25,8 @@ export interface PoItemLite { } export interface DeliveryNoteItemRow { - /** Null when unmatched or unconvertible. */ + /** Null when unmatched or unconvertible. Non-null with `qty_received = 0` + * when the id is unverifiable (PO items could not be fetched). */ purchase_order_item: string | null; /** In the PO line's unit. Zero whenever purchase_order_item is null. */ qty_received: number; @@ -156,4 +157,4 @@ function asNumber(v: unknown): number | null { return Number.isFinite(n) ? n : null; } return null; -} \ No newline at end of file +} diff --git a/src/lib/utils.test.ts b/src/lib/utils.test.ts index 321bd1b..0243ecf 100644 --- a/src/lib/utils.test.ts +++ b/src/lib/utils.test.ts @@ -23,4 +23,4 @@ describe("formatQtyDual", () => { it("falls back to the raw figure when nothing converted", () => { expect(formatQtyDual(0, "ton", 12, "m3")).toBe("0 ton · 12 m3"); }); -}); \ No newline at end of file +});