diff --git a/.freebuff/project-id b/.freebuff/project-id new file mode 100644 index 0000000..54e901b --- /dev/null +++ b/.freebuff/project-id @@ -0,0 +1 @@ +fc645fce-5605-4c39-9e76-8b9f4b097058 diff --git a/src/app/api/delivery-notes/[id]/ocr-result/route.ts b/src/app/api/delivery-notes/[id]/ocr-result/route.ts index 1d8a6db..59494da 100644 --- a/src/app/api/delivery-notes/[id]/ocr-result/route.ts +++ b/src/app/api/delivery-notes/[id]/ocr-result/route.ts @@ -2,6 +2,12 @@ import { NextResponse } from "next/server"; import { pbAdmin } from "@/lib/pb"; import { nextBatchState, MAX_ATTEMPTS } from "@/lib/batch-state"; import type { PurchaseOrderItemRecord } from "@/lib/pb-types"; +import { findBestPoMatch } from "@/lib/po-matching"; +import { + resolveOcrTier, + classifyPoMatch, + type OcrMatchMethod, +} from "@/lib/ocr-result-tier"; /** * POST /api/delivery-notes/[id]/ocr-result @@ -32,14 +38,24 @@ import type { PurchaseOrderItemRecord } from "@/lib/pb-types"; * "raw_ocr": "DELIVERY NOTE\nDN-5678\n..." // full OCR text (optional) * } * - * Behaviour: - * - Overall confidence >= 0.8: creates received_items, sets - * processing_status="auto-verified" and is_verified=true. - * - 0.5..0.8: creates received_items as a draft, sets + * Behaviour (see resolveOcrTier for the authoritative rules): + * - Overall confidence >= 0.8 AND PO resolved: creates received_items, + * sets processing_status="auto-verified" and is_verified=true. + * - Overall confidence >= 0.8 BUT PO did NOT resolve (OCR was + * confident in the slip but the PO number didn't fuzzy-match + * any PO we have on file): DOWNGRADES to draft — we never + * auto-verify an orphaned DN. + * - 0.5..0.79: creates received_items as a draft, sets * processing_status="needs-review" and is_verified=false. * - < 0.5: doesn't create received_items, sets * processing_status="needs-review" and is_verified=false. * + * PO matching (in priority order): exact → normalised + * (case/punctuation only) → Levenshtein single-char edit. The chosen + * path + similarity score is recorded in raw_ocr_data so the + * purchasing exception inbox can show "matched by Levenshtein, 0.9" + * instead of a silent "no match". + * * Idempotency: if the webhook is replayed, received_items already * linked to this DN are updated rather than duplicated. */ @@ -102,37 +118,64 @@ export async function POST( return NextResponse.json({ error: "dn_not_found" }, { status: 404 }); } - // 3b. Auto-match the PO from the OCR'd po_number. We try exact match - // first (the user / AI types it precisely), then a fuzzy fallback - // (strip non-alphanumeric + lowercase) for "PO-217" vs "PO-00217" - // mismatches. If nothing matches, leave purchase_order null and flag - // for manual review via the purchasing page. + // 3b. Auto-match the PO from the OCR'd po_number. We try three paths + // in order: exact → normalised (case/punctuation only) → Levenshtein + // (single-char OCR error). If nothing matches, leave purchase_order + // null and the tier downgrade (below) will mark the DN as + // needs-review regardless of OCR confidence. let matchedPoId: string | null = null; + let poResolvedNumber: string | null = null; + let poMatchMethod: OcrMatchMethod = "none"; + let poMatchScore: number | null = null; if (poNumber) { try { const exact = await pb .collection("purchase_orders") .getFirstListItem(`purchase_order_number="${escapePb(poNumber)}"`, { - fields: "id", + fields: "id,purchase_order_number", }); matchedPoId = exact.id; + poResolvedNumber = exact.purchase_order_number ?? null; + poMatchMethod = "exact"; + poMatchScore = 1; } catch { + // Exact miss — fall through to normalised lookup. const normalised = poNumber.replace(/[^a-z0-9]/gi, "").toLowerCase(); if (normalised.length >= 3) { try { - const candidates = await pb + const candidates = (await pb .collection("purchase_orders") .getFullList({ fields: "id,purchase_order_number", perPage: 200, - }); - const hit = candidates.find( + })) as unknown as Array<{ + id: string; + purchase_order_number: string; + }>; + const normHit = candidates.find( (c) => (c.purchase_order_number ?? "") .replace(/[^a-z0-9]/gi, "") .toLowerCase() === normalised, ); - if (hit) matchedPoId = hit.id; + if (normHit) { + matchedPoId = normHit.id; + poResolvedNumber = normHit.purchase_order_number; + poMatchMethod = "normalised"; + poMatchScore = 1; + } else { + // Final fallback: Levenshtein. Same helper the by-number + // route uses, so behaviour is consistent end-to-end. + const fuzzy = findBestPoMatch(poNumber, candidates, { + threshold: 0.75, + }); + if (fuzzy) { + matchedPoId = fuzzy.id; + poResolvedNumber = fuzzy.purchase_order_number; + poMatchMethod = "levenshtein"; + poMatchScore = fuzzy.score; + } + } } catch (err) { console.warn("[dn-ocr-result] fuzzy PO lookup failed:", err); } @@ -141,12 +184,16 @@ export async function POST( } // 4. Determine the tier and update the DN status / OCR fields. - // Tiers (see header doc): - // >= 0.8 → auto-verified, mark is_verified=true - // 0.5..0.79 → needs-review, draft received_items - // < 0.5 → needs-review, no received_items (user captures manually) - const tier = - confidence >= 0.8 ? "auto" : confidence >= 0.5 ? "draft" : "manual"; + // Tiers (see header doc + resolveOcrTier for rules): + // >= 0.8 AND PO resolved → auto-verified, mark is_verified=true + // >= 0.8 BUT PO unresolved → draft (downgraded — don't auto-verify + // an orphaned DN with no PO link) + // 0.5..0.79 → draft, received_items created for review + // < 0.5 → manual, no received_items (user captures manually) + const tier = resolveOcrTier({ + confidence, + poResolved: matchedPoId !== null, + }); const processingStatus = tier === "auto" ? "auto-verified" : "needs-review"; const isVerified = tier === "auto"; @@ -177,8 +224,16 @@ export async function POST( raw_ocr_data: { source: "n8n-ocr", po_number: poNumber, + po_resolved_number: poResolvedNumber, + po_match_method: classifyPoMatch( + poNumber ?? "", + poResolvedNumber, + poMatchMethod, + ), + po_match_score: poMatchScore, delivery_note_number: deliveryNoteNumber, confidence, + tier, items: items, raw_ocr: typeof rawOcr === "string" ? rawOcr.slice(0, 50000) : null, received_at: new Date().toISOString(), diff --git a/src/app/api/purchase-orders/by-number/[poNumber]/items/route.ts b/src/app/api/purchase-orders/by-number/[poNumber]/items/route.ts index 9f96318..6a6c7f2 100644 --- a/src/app/api/purchase-orders/by-number/[poNumber]/items/route.ts +++ b/src/app/api/purchase-orders/by-number/[poNumber]/items/route.ts @@ -1,5 +1,6 @@ import { NextResponse } from "next/server"; import { pbAdmin } from "@/lib/pb"; +import { findBestPoMatch } from "@/lib/po-matching"; /** * GET /api/purchase-orders/by-number/[poNumber]/items @@ -20,39 +21,6 @@ import { pbAdmin } from "@/lib/pb"; * 502 pb_lookup_failed */ -/// -// / Levenshtein distance and similarity for fuzzy PO matching. -/// -function levenshteinDistance(a: string, b: string): number { - if (a === b) return 0; - if (a.length === 0) return b.length; - if (b.length === 0) return a.length; - - let prev = new Array(b.length + 1); - let curr = new Array(b.length + 1); - for (let j = 0; j <= b.length; j++) prev[j] = j; - - for (let i = 1; i <= a.length; i++) { - curr[0] = i; - for (let j = 1; j <= b.length; j++) { - const cost = a[i - 1] === b[j - 1] ? 0 : 1; - curr[j] = Math.min( - curr[j - 1] + 1, // insertion - prev[j] + 1, // deletion - prev[j - 1] + cost, // substitution - ); - } - [prev, curr] = [curr, prev]; - } - return prev[b.length]; -} - -function levenshteinSimilarity(a: string, b: string): number { - const maxLen = Math.max(a.length, b.length); - if (maxLen === 0) return 1; - return 1 - levenshteinDistance(a, b) / maxLen; -} - export async function GET( request: Request, { params }: { params: Promise<{ poNumber: string }> }, @@ -103,39 +71,31 @@ export async function GET( // SRU 11987"), or picks up the wrong number from a noisy slip. // We have ~15 POs in PocketBase (one supplier per PO), so it's // cheap to score the OCR'd value against all of them and pick the - // best. Levenshtein on the normalised (alphanumeric, lowercase) - // string handles single-character insertions/deletions perfectly. - const target = sanitisedPoNumber.replace(/[^a-z0-9]/gi, "").toLowerCase(); - if (target.length >= 4) { - try { - const candidates = await pb - .collection("purchase_orders") - .getFullList({ - fields: "id,purchase_order_number,supplier", - perPage: 500, - }); - let bestScore = 0; - let bestHit: { id: string; purchase_order_number: string; supplier: string } | null = null; - for (const c of candidates) { - const candidate = (c.purchase_order_number ?? "") - .replace(/[^a-z0-9]/gi, "") - .toLowerCase(); - if (candidate.length === 0) continue; - const score = levenshteinSimilarity(target, candidate); - if (score > bestScore) { - bestScore = score; - bestHit = c as unknown as { id: string; purchase_order_number: string; supplier: string }; - } - } - // Threshold tuned so single-char insert/delete still matches - // (similarity 0.9 for 10-char strings), but completely different - // POs (similarity < 0.5) don't. - if (bestHit && bestScore >= 0.75) { - po = bestHit; - } - } catch (err) { - console.error("[po-items] fuzzy lookup failed:", err); + // best. The Levenshtein-based helper handles single-character + // insertions/deletions/substitutions perfectly. + try { + const candidates = (await pb + .collection("purchase_orders") + .getFullList({ + fields: "id,purchase_order_number,supplier", + perPage: 500, + })) as unknown as Array<{ + id: string; + purchase_order_number: string; + supplier?: string; + }>; + const match = findBestPoMatch(sanitisedPoNumber, candidates, { + threshold: 0.75, + }); + if (match) { + po = { + id: match.id, + purchase_order_number: match.purchase_order_number, + supplier: match.supplier ?? "", + }; } + } catch (err) { + console.error("[po-items] fuzzy lookup failed:", err); } } diff --git a/src/lib/ocr-result-tier.test.ts b/src/lib/ocr-result-tier.test.ts new file mode 100644 index 0000000..fedc37f --- /dev/null +++ b/src/lib/ocr-result-tier.test.ts @@ -0,0 +1,71 @@ +import { describe, it, expect } from "vitest"; +import { + resolveOcrTier, + classifyPoMatch, + type OcrMatchMethod, +} from "./ocr-result-tier"; + +describe("resolveOcrTier", () => { + it("auto when confidence >= 0.8 AND PO resolved", () => { + expect(resolveOcrTier({ confidence: 0.8, poResolved: true })).toBe("auto"); + expect(resolveOcrTier({ confidence: 0.95, poResolved: true })).toBe("auto"); + }); + + it("draft when confidence 0.5..0.79 regardless of PO", () => { + expect(resolveOcrTier({ confidence: 0.5, poResolved: true })).toBe("draft"); + expect(resolveOcrTier({ confidence: 0.79, poResolved: true })).toBe("draft"); + expect(resolveOcrTier({ confidence: 0.7, poResolved: false })).toBe("draft"); + }); + + it("manual when confidence < 0.5", () => { + expect(resolveOcrTier({ confidence: 0.49, poResolved: true })).toBe("manual"); + expect(resolveOcrTier({ confidence: 0, poResolved: true })).toBe("manual"); + }); + + it("PO-not-resolved DOWNGRADES auto → draft even at high confidence", () => { + // The bug we're fixing: a 0.92-confidence OCR with no PO link + // used to be auto-verified with every line orphaned. Now it falls + // back to draft so the user reviews the PO link manually. + expect(resolveOcrTier({ confidence: 0.92, poResolved: false })).toBe("draft"); + expect(resolveOcrTier({ confidence: 0.81, poResolved: false })).toBe("draft"); + }); + + it("PO-not-resolved does not change draft or manual tiers", () => { + expect(resolveOcrTier({ confidence: 0.7, poResolved: false })).toBe("draft"); + expect(resolveOcrTier({ confidence: 0.3, poResolved: false })).toBe("manual"); + }); + + it("clamps confidence above 1 to 1", () => { + // Claude sometimes returns confidence as a percentage 0..100. + expect(resolveOcrTier({ confidence: 95, poResolved: true })).toBe("auto"); + }); +}); + +describe("classifyPoMatch", () => { + it("exact for a literal string match against a real PO", () => { + expect(classifyPoMatch("MEGCPO9239", "MEGCPO9239", "exact")).toBe("exact"); + }); + + it("normalised when case or punctuation differs but alphanumeric lowercase matches", () => { + expect(classifyPoMatch("MEGCPO9239", "MEGCPO9239", "normalised")).toBe( + "normalised", + ); + }); + + it("levenshtein when best match needed fuzzy edit distance", () => { + expect(classifyPoMatch("MEGCP09239", "MEGCPO9239", "levenshtein")).toBe( + "levenshtein", + ); + }); + + it("none when nothing resolved above threshold", () => { + expect(classifyPoMatch("MEGCP09239", null, "none")).toBe("none"); + }); + + it("returns one of the four enum values", () => { + const values: OcrMatchMethod[] = ["exact", "normalised", "levenshtein", "none"]; + for (const v of values) { + expect(values).toContain(v); + } + }); +}); diff --git a/src/lib/ocr-result-tier.ts b/src/lib/ocr-result-tier.ts new file mode 100644 index 0000000..c46c75e --- /dev/null +++ b/src/lib/ocr-result-tier.ts @@ -0,0 +1,61 @@ +/** + * Pure helpers for the delivery-note OCR ingest endpoint. + * + * Lives outside the route so we can unit-test the tier-resolution + * rules (especially the "PO didn't resolve → downgrade" rule) without + * touching PocketBase or n8n. + * + * Why the PO-resolved downgrade matters: + * Before this, a slip with confidence 0.92 but a garbled PO + * number would be marked auto-verified with every received_item + * linked to nothing. The user only noticed when reconciling the + * exception inbox — by which point the audit trail had already + * been stamped as "verified". Now the tier falls back to "draft" + * so the user is forced to review. + */ + +export type OcrTier = "auto" | "draft" | "manual"; + +export type OcrMatchMethod = "exact" | "normalised" | "levenshtein" | "none"; + +/** + * Map the OCR confidence + whether the PO actually resolved to a tier. + * + * confidence poResolved tier + * ---------- ---------- ----- + * >= 0.8 true auto + * >= 0.8 false draft (NEW: don't auto-verify orphaned DN) + * 0.5..0.79 * draft + * < 0.5 * manual + * + * Confidence is clamped to [0, 1]; values > 1 are treated as percentages + * (95 → 0.95) — Claude sometimes returns 0..100, and the route's + * `clamp01` helper passes through anything that looks like 0..100. + */ +export function resolveOcrTier(input: { + confidence: number; + poResolved: boolean; +}): OcrTier { + const c = input.confidence > 1 ? input.confidence / 100 : input.confidence; + if (c >= 0.8 && input.poResolved) return "auto"; + if (c >= 0.5) return "draft"; + return "manual"; +} + +/** + * Record-keeping helper: given the raw PO number from OCR and the + * resolved PO number (or null), return which matching path took us + * from raw → resolved. Surfaced in raw_ocr_data so the purchasing + * exception inbox can show "matched by Levenshtein, similarity 0.9" + * instead of a silent "no match". + * + * The path is supplied by the caller (we don't recompute it here) + * so this function stays pure. + */ +export function classifyPoMatch( + _raw: string, + _resolved: string | null, + path: OcrMatchMethod, +): OcrMatchMethod { + return path; +} diff --git a/src/lib/po-matching.test.ts b/src/lib/po-matching.test.ts new file mode 100644 index 0000000..e803d4f --- /dev/null +++ b/src/lib/po-matching.test.ts @@ -0,0 +1,203 @@ +import { describe, it, expect } from "vitest"; +import { + levenshteinDistance, + levenshteinSimilarity, + normalisePoNumber, + findBestPoMatch, + extractPoCandidate, +} from "./po-matching"; + +describe("levenshteinDistance", () => { + it("returns 0 for identical strings", () => { + expect(levenshteinDistance("abc", "abc")).toBe(0); + }); + + it("returns the length of the other string when one is empty", () => { + expect(levenshteinDistance("", "abc")).toBe(3); + expect(levenshteinDistance("abc", "")).toBe(3); + }); + + it("counts single substitution as 1", () => { + expect(levenshteinDistance("megcp09239", "megcpo9239")).toBe(1); + }); + + it("counts transposition of two adjacent chars as 2 (sub + sub)", () => { + // "ab" -> "ba" needs 2 substitutions in classic Levenshtein + expect(levenshteinDistance("ab", "ba")).toBe(2); + }); + + it("counts insertion as 1", () => { + expect(levenshteinDistance("megcp9239", "megcpo9239")).toBe(1); + }); + + it("counts deletion as 1", () => { + expect(levenshteinDistance("megcpo9239", "megcp9239")).toBe(1); + }); + + it("is symmetric", () => { + expect(levenshteinDistance("abc", "xbc")).toBe(levenshteinDistance("xbc", "abc")); + }); +}); + +describe("levenshteinSimilarity", () => { + it("returns 1 for identical strings", () => { + expect(levenshteinSimilarity("megcpo9239", "megcpo9239")).toBe(1); + }); + + it("returns 0 for completely different strings", () => { + // a='ab', b='cd' -> distance 2, max len 2 -> sim 0 + expect(levenshteinSimilarity("ab", "cd")).toBe(0); + }); + + it("returns 1 - distance/maxLen for similar strings", () => { + // distance 1, maxLen 10 -> 0.9 + expect(levenshteinSimilarity("megcp09239", "megcpo9239")).toBeCloseTo(0.9, 5); + }); + + it("returns 1 when both strings are empty", () => { + expect(levenshteinSimilarity("", "")).toBe(1); + }); +}); + +describe("normalisePoNumber", () => { + it("strips non-alphanumeric chars", () => { + expect(normalisePoNumber("MEGCPO-9239")).toBe("megcpo9239"); + }); + + it("lowercases", () => { + expect(normalisePoNumber("MEGCP09239")).toBe("megcp09239"); + }); + + it("returns empty string for null-ish input", () => { + expect(normalisePoNumber(null)).toBe(""); + expect(normalisePoNumber(undefined)).toBe(""); + expect(normalisePoNumber("")).toBe(""); + expect(normalisePoNumber(42)).toBe(""); + }); +}); + +describe("extractPoCandidate", () => { + it("returns the cleaned single PO when input has no separator", () => { + expect(extractPoCandidate("MEGCP09239")).toBe("MEGCP09239"); + }); + + it("strips a leading #", () => { + expect(extractPoCandidate("#MEGCPO9239")).toBe("MEGCPO9239"); + }); + + it("trims surrounding whitespace", () => { + expect(extractPoCandidate(" MEGCPO9239 ")).toBe("MEGCPO9239"); + }); + + it("picks the longest alphanumeric token when input has two POs separated by /", () => { + // MEGCP09150 (len 10) vs SRU11987 (len 8) -> MEGCP09150 wins + expect(extractPoCandidate("MEGCP09150 / SRU 11987")).toBe("MEGCP09150"); + }); + + it("splits on comma, semicolon, newline, pipe", () => { + expect(extractPoCandidate("MEGCP09239,FOO1234")).toBe("MEGCP09239"); + expect(extractPoCandidate("MEGCP09239;FOO1234")).toBe("MEGCP09239"); + expect(extractPoCandidate("MEGCP09239\nFOO1234")).toBe("MEGCP09239"); + expect(extractPoCandidate("MEGCP09239|FOO1234")).toBe("MEGCP09239"); + }); + + it("returns empty string when input is null or empty", () => { + expect(extractPoCandidate(null)).toBe(""); + expect(extractPoCandidate(undefined)).toBe(""); + expect(extractPoCandidate("")).toBe(""); + }); + + it("returns the single token if no token meets length>=6", () => { + // No token has length >=6 with digit+letter, but the first token exists + // and gets the looser fallback. Here "abc12" is length 5, below minLength, + // but tokens[0] fallback uses the first token regardless of length. + expect(extractPoCandidate("abc12 def")).toBe("abc12"); + }); + + it("returns empty string when input is only short tokens and no token[0]", () => { + // Both tokens below threshold and no fallback possible + expect(extractPoCandidate("/")).toBe(""); + }); + + it("does not include tokens that lack digits", () => { + // ABCDEF (no digit) is rejected even if longest + expect(extractPoCandidate("ABCDEF / PO12345")).toBe("PO12345"); + }); + + it("does not include tokens that lack letters", () => { + // 123456 (no letter) is rejected + expect(extractPoCandidate("123456 / PO12345")).toBe("PO12345"); + }); + + it("accepts a custom minLength", () => { + // minLength=6: short token rejected, long token wins + expect(extractPoCandidate("AB / PO12345", { minLength: 6 })).toBe("PO12345"); + // minLength=3: short token accepted but longer token still wins + expect(extractPoCandidate("AB1 / AB12XYZ", { minLength: 3 })).toBe("AB12XYZ"); + // minLength=10: nothing passes, fallback picks first token + expect(extractPoCandidate("PO12345 / AB12", { minLength: 10 })).toBe("PO12345"); + }); +}); + +describe("findBestPoMatch", () => { + const candidates = [ + { id: "po1", purchase_order_number: "MEGCPO9239", supplier: "Builders Warehouse" }, + { id: "po2", purchase_order_number: "MEGCPO9109", supplier: "Vula Civil" }, + { id: "po3", purchase_order_number: "MEGCPO9150", supplier: "Signs R Us" }, + { id: "po4", purchase_order_number: "MEGCPO9165", supplier: "Raumix" }, + { id: "po5", purchase_order_number: "MEGCPO9240", supplier: "Silverton" }, + ]; + + it("returns null when target is empty", () => { + expect(findBestPoMatch("", candidates)).toBeNull(); + expect(findBestPoMatch(null, candidates)).toBeNull(); + }); + + it("returns null when no candidate exceeds the default threshold (0.75)", () => { + // "ABC12345" doesn't resemble any candidate strongly + expect(findBestPoMatch("ABC12345", candidates)).toBeNull(); + }); + + it("fuzzy-matches a 0↔O OCR error (distance 1, sim 0.9)", () => { + const hit = findBestPoMatch("MEGCP09239", candidates); + expect(hit).not.toBeNull(); + expect(hit?.id).toBe("po1"); + expect(hit?.score).toBeCloseTo(0.9, 5); + }); + + it("fuzzy-matches a missing O (distance 1, sim 0.9)", () => { + // MEGCP09150 -> MEGCPO9150 (missing O between P and 9) + const hit = findBestPoMatch("MEGCP09150", candidates); + expect(hit).not.toBeNull(); + expect(hit?.id).toBe("po3"); + }); + + it("lowercase target is normalised", () => { + const hit = findBestPoMatch("megcp09239", candidates); + expect(hit?.id).toBe("po1"); + }); + + it("strips non-alphanumeric noise", () => { + const hit = findBestPoMatch("MEG-CPO-9239", candidates); + expect(hit?.id).toBe("po1"); + }); + + it("respects a custom threshold", () => { + // MEGCP09234 -> MEGCPO9239 has distance 2, sim 0.8. + // Below default 0.85, above default 0.75. + expect(findBestPoMatch("MEGCP09234", candidates, { threshold: 0.6 })?.id).toBe("po1"); + expect(findBestPoMatch("MEGCP09234", candidates, { threshold: 0.85 })).toBeNull(); + }); + + it("returns the highest-scoring candidate when multiple exceed threshold", () => { + // Two PO prefixes that are close, only the closest wins + const hit = findBestPoMatch("MEGCP09239", candidates); + expect(hit?.id).toBe("po1"); + }); + + it("skips candidates with empty purchase_order_number", () => { + const noisy = [{ id: "x", purchase_order_number: "", supplier: "" }, ...candidates]; + const hit = findBestPoMatch("MEGCP09239", noisy); + expect(hit?.id).toBe("po1"); + }); +}); diff --git a/src/lib/po-matching.ts b/src/lib/po-matching.ts new file mode 100644 index 0000000..f07b170 --- /dev/null +++ b/src/lib/po-matching.ts @@ -0,0 +1,176 @@ +/** + * Pure PO-number matching helpers shared between the OCR pipeline + * (n8n workflow) and the OCR-result ingest endpoint. + * + * Why this lives in one place: + * - The same fuzzy logic was previously duplicated in two route files + * with subtly different thresholds and different normalisation + * rules. OCR errors that one place resolved were leaking through + * the other. + * - Pulling the algorithms into pure functions lets us unit-test + * them without touching PocketBase or n8n. + * + * What it does: + * - normalisePoNumber — strip non-alphanumeric, lowercase. Used as + * the equivalence key for "did OCR read the same PO we have on + * file?". + * - extractPoCandidate — clean OCR's `po_number` field when it + * arrives contaminated (leading "#", trailing junk, two POs + * separated by "/", etc.). Returns the single best candidate + * string. + * - findBestPoMatch — given a possibly-messy target and a list of + * real POs, return the closest above a similarity threshold (or + * null). This is the second-pass fuzzy fallback after exact + + * normalised lookup. + * - levenshteinDistance / levenshteinSimilarity — primitives used + * by findBestPoMatch. Exported for callers that want to roll + * their own scoring (e.g. the per-line item matcher in n8n). + */ + +/** + * Classic Levenshtein edit distance. Iterative two-row implementation: + * O(len(a) * len(b)) time, O(min(len(a), len(b))) space. + */ +export function levenshteinDistance(a: string, b: string): number { + if (a === b) return 0; + if (a.length === 0) return b.length; + if (b.length === 0) return a.length; + + let prev = new Array(b.length + 1); + let curr = new Array(b.length + 1); + for (let j = 0; j <= b.length; j++) prev[j] = j; + + for (let i = 1; i <= a.length; i++) { + curr[0] = i; + for (let j = 1; j <= b.length; j++) { + const cost = a[i - 1] === b[j - 1] ? 0 : 1; + curr[j] = Math.min( + curr[j - 1] + 1, // insertion + prev[j] + 1, // deletion + prev[j - 1] + cost, // substitution + ); + } + [prev, curr] = [curr, prev]; + } + return prev[b.length]; +} + +/** + * Levenshtein distance normalised to [0, 1] where 1 = identical and + * 0 = completely different (one side empty or max edits equal maxLen). + */ +export function levenshteinSimilarity(a: string, b: string): number { + const maxLen = Math.max(a.length, b.length); + if (maxLen === 0) return 1; + return 1 - levenshteinDistance(a, b) / maxLen; +} + +/** + * Equivalence key for a PO number. Strips everything that isn't a + * letter or digit and lowercases so "PO-1234" matches "po1234". + * + * Non-string input (null / undefined / number) returns "" so callers + * can safely pass through whatever they get from OCR. + */ +export function normalisePoNumber(input: unknown): string { + if (typeof input !== "string") return ""; + return input.replace(/[^a-z0-9]/gi, "").toLowerCase(); +} + +export type PoCandidate = { + id: string; + purchase_order_number: string; + supplier?: string; +}; + +export type PoMatch = PoCandidate & { score: number }; + +export type FindBestPoMatchOptions = { + /** Minimum similarity to accept (default 0.75 — handles 1-char edit on 10-char PO). */ + threshold?: number; +}; + +/** + * Return the candidate PO whose normalised form is closest to the + * target, or null if none exceed `threshold`. Both sides are + * normalised before comparison so "PO-217", "po217", "PO00217" all + * collapse to the same key. + * + * Used by: + * - /api/purchase-orders/by-number/[poNumber]/items — second-pass + * fallback when exact lookup misses. + * - /api/delivery-notes/[id]/ocr-result — last-resort auto-link + * after n8n posts back a raw OCR value. + */ +export function findBestPoMatch( + target: unknown, + candidates: PoCandidate[], + opts: FindBestPoMatchOptions = {}, +): PoMatch | null { + const threshold = opts.threshold ?? 0.75; + const t = normalisePoNumber(target); + if (t.length < 4) return null; + + let bestScore = 0; + let bestHit: PoCandidate | null = null; + for (const c of candidates) { + const candidate = normalisePoNumber(c.purchase_order_number); + if (candidate.length === 0) continue; + const score = levenshteinSimilarity(t, candidate); + if (score > bestScore) { + bestScore = score; + bestHit = c; + } + } + if (!bestHit || bestScore < threshold) return null; + return { ...bestHit, score: bestScore }; +} + +export type ExtractPoCandidateOptions = { + /** Minimum length for a token to be considered as a real PO (default 6). */ + minLength?: number; +}; + +/** + * Clean OCR's `po_number` field. Real-world OCR noise we handle: + * + * "#MEGCPO9165" leading "#" + * "MEGCP09239" missing O (0/O confusion) + * "MEGCP09150 / SRU 11987" two POs separated by "/" + * "MEGCPO9240 " trailing whitespace + * "PO12345\nREF9999" newline separator + * null vision LLM couldn't find a PO + * + * Strategy: split on common separators, keep only alphanumeric + * tokens containing both a digit and a letter, pick the longest + * (longer PO numbers are less likely to be a supplier reference). + * If no token meets the bar, fall back to the first token's cleaned + * form — better than dropping the value entirely. + * + * Always returns a string (empty when nothing usable). + */ +export function extractPoCandidate( + raw: unknown, + opts: ExtractPoCandidateOptions = {}, +): string { + if (typeof raw !== "string") return ""; + const minLength = opts.minLength ?? 6; + // Whitespace and the usual separator chars. Multi-PO strings like + // "MEGCP09150 / SRU 11987" or "MEGCP09239 SRU 11987" should split on + // either form. + const tokens = raw + .split(/[\s\/,;|\n\r]+/) + .map((s) => s.trim()) + .filter(Boolean); + let best = ""; + for (const t of tokens) { + const clean = t.replace(/[^a-zA-Z0-9]/g, ""); + if (clean.length < minLength) continue; + if (!/[0-9]/.test(clean) || !/[a-zA-Z]/.test(clean)) continue; + if (clean.length > best.length) best = clean; + } + if (!best && tokens[0]) { + best = tokens[0].replace(/[^a-zA-Z0-9]/g, ""); + } + return best || ""; +}