feat(ocr-result, po-items): enhance PO matching logic with fuzzy search
Updated the OCR result processing to include a more robust PO matching mechanism. The new implementation prioritizes exact matches, followed by normalized comparisons, and finally employs a Levenshtein-based fuzzy match for improved accuracy. This change ensures that delivery notes are correctly linked to purchase orders, even when OCR introduces minor errors. Additionally, the behavior for handling confidence levels and PO resolution has been clarified in the documentation, enhancing overall processing reliability.
This commit is contained in:
DEls
parent
2fcd7550de
commit
d03df384ea
7 changed files with 612 additions and 85 deletions
1
.freebuff/project-id
Normal file
1
.freebuff/project-id
Normal file
|
|
@ -0,0 +1 @@
|
|||
fc645fce-5605-4c39-9e76-8b9f4b097058
|
||||
|
|
@ -2,6 +2,12 @@ import { NextResponse } from "next/server";
|
|||
import { pbAdmin } from "@/lib/pb";
|
||||
import { nextBatchState, MAX_ATTEMPTS } from "@/lib/batch-state";
|
||||
import type { PurchaseOrderItemRecord } from "@/lib/pb-types";
|
||||
import { findBestPoMatch } from "@/lib/po-matching";
|
||||
import {
|
||||
resolveOcrTier,
|
||||
classifyPoMatch,
|
||||
type OcrMatchMethod,
|
||||
} from "@/lib/ocr-result-tier";
|
||||
|
||||
/**
|
||||
* POST /api/delivery-notes/[id]/ocr-result
|
||||
|
|
@ -32,14 +38,24 @@ import type { PurchaseOrderItemRecord } from "@/lib/pb-types";
|
|||
* "raw_ocr": "DELIVERY NOTE\nDN-5678\n..." // full OCR text (optional)
|
||||
* }
|
||||
*
|
||||
* Behaviour:
|
||||
* - Overall confidence >= 0.8: creates received_items, sets
|
||||
* processing_status="auto-verified" and is_verified=true.
|
||||
* - 0.5..0.8: creates received_items as a draft, sets
|
||||
* Behaviour (see resolveOcrTier for the authoritative rules):
|
||||
* - Overall confidence >= 0.8 AND PO resolved: creates received_items,
|
||||
* sets processing_status="auto-verified" and is_verified=true.
|
||||
* - Overall confidence >= 0.8 BUT PO did NOT resolve (OCR was
|
||||
* confident in the slip but the PO number didn't fuzzy-match
|
||||
* any PO we have on file): DOWNGRADES to draft — we never
|
||||
* auto-verify an orphaned DN.
|
||||
* - 0.5..0.79: creates received_items as a draft, sets
|
||||
* processing_status="needs-review" and is_verified=false.
|
||||
* - < 0.5: doesn't create received_items, sets
|
||||
* processing_status="needs-review" and is_verified=false.
|
||||
*
|
||||
* PO matching (in priority order): exact → normalised
|
||||
* (case/punctuation only) → Levenshtein single-char edit. The chosen
|
||||
* path + similarity score is recorded in raw_ocr_data so the
|
||||
* purchasing exception inbox can show "matched by Levenshtein, 0.9"
|
||||
* instead of a silent "no match".
|
||||
*
|
||||
* Idempotency: if the webhook is replayed, received_items already
|
||||
* linked to this DN are updated rather than duplicated.
|
||||
*/
|
||||
|
|
@ -102,37 +118,64 @@ export async function POST(
|
|||
return NextResponse.json({ error: "dn_not_found" }, { status: 404 });
|
||||
}
|
||||
|
||||
// 3b. Auto-match the PO from the OCR'd po_number. We try exact match
|
||||
// first (the user / AI types it precisely), then a fuzzy fallback
|
||||
// (strip non-alphanumeric + lowercase) for "PO-217" vs "PO-00217"
|
||||
// mismatches. If nothing matches, leave purchase_order null and flag
|
||||
// for manual review via the purchasing page.
|
||||
// 3b. Auto-match the PO from the OCR'd po_number. We try three paths
|
||||
// in order: exact → normalised (case/punctuation only) → Levenshtein
|
||||
// (single-char OCR error). If nothing matches, leave purchase_order
|
||||
// null and the tier downgrade (below) will mark the DN as
|
||||
// needs-review regardless of OCR confidence.
|
||||
let matchedPoId: string | null = null;
|
||||
let poResolvedNumber: string | null = null;
|
||||
let poMatchMethod: OcrMatchMethod = "none";
|
||||
let poMatchScore: number | null = null;
|
||||
if (poNumber) {
|
||||
try {
|
||||
const exact = await pb
|
||||
.collection("purchase_orders")
|
||||
.getFirstListItem(`purchase_order_number="${escapePb(poNumber)}"`, {
|
||||
fields: "id",
|
||||
fields: "id,purchase_order_number",
|
||||
});
|
||||
matchedPoId = exact.id;
|
||||
poResolvedNumber = exact.purchase_order_number ?? null;
|
||||
poMatchMethod = "exact";
|
||||
poMatchScore = 1;
|
||||
} catch {
|
||||
// Exact miss — fall through to normalised lookup.
|
||||
const normalised = poNumber.replace(/[^a-z0-9]/gi, "").toLowerCase();
|
||||
if (normalised.length >= 3) {
|
||||
try {
|
||||
const candidates = await pb
|
||||
const candidates = (await pb
|
||||
.collection("purchase_orders")
|
||||
.getFullList({
|
||||
fields: "id,purchase_order_number",
|
||||
perPage: 200,
|
||||
});
|
||||
const hit = candidates.find(
|
||||
})) as unknown as Array<{
|
||||
id: string;
|
||||
purchase_order_number: string;
|
||||
}>;
|
||||
const normHit = candidates.find(
|
||||
(c) =>
|
||||
(c.purchase_order_number ?? "")
|
||||
.replace(/[^a-z0-9]/gi, "")
|
||||
.toLowerCase() === normalised,
|
||||
);
|
||||
if (hit) matchedPoId = hit.id;
|
||||
if (normHit) {
|
||||
matchedPoId = normHit.id;
|
||||
poResolvedNumber = normHit.purchase_order_number;
|
||||
poMatchMethod = "normalised";
|
||||
poMatchScore = 1;
|
||||
} else {
|
||||
// Final fallback: Levenshtein. Same helper the by-number
|
||||
// route uses, so behaviour is consistent end-to-end.
|
||||
const fuzzy = findBestPoMatch(poNumber, candidates, {
|
||||
threshold: 0.75,
|
||||
});
|
||||
if (fuzzy) {
|
||||
matchedPoId = fuzzy.id;
|
||||
poResolvedNumber = fuzzy.purchase_order_number;
|
||||
poMatchMethod = "levenshtein";
|
||||
poMatchScore = fuzzy.score;
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.warn("[dn-ocr-result] fuzzy PO lookup failed:", err);
|
||||
}
|
||||
|
|
@ -141,12 +184,16 @@ export async function POST(
|
|||
}
|
||||
|
||||
// 4. Determine the tier and update the DN status / OCR fields.
|
||||
// Tiers (see header doc):
|
||||
// >= 0.8 → auto-verified, mark is_verified=true
|
||||
// 0.5..0.79 → needs-review, draft received_items
|
||||
// < 0.5 → needs-review, no received_items (user captures manually)
|
||||
const tier =
|
||||
confidence >= 0.8 ? "auto" : confidence >= 0.5 ? "draft" : "manual";
|
||||
// Tiers (see header doc + resolveOcrTier for rules):
|
||||
// >= 0.8 AND PO resolved → auto-verified, mark is_verified=true
|
||||
// >= 0.8 BUT PO unresolved → draft (downgraded — don't auto-verify
|
||||
// an orphaned DN with no PO link)
|
||||
// 0.5..0.79 → draft, received_items created for review
|
||||
// < 0.5 → manual, no received_items (user captures manually)
|
||||
const tier = resolveOcrTier({
|
||||
confidence,
|
||||
poResolved: matchedPoId !== null,
|
||||
});
|
||||
const processingStatus = tier === "auto" ? "auto-verified" : "needs-review";
|
||||
const isVerified = tier === "auto";
|
||||
|
||||
|
|
@ -177,8 +224,16 @@ export async function POST(
|
|||
raw_ocr_data: {
|
||||
source: "n8n-ocr",
|
||||
po_number: poNumber,
|
||||
po_resolved_number: poResolvedNumber,
|
||||
po_match_method: classifyPoMatch(
|
||||
poNumber ?? "",
|
||||
poResolvedNumber,
|
||||
poMatchMethod,
|
||||
),
|
||||
po_match_score: poMatchScore,
|
||||
delivery_note_number: deliveryNoteNumber,
|
||||
confidence,
|
||||
tier,
|
||||
items: items,
|
||||
raw_ocr: typeof rawOcr === "string" ? rawOcr.slice(0, 50000) : null,
|
||||
received_at: new Date().toISOString(),
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import { NextResponse } from "next/server";
|
||||
import { pbAdmin } from "@/lib/pb";
|
||||
import { findBestPoMatch } from "@/lib/po-matching";
|
||||
|
||||
/**
|
||||
* GET /api/purchase-orders/by-number/[poNumber]/items
|
||||
|
|
@ -20,39 +21,6 @@ import { pbAdmin } from "@/lib/pb";
|
|||
* 502 pb_lookup_failed
|
||||
*/
|
||||
|
||||
///
|
||||
// / Levenshtein distance and similarity for fuzzy PO matching.
|
||||
///
|
||||
function levenshteinDistance(a: string, b: string): number {
|
||||
if (a === b) return 0;
|
||||
if (a.length === 0) return b.length;
|
||||
if (b.length === 0) return a.length;
|
||||
|
||||
let prev = new Array(b.length + 1);
|
||||
let curr = new Array(b.length + 1);
|
||||
for (let j = 0; j <= b.length; j++) prev[j] = j;
|
||||
|
||||
for (let i = 1; i <= a.length; i++) {
|
||||
curr[0] = i;
|
||||
for (let j = 1; j <= b.length; j++) {
|
||||
const cost = a[i - 1] === b[j - 1] ? 0 : 1;
|
||||
curr[j] = Math.min(
|
||||
curr[j - 1] + 1, // insertion
|
||||
prev[j] + 1, // deletion
|
||||
prev[j - 1] + cost, // substitution
|
||||
);
|
||||
}
|
||||
[prev, curr] = [curr, prev];
|
||||
}
|
||||
return prev[b.length];
|
||||
}
|
||||
|
||||
function levenshteinSimilarity(a: string, b: string): number {
|
||||
const maxLen = Math.max(a.length, b.length);
|
||||
if (maxLen === 0) return 1;
|
||||
return 1 - levenshteinDistance(a, b) / maxLen;
|
||||
}
|
||||
|
||||
export async function GET(
|
||||
request: Request,
|
||||
{ params }: { params: Promise<{ poNumber: string }> },
|
||||
|
|
@ -103,39 +71,31 @@ export async function GET(
|
|||
// SRU 11987"), or picks up the wrong number from a noisy slip.
|
||||
// We have ~15 POs in PocketBase (one supplier per PO), so it's
|
||||
// cheap to score the OCR'd value against all of them and pick the
|
||||
// best. Levenshtein on the normalised (alphanumeric, lowercase)
|
||||
// string handles single-character insertions/deletions perfectly.
|
||||
const target = sanitisedPoNumber.replace(/[^a-z0-9]/gi, "").toLowerCase();
|
||||
if (target.length >= 4) {
|
||||
try {
|
||||
const candidates = await pb
|
||||
.collection("purchase_orders")
|
||||
.getFullList({
|
||||
fields: "id,purchase_order_number,supplier",
|
||||
perPage: 500,
|
||||
});
|
||||
let bestScore = 0;
|
||||
let bestHit: { id: string; purchase_order_number: string; supplier: string } | null = null;
|
||||
for (const c of candidates) {
|
||||
const candidate = (c.purchase_order_number ?? "")
|
||||
.replace(/[^a-z0-9]/gi, "")
|
||||
.toLowerCase();
|
||||
if (candidate.length === 0) continue;
|
||||
const score = levenshteinSimilarity(target, candidate);
|
||||
if (score > bestScore) {
|
||||
bestScore = score;
|
||||
bestHit = c as unknown as { id: string; purchase_order_number: string; supplier: string };
|
||||
}
|
||||
}
|
||||
// Threshold tuned so single-char insert/delete still matches
|
||||
// (similarity 0.9 for 10-char strings), but completely different
|
||||
// POs (similarity < 0.5) don't.
|
||||
if (bestHit && bestScore >= 0.75) {
|
||||
po = bestHit;
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("[po-items] fuzzy lookup failed:", err);
|
||||
// best. The Levenshtein-based helper handles single-character
|
||||
// insertions/deletions/substitutions perfectly.
|
||||
try {
|
||||
const candidates = (await pb
|
||||
.collection("purchase_orders")
|
||||
.getFullList({
|
||||
fields: "id,purchase_order_number,supplier",
|
||||
perPage: 500,
|
||||
})) as unknown as Array<{
|
||||
id: string;
|
||||
purchase_order_number: string;
|
||||
supplier?: string;
|
||||
}>;
|
||||
const match = findBestPoMatch(sanitisedPoNumber, candidates, {
|
||||
threshold: 0.75,
|
||||
});
|
||||
if (match) {
|
||||
po = {
|
||||
id: match.id,
|
||||
purchase_order_number: match.purchase_order_number,
|
||||
supplier: match.supplier ?? "",
|
||||
};
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("[po-items] fuzzy lookup failed:", err);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
71
src/lib/ocr-result-tier.test.ts
Normal file
71
src/lib/ocr-result-tier.test.ts
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
import { describe, it, expect } from "vitest";
|
||||
import {
|
||||
resolveOcrTier,
|
||||
classifyPoMatch,
|
||||
type OcrMatchMethod,
|
||||
} from "./ocr-result-tier";
|
||||
|
||||
describe("resolveOcrTier", () => {
|
||||
it("auto when confidence >= 0.8 AND PO resolved", () => {
|
||||
expect(resolveOcrTier({ confidence: 0.8, poResolved: true })).toBe("auto");
|
||||
expect(resolveOcrTier({ confidence: 0.95, poResolved: true })).toBe("auto");
|
||||
});
|
||||
|
||||
it("draft when confidence 0.5..0.79 regardless of PO", () => {
|
||||
expect(resolveOcrTier({ confidence: 0.5, poResolved: true })).toBe("draft");
|
||||
expect(resolveOcrTier({ confidence: 0.79, poResolved: true })).toBe("draft");
|
||||
expect(resolveOcrTier({ confidence: 0.7, poResolved: false })).toBe("draft");
|
||||
});
|
||||
|
||||
it("manual when confidence < 0.5", () => {
|
||||
expect(resolveOcrTier({ confidence: 0.49, poResolved: true })).toBe("manual");
|
||||
expect(resolveOcrTier({ confidence: 0, poResolved: true })).toBe("manual");
|
||||
});
|
||||
|
||||
it("PO-not-resolved DOWNGRADES auto → draft even at high confidence", () => {
|
||||
// The bug we're fixing: a 0.92-confidence OCR with no PO link
|
||||
// used to be auto-verified with every line orphaned. Now it falls
|
||||
// back to draft so the user reviews the PO link manually.
|
||||
expect(resolveOcrTier({ confidence: 0.92, poResolved: false })).toBe("draft");
|
||||
expect(resolveOcrTier({ confidence: 0.81, poResolved: false })).toBe("draft");
|
||||
});
|
||||
|
||||
it("PO-not-resolved does not change draft or manual tiers", () => {
|
||||
expect(resolveOcrTier({ confidence: 0.7, poResolved: false })).toBe("draft");
|
||||
expect(resolveOcrTier({ confidence: 0.3, poResolved: false })).toBe("manual");
|
||||
});
|
||||
|
||||
it("clamps confidence above 1 to 1", () => {
|
||||
// Claude sometimes returns confidence as a percentage 0..100.
|
||||
expect(resolveOcrTier({ confidence: 95, poResolved: true })).toBe("auto");
|
||||
});
|
||||
});
|
||||
|
||||
describe("classifyPoMatch", () => {
|
||||
it("exact for a literal string match against a real PO", () => {
|
||||
expect(classifyPoMatch("MEGCPO9239", "MEGCPO9239", "exact")).toBe("exact");
|
||||
});
|
||||
|
||||
it("normalised when case or punctuation differs but alphanumeric lowercase matches", () => {
|
||||
expect(classifyPoMatch("MEGCPO9239", "MEGCPO9239", "normalised")).toBe(
|
||||
"normalised",
|
||||
);
|
||||
});
|
||||
|
||||
it("levenshtein when best match needed fuzzy edit distance", () => {
|
||||
expect(classifyPoMatch("MEGCP09239", "MEGCPO9239", "levenshtein")).toBe(
|
||||
"levenshtein",
|
||||
);
|
||||
});
|
||||
|
||||
it("none when nothing resolved above threshold", () => {
|
||||
expect(classifyPoMatch("MEGCP09239", null, "none")).toBe("none");
|
||||
});
|
||||
|
||||
it("returns one of the four enum values", () => {
|
||||
const values: OcrMatchMethod[] = ["exact", "normalised", "levenshtein", "none"];
|
||||
for (const v of values) {
|
||||
expect(values).toContain(v);
|
||||
}
|
||||
});
|
||||
});
|
||||
61
src/lib/ocr-result-tier.ts
Normal file
61
src/lib/ocr-result-tier.ts
Normal file
|
|
@ -0,0 +1,61 @@
|
|||
/**
|
||||
* Pure helpers for the delivery-note OCR ingest endpoint.
|
||||
*
|
||||
* Lives outside the route so we can unit-test the tier-resolution
|
||||
* rules (especially the "PO didn't resolve → downgrade" rule) without
|
||||
* touching PocketBase or n8n.
|
||||
*
|
||||
* Why the PO-resolved downgrade matters:
|
||||
* Before this, a slip with confidence 0.92 but a garbled PO
|
||||
* number would be marked auto-verified with every received_item
|
||||
* linked to nothing. The user only noticed when reconciling the
|
||||
* exception inbox — by which point the audit trail had already
|
||||
* been stamped as "verified". Now the tier falls back to "draft"
|
||||
* so the user is forced to review.
|
||||
*/
|
||||
|
||||
export type OcrTier = "auto" | "draft" | "manual";
|
||||
|
||||
export type OcrMatchMethod = "exact" | "normalised" | "levenshtein" | "none";
|
||||
|
||||
/**
|
||||
* Map the OCR confidence + whether the PO actually resolved to a tier.
|
||||
*
|
||||
* confidence poResolved tier
|
||||
* ---------- ---------- -----
|
||||
* >= 0.8 true auto
|
||||
* >= 0.8 false draft (NEW: don't auto-verify orphaned DN)
|
||||
* 0.5..0.79 * draft
|
||||
* < 0.5 * manual
|
||||
*
|
||||
* Confidence is clamped to [0, 1]; values > 1 are treated as percentages
|
||||
* (95 → 0.95) — Claude sometimes returns 0..100, and the route's
|
||||
* `clamp01` helper passes through anything that looks like 0..100.
|
||||
*/
|
||||
export function resolveOcrTier(input: {
|
||||
confidence: number;
|
||||
poResolved: boolean;
|
||||
}): OcrTier {
|
||||
const c = input.confidence > 1 ? input.confidence / 100 : input.confidence;
|
||||
if (c >= 0.8 && input.poResolved) return "auto";
|
||||
if (c >= 0.5) return "draft";
|
||||
return "manual";
|
||||
}
|
||||
|
||||
/**
|
||||
* Record-keeping helper: given the raw PO number from OCR and the
|
||||
* resolved PO number (or null), return which matching path took us
|
||||
* from raw → resolved. Surfaced in raw_ocr_data so the purchasing
|
||||
* exception inbox can show "matched by Levenshtein, similarity 0.9"
|
||||
* instead of a silent "no match".
|
||||
*
|
||||
* The path is supplied by the caller (we don't recompute it here)
|
||||
* so this function stays pure.
|
||||
*/
|
||||
export function classifyPoMatch(
|
||||
_raw: string,
|
||||
_resolved: string | null,
|
||||
path: OcrMatchMethod,
|
||||
): OcrMatchMethod {
|
||||
return path;
|
||||
}
|
||||
203
src/lib/po-matching.test.ts
Normal file
203
src/lib/po-matching.test.ts
Normal file
|
|
@ -0,0 +1,203 @@
|
|||
import { describe, it, expect } from "vitest";
|
||||
import {
|
||||
levenshteinDistance,
|
||||
levenshteinSimilarity,
|
||||
normalisePoNumber,
|
||||
findBestPoMatch,
|
||||
extractPoCandidate,
|
||||
} from "./po-matching";
|
||||
|
||||
describe("levenshteinDistance", () => {
|
||||
it("returns 0 for identical strings", () => {
|
||||
expect(levenshteinDistance("abc", "abc")).toBe(0);
|
||||
});
|
||||
|
||||
it("returns the length of the other string when one is empty", () => {
|
||||
expect(levenshteinDistance("", "abc")).toBe(3);
|
||||
expect(levenshteinDistance("abc", "")).toBe(3);
|
||||
});
|
||||
|
||||
it("counts single substitution as 1", () => {
|
||||
expect(levenshteinDistance("megcp09239", "megcpo9239")).toBe(1);
|
||||
});
|
||||
|
||||
it("counts transposition of two adjacent chars as 2 (sub + sub)", () => {
|
||||
// "ab" -> "ba" needs 2 substitutions in classic Levenshtein
|
||||
expect(levenshteinDistance("ab", "ba")).toBe(2);
|
||||
});
|
||||
|
||||
it("counts insertion as 1", () => {
|
||||
expect(levenshteinDistance("megcp9239", "megcpo9239")).toBe(1);
|
||||
});
|
||||
|
||||
it("counts deletion as 1", () => {
|
||||
expect(levenshteinDistance("megcpo9239", "megcp9239")).toBe(1);
|
||||
});
|
||||
|
||||
it("is symmetric", () => {
|
||||
expect(levenshteinDistance("abc", "xbc")).toBe(levenshteinDistance("xbc", "abc"));
|
||||
});
|
||||
});
|
||||
|
||||
describe("levenshteinSimilarity", () => {
|
||||
it("returns 1 for identical strings", () => {
|
||||
expect(levenshteinSimilarity("megcpo9239", "megcpo9239")).toBe(1);
|
||||
});
|
||||
|
||||
it("returns 0 for completely different strings", () => {
|
||||
// a='ab', b='cd' -> distance 2, max len 2 -> sim 0
|
||||
expect(levenshteinSimilarity("ab", "cd")).toBe(0);
|
||||
});
|
||||
|
||||
it("returns 1 - distance/maxLen for similar strings", () => {
|
||||
// distance 1, maxLen 10 -> 0.9
|
||||
expect(levenshteinSimilarity("megcp09239", "megcpo9239")).toBeCloseTo(0.9, 5);
|
||||
});
|
||||
|
||||
it("returns 1 when both strings are empty", () => {
|
||||
expect(levenshteinSimilarity("", "")).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("normalisePoNumber", () => {
|
||||
it("strips non-alphanumeric chars", () => {
|
||||
expect(normalisePoNumber("MEGCPO-9239")).toBe("megcpo9239");
|
||||
});
|
||||
|
||||
it("lowercases", () => {
|
||||
expect(normalisePoNumber("MEGCP09239")).toBe("megcp09239");
|
||||
});
|
||||
|
||||
it("returns empty string for null-ish input", () => {
|
||||
expect(normalisePoNumber(null)).toBe("");
|
||||
expect(normalisePoNumber(undefined)).toBe("");
|
||||
expect(normalisePoNumber("")).toBe("");
|
||||
expect(normalisePoNumber(42)).toBe("");
|
||||
});
|
||||
});
|
||||
|
||||
describe("extractPoCandidate", () => {
|
||||
it("returns the cleaned single PO when input has no separator", () => {
|
||||
expect(extractPoCandidate("MEGCP09239")).toBe("MEGCP09239");
|
||||
});
|
||||
|
||||
it("strips a leading #", () => {
|
||||
expect(extractPoCandidate("#MEGCPO9239")).toBe("MEGCPO9239");
|
||||
});
|
||||
|
||||
it("trims surrounding whitespace", () => {
|
||||
expect(extractPoCandidate(" MEGCPO9239 ")).toBe("MEGCPO9239");
|
||||
});
|
||||
|
||||
it("picks the longest alphanumeric token when input has two POs separated by /", () => {
|
||||
// MEGCP09150 (len 10) vs SRU11987 (len 8) -> MEGCP09150 wins
|
||||
expect(extractPoCandidate("MEGCP09150 / SRU 11987")).toBe("MEGCP09150");
|
||||
});
|
||||
|
||||
it("splits on comma, semicolon, newline, pipe", () => {
|
||||
expect(extractPoCandidate("MEGCP09239,FOO1234")).toBe("MEGCP09239");
|
||||
expect(extractPoCandidate("MEGCP09239;FOO1234")).toBe("MEGCP09239");
|
||||
expect(extractPoCandidate("MEGCP09239\nFOO1234")).toBe("MEGCP09239");
|
||||
expect(extractPoCandidate("MEGCP09239|FOO1234")).toBe("MEGCP09239");
|
||||
});
|
||||
|
||||
it("returns empty string when input is null or empty", () => {
|
||||
expect(extractPoCandidate(null)).toBe("");
|
||||
expect(extractPoCandidate(undefined)).toBe("");
|
||||
expect(extractPoCandidate("")).toBe("");
|
||||
});
|
||||
|
||||
it("returns the single token if no token meets length>=6", () => {
|
||||
// No token has length >=6 with digit+letter, but the first token exists
|
||||
// and gets the looser fallback. Here "abc12" is length 5, below minLength,
|
||||
// but tokens[0] fallback uses the first token regardless of length.
|
||||
expect(extractPoCandidate("abc12 def")).toBe("abc12");
|
||||
});
|
||||
|
||||
it("returns empty string when input is only short tokens and no token[0]", () => {
|
||||
// Both tokens below threshold and no fallback possible
|
||||
expect(extractPoCandidate("/")).toBe("");
|
||||
});
|
||||
|
||||
it("does not include tokens that lack digits", () => {
|
||||
// ABCDEF (no digit) is rejected even if longest
|
||||
expect(extractPoCandidate("ABCDEF / PO12345")).toBe("PO12345");
|
||||
});
|
||||
|
||||
it("does not include tokens that lack letters", () => {
|
||||
// 123456 (no letter) is rejected
|
||||
expect(extractPoCandidate("123456 / PO12345")).toBe("PO12345");
|
||||
});
|
||||
|
||||
it("accepts a custom minLength", () => {
|
||||
// minLength=6: short token rejected, long token wins
|
||||
expect(extractPoCandidate("AB / PO12345", { minLength: 6 })).toBe("PO12345");
|
||||
// minLength=3: short token accepted but longer token still wins
|
||||
expect(extractPoCandidate("AB1 / AB12XYZ", { minLength: 3 })).toBe("AB12XYZ");
|
||||
// minLength=10: nothing passes, fallback picks first token
|
||||
expect(extractPoCandidate("PO12345 / AB12", { minLength: 10 })).toBe("PO12345");
|
||||
});
|
||||
});
|
||||
|
||||
describe("findBestPoMatch", () => {
|
||||
const candidates = [
|
||||
{ id: "po1", purchase_order_number: "MEGCPO9239", supplier: "Builders Warehouse" },
|
||||
{ id: "po2", purchase_order_number: "MEGCPO9109", supplier: "Vula Civil" },
|
||||
{ id: "po3", purchase_order_number: "MEGCPO9150", supplier: "Signs R Us" },
|
||||
{ id: "po4", purchase_order_number: "MEGCPO9165", supplier: "Raumix" },
|
||||
{ id: "po5", purchase_order_number: "MEGCPO9240", supplier: "Silverton" },
|
||||
];
|
||||
|
||||
it("returns null when target is empty", () => {
|
||||
expect(findBestPoMatch("", candidates)).toBeNull();
|
||||
expect(findBestPoMatch(null, candidates)).toBeNull();
|
||||
});
|
||||
|
||||
it("returns null when no candidate exceeds the default threshold (0.75)", () => {
|
||||
// "ABC12345" doesn't resemble any candidate strongly
|
||||
expect(findBestPoMatch("ABC12345", candidates)).toBeNull();
|
||||
});
|
||||
|
||||
it("fuzzy-matches a 0↔O OCR error (distance 1, sim 0.9)", () => {
|
||||
const hit = findBestPoMatch("MEGCP09239", candidates);
|
||||
expect(hit).not.toBeNull();
|
||||
expect(hit?.id).toBe("po1");
|
||||
expect(hit?.score).toBeCloseTo(0.9, 5);
|
||||
});
|
||||
|
||||
it("fuzzy-matches a missing O (distance 1, sim 0.9)", () => {
|
||||
// MEGCP09150 -> MEGCPO9150 (missing O between P and 9)
|
||||
const hit = findBestPoMatch("MEGCP09150", candidates);
|
||||
expect(hit).not.toBeNull();
|
||||
expect(hit?.id).toBe("po3");
|
||||
});
|
||||
|
||||
it("lowercase target is normalised", () => {
|
||||
const hit = findBestPoMatch("megcp09239", candidates);
|
||||
expect(hit?.id).toBe("po1");
|
||||
});
|
||||
|
||||
it("strips non-alphanumeric noise", () => {
|
||||
const hit = findBestPoMatch("MEG-CPO-9239", candidates);
|
||||
expect(hit?.id).toBe("po1");
|
||||
});
|
||||
|
||||
it("respects a custom threshold", () => {
|
||||
// MEGCP09234 -> MEGCPO9239 has distance 2, sim 0.8.
|
||||
// Below default 0.85, above default 0.75.
|
||||
expect(findBestPoMatch("MEGCP09234", candidates, { threshold: 0.6 })?.id).toBe("po1");
|
||||
expect(findBestPoMatch("MEGCP09234", candidates, { threshold: 0.85 })).toBeNull();
|
||||
});
|
||||
|
||||
it("returns the highest-scoring candidate when multiple exceed threshold", () => {
|
||||
// Two PO prefixes that are close, only the closest wins
|
||||
const hit = findBestPoMatch("MEGCP09239", candidates);
|
||||
expect(hit?.id).toBe("po1");
|
||||
});
|
||||
|
||||
it("skips candidates with empty purchase_order_number", () => {
|
||||
const noisy = [{ id: "x", purchase_order_number: "", supplier: "" }, ...candidates];
|
||||
const hit = findBestPoMatch("MEGCP09239", noisy);
|
||||
expect(hit?.id).toBe("po1");
|
||||
});
|
||||
});
|
||||
176
src/lib/po-matching.ts
Normal file
176
src/lib/po-matching.ts
Normal file
|
|
@ -0,0 +1,176 @@
|
|||
/**
|
||||
* Pure PO-number matching helpers shared between the OCR pipeline
|
||||
* (n8n workflow) and the OCR-result ingest endpoint.
|
||||
*
|
||||
* Why this lives in one place:
|
||||
* - The same fuzzy logic was previously duplicated in two route files
|
||||
* with subtly different thresholds and different normalisation
|
||||
* rules. OCR errors that one place resolved were leaking through
|
||||
* the other.
|
||||
* - Pulling the algorithms into pure functions lets us unit-test
|
||||
* them without touching PocketBase or n8n.
|
||||
*
|
||||
* What it does:
|
||||
* - normalisePoNumber — strip non-alphanumeric, lowercase. Used as
|
||||
* the equivalence key for "did OCR read the same PO we have on
|
||||
* file?".
|
||||
* - extractPoCandidate — clean OCR's `po_number` field when it
|
||||
* arrives contaminated (leading "#", trailing junk, two POs
|
||||
* separated by "/", etc.). Returns the single best candidate
|
||||
* string.
|
||||
* - findBestPoMatch — given a possibly-messy target and a list of
|
||||
* real POs, return the closest above a similarity threshold (or
|
||||
* null). This is the second-pass fuzzy fallback after exact +
|
||||
* normalised lookup.
|
||||
* - levenshteinDistance / levenshteinSimilarity — primitives used
|
||||
* by findBestPoMatch. Exported for callers that want to roll
|
||||
* their own scoring (e.g. the per-line item matcher in n8n).
|
||||
*/
|
||||
|
||||
/**
|
||||
* Classic Levenshtein edit distance. Iterative two-row implementation:
|
||||
* O(len(a) * len(b)) time, O(min(len(a), len(b))) space.
|
||||
*/
|
||||
export function levenshteinDistance(a: string, b: string): number {
|
||||
if (a === b) return 0;
|
||||
if (a.length === 0) return b.length;
|
||||
if (b.length === 0) return a.length;
|
||||
|
||||
let prev = new Array(b.length + 1);
|
||||
let curr = new Array(b.length + 1);
|
||||
for (let j = 0; j <= b.length; j++) prev[j] = j;
|
||||
|
||||
for (let i = 1; i <= a.length; i++) {
|
||||
curr[0] = i;
|
||||
for (let j = 1; j <= b.length; j++) {
|
||||
const cost = a[i - 1] === b[j - 1] ? 0 : 1;
|
||||
curr[j] = Math.min(
|
||||
curr[j - 1] + 1, // insertion
|
||||
prev[j] + 1, // deletion
|
||||
prev[j - 1] + cost, // substitution
|
||||
);
|
||||
}
|
||||
[prev, curr] = [curr, prev];
|
||||
}
|
||||
return prev[b.length];
|
||||
}
|
||||
|
||||
/**
|
||||
* Levenshtein distance normalised to [0, 1] where 1 = identical and
|
||||
* 0 = completely different (one side empty or max edits equal maxLen).
|
||||
*/
|
||||
export function levenshteinSimilarity(a: string, b: string): number {
|
||||
const maxLen = Math.max(a.length, b.length);
|
||||
if (maxLen === 0) return 1;
|
||||
return 1 - levenshteinDistance(a, b) / maxLen;
|
||||
}
|
||||
|
||||
/**
|
||||
* Equivalence key for a PO number. Strips everything that isn't a
|
||||
* letter or digit and lowercases so "PO-1234" matches "po1234".
|
||||
*
|
||||
* Non-string input (null / undefined / number) returns "" so callers
|
||||
* can safely pass through whatever they get from OCR.
|
||||
*/
|
||||
export function normalisePoNumber(input: unknown): string {
|
||||
if (typeof input !== "string") return "";
|
||||
return input.replace(/[^a-z0-9]/gi, "").toLowerCase();
|
||||
}
|
||||
|
||||
export type PoCandidate = {
|
||||
id: string;
|
||||
purchase_order_number: string;
|
||||
supplier?: string;
|
||||
};
|
||||
|
||||
export type PoMatch = PoCandidate & { score: number };
|
||||
|
||||
export type FindBestPoMatchOptions = {
|
||||
/** Minimum similarity to accept (default 0.75 — handles 1-char edit on 10-char PO). */
|
||||
threshold?: number;
|
||||
};
|
||||
|
||||
/**
|
||||
* Return the candidate PO whose normalised form is closest to the
|
||||
* target, or null if none exceed `threshold`. Both sides are
|
||||
* normalised before comparison so "PO-217", "po217", "PO00217" all
|
||||
* collapse to the same key.
|
||||
*
|
||||
* Used by:
|
||||
* - /api/purchase-orders/by-number/[poNumber]/items — second-pass
|
||||
* fallback when exact lookup misses.
|
||||
* - /api/delivery-notes/[id]/ocr-result — last-resort auto-link
|
||||
* after n8n posts back a raw OCR value.
|
||||
*/
|
||||
export function findBestPoMatch(
|
||||
target: unknown,
|
||||
candidates: PoCandidate[],
|
||||
opts: FindBestPoMatchOptions = {},
|
||||
): PoMatch | null {
|
||||
const threshold = opts.threshold ?? 0.75;
|
||||
const t = normalisePoNumber(target);
|
||||
if (t.length < 4) return null;
|
||||
|
||||
let bestScore = 0;
|
||||
let bestHit: PoCandidate | null = null;
|
||||
for (const c of candidates) {
|
||||
const candidate = normalisePoNumber(c.purchase_order_number);
|
||||
if (candidate.length === 0) continue;
|
||||
const score = levenshteinSimilarity(t, candidate);
|
||||
if (score > bestScore) {
|
||||
bestScore = score;
|
||||
bestHit = c;
|
||||
}
|
||||
}
|
||||
if (!bestHit || bestScore < threshold) return null;
|
||||
return { ...bestHit, score: bestScore };
|
||||
}
|
||||
|
||||
export type ExtractPoCandidateOptions = {
|
||||
/** Minimum length for a token to be considered as a real PO (default 6). */
|
||||
minLength?: number;
|
||||
};
|
||||
|
||||
/**
|
||||
* Clean OCR's `po_number` field. Real-world OCR noise we handle:
|
||||
*
|
||||
* "#MEGCPO9165" leading "#"
|
||||
* "MEGCP09239" missing O (0/O confusion)
|
||||
* "MEGCP09150 / SRU 11987" two POs separated by "/"
|
||||
* "MEGCPO9240 " trailing whitespace
|
||||
* "PO12345\nREF9999" newline separator
|
||||
* null vision LLM couldn't find a PO
|
||||
*
|
||||
* Strategy: split on common separators, keep only alphanumeric
|
||||
* tokens containing both a digit and a letter, pick the longest
|
||||
* (longer PO numbers are less likely to be a supplier reference).
|
||||
* If no token meets the bar, fall back to the first token's cleaned
|
||||
* form — better than dropping the value entirely.
|
||||
*
|
||||
* Always returns a string (empty when nothing usable).
|
||||
*/
|
||||
export function extractPoCandidate(
|
||||
raw: unknown,
|
||||
opts: ExtractPoCandidateOptions = {},
|
||||
): string {
|
||||
if (typeof raw !== "string") return "";
|
||||
const minLength = opts.minLength ?? 6;
|
||||
// Whitespace and the usual separator chars. Multi-PO strings like
|
||||
// "MEGCP09150 / SRU 11987" or "MEGCP09239 SRU 11987" should split on
|
||||
// either form.
|
||||
const tokens = raw
|
||||
.split(/[\s\/,;|\n\r]+/)
|
||||
.map((s) => s.trim())
|
||||
.filter(Boolean);
|
||||
let best = "";
|
||||
for (const t of tokens) {
|
||||
const clean = t.replace(/[^a-zA-Z0-9]/g, "");
|
||||
if (clean.length < minLength) continue;
|
||||
if (!/[0-9]/.test(clean) || !/[a-zA-Z]/.test(clean)) continue;
|
||||
if (clean.length > best.length) best = clean;
|
||||
}
|
||||
if (!best && tokens[0]) {
|
||||
best = tokens[0].replace(/[^a-zA-Z0-9]/g, "");
|
||||
}
|
||||
return best || "";
|
||||
}
|
||||
Loading…
Reference in a new issue