From 16a1cec562a8b9b755cfc3db2d2343775687abb4 Mon Sep 17 00:00:00 2001 From: Dave Date: Sat, 24 Oct 2026 22:52:48 +0200 Subject: [PATCH] feat(pos-match): one size-aware shortlist for the whole app MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The POS shortlist ranked by shared words only, so "BOLT HEX M16X60 GR8.8 ZP" put the M22x65, the M27x170 and the M20x160 all on 0.75 — they share "BOLT 8.8 ZP" — and the M16 the customer asked for never reached the AI. pos-match.ts is now the shared POS matcher: the word pull plus a size-targeted pull (name ~ "M16" || description ~ "M16"), ranked by the size comparison and then by word score. A line with no size information keeps today's word-only behaviour. - shortlistStockItems delegates to it, so the quote assistant gets the better list for free. - the intel popover's fuzzy stock section uses the same ranking, so the right size wins there too. - tokenizeIntel / informativeTokens / combinedScore moved to item-match-text.ts so the matcher and the intel service can share one implementation without an import cycle. PocketBase has no IN operator: every filter here is an OR-chain of ~ and = clauses, escaped with escapeForPbString, capped at 300 rows. Co-Authored-By: Claude Opus 5.5 --- src/lib/item-match-text.ts | 73 ++++ src/lib/server/item-intel-service.ts | 171 ++++----- src/lib/server/pos-match.test.ts | 195 ++++++++++ src/lib/server/pos-match.ts | 354 ++++++++++++++++++ .../server/quote-assistant-shortlist.test.ts | 26 +- src/lib/server/quote-assistant-shortlist.ts | 146 ++------ src/lib/types.ts | 25 +- 7 files changed, 767 insertions(+), 223 deletions(-) create mode 100644 src/lib/item-match-text.ts create mode 100644 src/lib/server/pos-match.test.ts create mode 100644 src/lib/server/pos-match.ts diff --git a/src/lib/item-match-text.ts b/src/lib/item-match-text.ts new file mode 100644 index 0000000..f9b32e4 --- /dev/null +++ b/src/lib/item-match-text.ts @@ -0,0 +1,73 @@ +/** + * item-match-text — the pure text scoring behind the POS matcher. + * + * These three helpers used to live in `item-intel-service.ts`. They + * moved here so that `pos-match.ts` (the shared, size-aware POS + * matcher) and `item-intel-service.ts` (the ⓘ popover) can share one + * implementation without importing each other. + * + * Pure and client-safe: no PocketBase, no AI, no DOM. + */ + +import { jaccard } from "@/lib/server/order-quote-match"; + +/** Tokenizer tuned for item-intel fuzzy matching. + * + * Differs from `order-quote-match.tokenize` in three ways that + * matter for catalogue fuzziness: + * + * 1. Keeps short numeric tokens ("50", "20L", "M20") — these + * carry product identity (sizes, capacities, model numbers). + * Dropping them made every SAE oil tie at overlap=1.0 and + * ruined price_list matching against the AFRIQ SKUs. + * 2. Smaller stopword set — only tokens that are pure noise + * across every catalogue. No "mm" / "qty" filter because + * "qty" doesn't appear in our descriptions and "mm" only + * does so meaningfully (e.g. "M6 X 30MM" — keep it). + * 3. Allows single-character numeric tokens ("L", "5") so + * compound unit strings like "20LT" or "5L" tokenise as + * one unit token (the regex matches the whole run, not + * per-character). + */ +const INTEL_STOPWORDS = new Set([ + "the", "and", "with", "for", "of", "to", "a", "an", "in", "on", "at", + "by", "from", "as", "is", "or", "x", +]); + +export function tokenizeIntel(s: string): Set { + const out = new Set(); + if (!s) return out; + for (const tok of String(s).toLowerCase().match(/[a-z0-9]+/g) || []) { + if (INTEL_STOPWORDS.has(tok)) continue; + out.add(tok); + } + return out; +} + +/** Overlap coefficient: |A ∩ B| / min(|A|, |B|). + * Better than Jaccard for "short query against a rich candidate" — + * doesn't penalise extra tokens on the candidate side. Used for + * ranking against the price_list `search_text` blob. */ +export function overlapScore(a: Set, b: Set): number { + if (a.size === 0 || b.size === 0) return 0; + let inter = 0; + for (const t of a) if (b.has(t)) inter++; + return inter / Math.min(a.size, b.size); +} + +/** Drop stopwords + short tokens. */ +const INFORMATIVE_MIN_LEN = 3; + +/** Pick the top N tokens by length desc (most distinctive first). */ +export function informativeTokens(s: string, max = 4): string[] { + const toks = tokenizeIntel(s || ""); + return [...toks] + .filter((t) => !INTEL_STOPWORDS.has(t) && t.length >= INFORMATIVE_MIN_LEN) + .sort((a, b) => b.length - a.length) + .slice(0, max); +} + +/** Combined Jaccard + overlap score, capped at 1. */ +export function combinedScore(a: Set, b: Set): number { + return Math.max(jaccard(a, b), overlapScore(a, b)); +} diff --git a/src/lib/server/item-intel-service.ts b/src/lib/server/item-intel-service.ts index 725f47f..bf24b14 100644 --- a/src/lib/server/item-intel-service.ts +++ b/src/lib/server/item-intel-service.ts @@ -72,12 +72,19 @@ */ import { getAdminPocketBase } from "@/lib/server/pocketbase-admin"; -import { jaccard } from "@/lib/server/order-quote-match"; import { escapeForPbString, normaliseSku } from "@/lib/server/price-list-match"; +import { + combinedScore, + informativeTokens, + overlapScore, + tokenizeIntel, +} from "@/lib/item-match-text"; +import { loadSuppliersById, shortlistPosItems } from "@/lib/server/pos-match"; import type { ItemIntel, ItemIntelAnchor, ItemIntelMatchKind, + ItemIntelPosPick, ItemIntelPriceListRow, ItemIntelQuoteRow, ItemIntelOrderRow, @@ -87,50 +94,20 @@ import type { } from "@/lib/types"; import { ITEM_INTEL_LIMITS } from "@/lib/types"; +export { + combinedScore, + informativeTokens, + tokenizeIntel, +} from "@/lib/item-match-text"; +export { loadSuppliersById } from "@/lib/server/pos-match"; + // ─── Pure scoring helpers ──────────────────────────────────────────── - -/** Tokenizer tuned for item-intel fuzzy matching. - * - * Differs from `order-quote-match.tokenize` in three ways that - * matter for catalogue fuzziness: - * - * 1. Keeps short numeric tokens ("50", "20L", "M20") — these - * carry product identity (sizes, capacities, model numbers). - * Dropping them made every SAE oil tie at overlap=1.0 and - * ruined price_list matching against the AFRIQ SKUs. - * 2. Smaller stopword set — only tokens that are pure noise - * across every catalogue. No "mm" / "qty" filter because - * "qty" doesn't appear in our descriptions and "mm" only - * does so meaningfully (e.g. "M6 X 30MM" — keep it). - * 3. Allows single-character numeric tokens ("L", "5") so - * compound unit strings like "20LT" or "5L" tokenise as - * one unit token (the regex matches the whole run, not - * per-character). - */ -const INTEL_STOPWORDS = new Set([ - "the", "and", "with", "for", "of", "to", "a", "an", "in", "on", "at", - "by", "from", "as", "is", "or", "x", -]); -export function tokenizeIntel(s: string): Set { - const out = new Set(); - if (!s) return out; - for (const tok of String(s).toLowerCase().match(/[a-z0-9]+/g) || []) { - if (INTEL_STOPWORDS.has(tok)) continue; - out.add(tok); - } - return out; -} - -/** Overlap coefficient: |A ∩ B| / min(|A|, |B|). - * Better than Jaccard for "short query against a rich candidate" — - * doesn't penalise extra tokens on the candidate side. Used for - * ranking against the price_list `search_text` blob. */ -function overlapScore(a: Set, b: Set): number { - if (a.size === 0 || b.size === 0) return 0; - let inter = 0; - for (const t of a) if (b.has(t)) inter++; - return inter / Math.min(a.size, b.size); -} +// +// `tokenizeIntel`, `informativeTokens` and `combinedScore` moved to +// `@/lib/item-match-text` so the POS matcher (`pos-match.ts`) and this +// service can share one implementation without importing each other. +// They are re-exported above for the callers that still import them +// from here. /** Days between `iso` and `now`. Negative clamped to 0. */ function daysAgo(iso: string | null | undefined): number { @@ -141,18 +118,6 @@ function daysAgo(iso: string | null | undefined): number { return diffMs <= 0 ? 0 : Math.floor(diffMs / 86_400_000); } -/** Drop stopwords + short tokens. */ -const INFORMATIVE_MIN_LEN = 3; - -/** Pick the top N tokens by length desc (most distinctive first). */ -export function informativeTokens(s: string, max = 4): string[] { - const toks = tokenizeIntel(s || ""); - return [...toks] - .filter((t) => !INTEL_STOPWORDS.has(t) && t.length >= INFORMATIVE_MIN_LEN) - .sort((a, b) => b.length - a.length) - .slice(0, max); -} - /** PocketBase `~` is case-insensitive substring match. OR'ing the top * informative tokens keeps the candidate set broad while letting * the in-app Jaccard/overlap rank with precision. */ @@ -162,11 +127,6 @@ function buildPbContainsOR(field: string, targetDesc: string, maxTokens = 4): st return toks.map((t) => `${field} ~ "${t}"`).join(" || "); } -/** Combined Jaccard + overlap score, capped at 1. */ -export function combinedScore(a: Set, b: Set): number { - return Math.max(jaccard(a, b), overlapScore(a, b)); -} - // ─── PB I/O helpers ─────────────────────────────────────────────────── type PbListResult = { items: T[]; totalItems?: number }; @@ -189,17 +149,6 @@ async function safeGetList( } } -export async function loadSuppliersById(pb: Awaited>): Promise> { - try { - const rows = await pb - .collection("suppliers") - .getFullList<{ id: string; suppliername: string }>({ fields: "id,suppliername", batch: 500 }); - return new Map(rows.map((s) => [s.id, s.suppliername])); - } catch { - return new Map(); - } -} - // ─── The 5 sections ────────────────────────────────────────────────── /** Columns we read from `stock_items` for the stock list. */ @@ -299,9 +248,10 @@ async function resolveAnchorStockItem( async function buildStock( pb: Awaited>, - target: { id: string; description: string }, + target: { id: string; description: string; sku?: string }, supplierNameById: Map, anchor?: ItemIntelAnchor, + matchKindOverride?: ItemIntelMatchKind, ): Promise<{ matches: ItemIntelStockRow[]; purchaseHistory: Array<{ stock_item_id: string; stock_item_sku: string; @@ -315,48 +265,62 @@ async function buildStock( purchase_number: string; }>; matchKind: ItemIntelMatchKind }> { const anchored = await resolveAnchorStockItem(pb, anchor); - const matchKind: ItemIntelMatchKind = anchored ? anchored.matchKind : "fuzzy"; + const matchKind: ItemIntelMatchKind = + matchKindOverride ?? (anchored ? anchored.matchKind : "fuzzy"); - const filter = buildPbContainsOR("description", target.description); - let candRows: PbListResult = { items: [] }; - if (filter) { - candRows = await safeGetList(pb, "stock_items", { - filter, - fields: STOCK_ITEM_FIELDS, - max: 500, - }); - } + // The fuzzy section uses the shared SIZE-AWARE shortlist, so the + // right-size row wins even without an AI pick. Before this, the + // ranking was by shared words only and every bolt tied on + // "BOLT 8.8 ZP". + const candidates = await shortlistPosItems( + pb, + { description: target.description, sku: target.sku }, + ITEM_INTEL_LIMITS.stock + PURCHASE_SCAN_CAP, + supplierNameById, + ); - const targetTokens = tokenizeIntel(target.description); - const rankedAll = candRows.items - .map((si) => ({ si, score: combinedScore(targetTokens, tokenizeIntel(si.description || "")) })) - .filter((r) => r.score >= 0.2) - .sort((a, b) => b.score - a.score) - // The anchored row is prepended separately — drop it from the fuzzy - // list so it can't appear twice. - .filter((r) => !anchored || r.si.id !== anchored.row.id); + // The anchored row is prepended separately — drop it from the fuzzy + // list so it can't appear twice. + const rankedAll: Array<{ + id: string; name: string; sku: string; description: string; + current_qty?: number; min_qty?: number | null; + cost_price: number; selling_price: number; supplier: string; score: number; + }> = candidates + .filter((c) => !anchored || c.id !== anchored.row.id) + .map((c) => ({ + id: c.id, + name: c.name, + sku: c.sku, + description: c.description, + current_qty: c.current_qty, + min_qty: c.min_qty ?? null, + cost_price: c.cost_price, + selling_price: c.selling_price, + supplier: c.supplier_id, + score: c.score, + })); // Anchored row first (score 1 — it IS the match), then the best // fuzzy candidates in the remaining slots. const forDisplay = anchored - ? [{ si: anchored.row, score: 1 }, ...rankedAll].slice(0, ITEM_INTEL_LIMITS.stock) + ? [{ ...anchored.row, score: 1 }, ...rankedAll].slice(0, ITEM_INTEL_LIMITS.stock) : rankedAll.slice(0, ITEM_INTEL_LIMITS.stock); // The purchase walk always covers the anchored row so its supplier // shows up in `supplier_history` even when the fuzzy ranking missed // it entirely. const forPurchases = anchored - ? [{ si: anchored.row, score: 1 }, ...rankedAll].slice(0, PURCHASE_SCAN_CAP) + ? [{ ...anchored.row, score: 1 }, ...rankedAll].slice(0, PURCHASE_SCAN_CAP) : rankedAll.slice(0, PURCHASE_SCAN_CAP); - const matches: ItemIntelStockRow[] = forDisplay.map(({ si, score }) => { + const matches: ItemIntelStockRow[] = forDisplay.map((si) => { const current_qty = si.current_qty || 0; const min_qty = si.min_qty ?? null; let status: ItemIntelStockRow["status"] = "out"; if (current_qty > 0) status = min_qty != null && current_qty <= min_qty ? "low" : "in_stock"; return { stock_item_id: si.id, - score: +score.toFixed(3), + score: +si.score.toFixed(3), name: si.name || "", sku: si.sku || "", description: si.description || "", @@ -376,7 +340,7 @@ async function buildStock( cost_price: number; quantity: number; purchased_at: string; days_ago: number; purchase_number: string; }> = (await Promise.all( - forPurchases.map(async ({ si }) => { + forPurchases.map(async (si) => { try { const spiRows = await pb .collection("stock_purchase_items") @@ -681,7 +645,7 @@ function composeSummary(input: { export async function populateItemIntel( itemId: string, collection: "quote_items" | "order_items", - options: { write?: boolean; anchor?: ItemIntelAnchor } = {}, + options: { write?: boolean; anchor?: ItemIntelAnchor; posPick?: ItemIntelPosPick } = {}, ): Promise { const write = options.write !== false; const pb = await getAdminPocketBase(); @@ -702,8 +666,18 @@ export async function populateItemIntel( const supplierNameById = await loadSuppliersById(pb); const generated_at = new Date().toISOString(); + // An AI pick pins the stock row it chose. It outranks any other + // anchor, and it is what `match_kind: "pos_pick"` reports. + const posPick = options.posPick; + const anchor: ItemIntelAnchor | undefined = posPick?.stock_item_id + ? { stockItemId: posPick.stock_item_id } + : options.anchor; + const matchKindOverride: ItemIntelMatchKind | undefined = posPick?.stock_item_id + ? "pos_pick" + : undefined; + const [stockRaw, price_list, recent_quote, recent_order] = await Promise.all([ - buildStock(pb, target, supplierNameById, options.anchor), + buildStock(pb, target, supplierNameById, anchor, matchKindOverride), buildPriceList(pb, target), buildQuoteHistory(pb, target, supplierNameById), buildOrderHistory(pb, target, supplierNameById), @@ -741,6 +715,7 @@ export async function populateItemIntel( recent_quote, recent_order, match_kind: stockRaw.matchKind, + ...(posPick ? { pos_pick: posPick } : {}), }; if (write) { diff --git a/src/lib/server/pos-match.test.ts b/src/lib/server/pos-match.test.ts new file mode 100644 index 0000000..74303c5 --- /dev/null +++ b/src/lib/server/pos-match.test.ts @@ -0,0 +1,195 @@ +import { describe, expect, it, vi } from "vitest"; + +import { shortlistPosItems, parseCandidateSize, type AdminPb } from "./pos-match"; + +type Row = { + id: string; + sku: string; + name: string; + description: string; + notes?: string; + unit?: string; + cost_price?: number; + selling_price?: number; + markup?: number; + supplier?: string; + current_qty?: number; + min_qty?: number; +}; + +/** + * A PocketBase mock that answers each pull with the rows that pull + * would really return — the word pull, the SKU pull and the size pull + * all get their own set, which is what makes the ranking testable. + */ +function makePb(pulls: { word?: Row[]; sku?: Row[]; size?: Row[]; kind?: Row[] }) { + const getList = vi.fn( + async (_page: number, _per: number, opts: { filter?: string; fields?: string }) => { + const filter = opts.filter || ""; + if (filter.includes('sku = "')) return { items: pulls.sku || [] }; + // The size pull asks for the thread/dimension verbatim ("M16"). + if (filter.includes('~ "M16"') || filter.includes('~ "150mm"')) { + return { items: pulls.size || [] }; + } + // The kind pull is the only one that asks for a single word. + if (/^is_active = true && \(name ~ "[a-z]+" \|\| description ~ "[a-z]+"\)$/.test(filter)) { + return { items: pulls.kind || [] }; + } + return { items: pulls.word || [] }; + }, + ); + const getFullList = vi.fn(async () => [ + { id: "sup1", suppliername: "Bolt Supplies" }, + { id: "sup2", suppliername: "Other Co" }, + ]); + const collection = vi.fn((name: string) => { + if (name === "stock_items") return { getList }; + if (name === "suppliers") return { getFullList }; + throw new Error(`unexpected collection ${name}`); + }); + return { pb: { collection } as unknown as AdminPb, collection, getList, getFullList }; +} + +const M16: Row = { + id: "m16", + sku: "BOLT-M16X60", + name: "HEX BOLT M16X60 GR8.8 ZP", + description: "HEX BOLT M16X60 GR8.8 ZP", + unit: "ea", + cost_price: 4, + selling_price: 6, + markup: 1.5, + supplier: "sup1", +}; + +const M22: Row = { + id: "m22", + sku: "BOLT-M22X65", + name: "HEX BOLT M22X65 GR8.8 ZP", + description: "HEX BOLT M22X65 GR8.8 ZP", + unit: "ea", + cost_price: 8, + selling_price: 12, + markup: 1.5, + supplier: "sup1", +}; + +describe("shortlistPosItems — the size-aware ranking", () => { + it("ranks the M16x60 row above the M22x65 even when the M22 shares more words", async () => { + const { pb } = makePb({ word: [M22, M16], size: [M16] }); + const out = await shortlistPosItems(pb, { description: "BOLT HEX M16X60 GR8.8 ZP" }); + expect(out.map((c) => c.id)).toEqual(["m16", "m22"]); + expect(out[0].sizeCompare.rank).toBe(0); + expect(out[0].sizeCompare.exact).toBe(true); + }); + + it("says what differs on the wrong candidate", async () => { + const { pb } = makePb({ word: [M22], size: [] }); + const out = await shortlistPosItems(pb, { description: "BOLT HEX M16X60 GR8.8 ZP" }); + expect(out[0].sizeCompare.differs).toContain("thread M22 vs M16"); + expect(out[0].sizeCompare.differs).toContain("length 65 vs 60"); + }); + + it("pulls the rows of the line's own size even when they share no words", async () => { + const { pb, getList } = makePb({ word: [], size: [M16] }); + const out = await shortlistPosItems(pb, { description: "BOLT HEX M16X60 GR8.8 ZP" }); + const sizePull = getList.mock.calls.find((c) => (c[2].filter || "").includes('~ "M16"')); + expect(sizePull).toBeTruthy(); + expect(sizePull![2].filter).toContain('name ~ "M16"'); + expect(sizePull![2].filter).toContain('description ~ "M16"'); + expect(sizePull![1]).toBeLessThanOrEqual(300); + expect(out.map((c) => c.id)).toEqual(["m16"]); + }); + + it("keeps an exact SKU hit first", async () => { + const skuRow: Row = { id: "exact", sku: "HB-10/50", name: "Bolt", description: "something else" }; + const { pb } = makePb({ word: [M22], sku: [skuRow] }); + const out = await shortlistPosItems(pb, { + description: "BOLT HEX M16X60 GR8.8 ZP", + sku: "HB 10 50", + }); + expect(out[0].id).toBe("exact"); + expect(out[0].score).toBe(1); + }); + + it("falls back to the word score for a line with no size information", async () => { + const rows: Row[] = [ + { id: "a", sku: "A", name: "widget blue", description: "widget blue" }, + { id: "b", sku: "B", name: "widget blue large", description: "widget blue large" }, + ]; + const { pb } = makePb({ word: rows }); + const out = await shortlistPosItems(pb, { description: "widget blue" }); + expect(out.length).toBe(2); + for (let i = 1; i < out.length; i++) { + expect(out[i - 1].score).toBeGreaterThanOrEqual(out[i].score); + } + }); + + it("never uses the IN operator — PocketBase has none", async () => { + const { pb, getList } = makePb({ word: [M16], size: [M16] }); + await shortlistPosItems(pb, { description: "BOLT HEX M16X60 GR8.8 ZP", sku: "BOLT-M16X60" }); + for (const call of getList.mock.calls) { + expect(call[2].filter).not.toMatch(/\bIN\b/); + } + }); + + it("escapes the strings that go into a filter", async () => { + const { pb, getList } = makePb({ word: [] }); + await shortlistPosItems(pb, { description: 'say "hello"', sku: 'AB"12\\3' }); + for (const call of getList.mock.calls) { + const filter = call[2].filter || ""; + expect(filter).not.toContain("\\"); + expect(filter).not.toMatch(/AB"|AB\\"/); + } + }); + + it("caps every pull and asks for explicit fields", async () => { + const { pb, getList } = makePb({ word: [M16], size: [M16] }); + await shortlistPosItems(pb, { description: "BOLT HEX M16X60 GR8.8 ZP" }); + for (const call of getList.mock.calls) { + expect(call[1]).toBeLessThanOrEqual(300); + expect(call[2].fields).toContain("selling_price"); + } + }); + + it("returns the size and the comparison with every candidate", async () => { + const { pb } = makePb({ word: [M16] }); + const out = await shortlistPosItems(pb, { description: "BOLT HEX M16X60 GR8.8 ZP" }); + expect(out[0].size).toMatchObject({ kind: "bolt", thread: "M16", length: 60, grade: "8.8" }); + expect(out[0].sizeCompare).toEqual({ exact: true, differs: [], rank: 0 }); + }); + + it("returns nothing when nothing matches", async () => { + const { pb } = makePb({ word: [] }); + expect(await shortlistPosItems(pb, { description: "" })).toEqual([]); + }); +}); + +describe("parseCandidateSize", () => { + it("reads the size from the name", () => { + expect(parseCandidateSize({ name: "HEX BOLT M16X60 GR8.8 ZP" })).toMatchObject({ + kind: "bolt", + thread: "M16", + length: 60, + grade: "8.8", + finish: "zp", + }); + }); + + it("takes the length from the description when the name has none", () => { + const size = parseCandidateSize({ + name: "BOLT M16", + description: "HEX BOLT M16X60 GR8.8 ZP", + }); + expect(size.thread).toBe("M16"); + expect(size.length).toBe(60); + expect(size.grade).toBe("8.8"); + }); + + it("falls back to the SKU", () => { + expect(parseCandidateSize({ name: "Bolt", sku: "M16X60" })).toMatchObject({ + thread: "M16", + length: 60, + }); + }); +}); diff --git a/src/lib/server/pos-match.ts b/src/lib/server/pos-match.ts new file mode 100644 index 0000000..7c51869 --- /dev/null +++ b/src/lib/server/pos-match.ts @@ -0,0 +1,354 @@ +/** + * pos-match — the shared, SIZE-AWARE POS matcher. + * + * One candidate pool, one ranking, one AI pick — used by the quote + * assistant, the order fill and the blue price check. + * + * ## Why size matters + * + * The shortlist used to rank by *shared words* only. For + * "BOLT HEX M16X60 GR8.8 ZP" every bolt in the catalogue shares + * "BOLT 8.8 ZP" and scores 0.75, so the M22x65, the M27x170 and the + * M20x160 all tied for first and the M16 the customer actually asked + * for never reached the AI. The price check then showed the price of + * the wrong item. + * + * So the pool is now built from two pulls: + * + * 1. the word pull (as before) — PB `~` OR'd over the top + * informative tokens of the line; + * 2. a SIZE pull — `name ~ "M16" || description ~ "M16"` — so the + * same-size rows enter the pool even when they share almost no + * words with the line. + * + * The ranking is then: an exact/normalised SKU hit first, then the + * size comparison (`compareSizes`), then the word score. A line with + * no size information at all keeps today's word-only behaviour. + * + * ## Filters + * + * PocketBase has NO `IN` operator — an id list is an OR-chain. Only + * `=`, `!=` and `~` are used here. Every string that goes into a + * filter is escaped with `escapeForPbString`. + */ + +import { getAdminPocketBase } from "@/lib/server/pocketbase-admin"; +import { escapeForPbString, normaliseSku } from "@/lib/server/price-list-match"; +import { + compareSizes, + KIND_WORDS, + parseItemSize, + SIZE_RANK_THREAD, + type ItemSize, + type SizeCompare, +} from "@/lib/item-size"; +import { combinedScore, informativeTokens, tokenizeIntel } from "@/lib/item-match-text"; + +export type AdminPb = Awaited>; + +/** One POS row offered to the AI as a possible match for a line. */ +export interface StockCandidate { + id: string; + name: string; + sku: string; + description: string; + notes: string; + unit: string; + cost_price: number; + selling_price: number; + markup: number; + supplier_id: string; + supplier_name: string; + score: number; + /** On hand — the intel popover shows the stock status. */ + current_qty?: number; + min_qty?: number; + /** The size parsed out of the candidate's text (present when the + * candidate came from `shortlistPosItems`). */ + size?: ItemSize; + /** How the candidate's size compares to the line's. */ + sizeCompare?: SizeCompare; +} + +/** A candidate that came from the size-aware shortlist. */ +export interface PosCandidate extends StockCandidate { + size: ItemSize; + sizeCompare: SizeCompare; +} + +/** How sure the AI is that a candidate is the same product. */ +export type PosMatchKind = "exact" | "closest" | "none"; + +/** What the AI picked for one line. */ +export interface PosPick { + line_id: string; + stock_item_id: string | null; + match: PosMatchKind; + /** What differs between the line and the picked item. */ + differs: string[]; + confidence: "high" | "medium" | "low" | null; + reason: string; +} + +// ─── Tunables ────────────────────────────────────────────────────────── + +const STOCK_FIELDS = + "id,sku,name,description,notes,unit,cost_price,selling_price,markup,supplier,current_qty,min_qty"; +/** The broad word pull. */ +const WORD_FETCH_LIMIT = 300; +/** The narrow SKU pull, so the SKU row is never crowded out. */ +const SKU_FETCH_LIMIT = 50; +/** The size-targeted pull. */ +const SIZE_FETCH_LIMIT = 300; +/** A candidate below this word score is dropped… unless its size + * matches, which is the whole point of the size pull. */ +const MIN_SCORE = 0.2; +/** How many informative tokens the word pull searches for. */ +const MAX_TOKENS = 4; +/** Lines per AI call in `pickPosItems`. */ +export const POS_PICK_MAX_LINES = 15; +/** How many candidates the AI sees per line. */ +export const POS_PICK_CANDIDATES = 10; + +type StockRow = { + id: string; + sku?: string; + name?: string; + description?: string; + notes?: string; + unit?: string; + cost_price?: number; + selling_price?: number; + markup?: number; + supplier?: string; + current_qty?: number; + min_qty?: number; +}; + +// ─── Supplier names ─────────────────────────────────────────────────── + +/** Supplier id → supplier name, for the candidates' `supplier_name`. */ +export async function loadSuppliersById(pb: AdminPb): Promise> { + try { + const rows = await pb + .collection("suppliers") + .getFullList<{ id: string; suppliername: string }>({ fields: "id,suppliername", batch: 500 }); + return new Map(rows.map((s) => [s.id, s.suppliername])); + } catch { + return new Map(); + } +} + +// ─── The candidate pool ─────────────────────────────────────────────── + +/** + * Pull the POS rows that could match a line. + * + * Three pulls, merged by id: + * - the word pull (today's pull); + * - the narrow SKU pull (when the line has a SKU); + * - the size pull (when the line has a thread or a dimension). + */ +async function pullCandidates( + pb: AdminPb, + line: { description: string; sku?: string; notes?: string }, + size: ItemSize, +): Promise> { + const rowsById = new Map(); + + const add = (rows: StockRow[]) => { + for (const row of rows) if (!rowsById.has(row.id)) rowsById.set(row.id, row); + }; + + // 1. The word pull. + const tokens = informativeTokens(line.description || "", MAX_TOKENS); + const wordClauses: string[] = []; + for (const t of tokens) { + const v = escapeForPbString(t); + wordClauses.push(`name ~ "${v}"`, `description ~ "${v}"`, `sku ~ "${v}"`); + } + + // `"` and `\` can't be escaped reliably in a PB filter string and never + // matter to a SKU match (normaliseSku drops them anyway). + const rawSku = (line.sku || "").replace(/[\\"]/g, "").trim(); + + if (wordClauses.length > 0) { + const res = await pb.collection("stock_items").getList(1, WORD_FETCH_LIMIT, { + filter: `is_active = true && (${wordClauses.join(" || ")})`, + fields: STOCK_FIELDS, + }); + add(res.items); + } + + // 2. The narrow SKU pull — the SKU row must never be crowded out by + // the broad (and unsorted) word pull. + if (rawSku) { + const skuClauses = [`sku = "${escapeForPbString(rawSku)}"`]; + const wantedSku = normaliseSku(rawSku); + if (wantedSku) skuClauses.push(`sku ~ "${escapeForPbString(wantedSku)}"`); + const narrow = await pb.collection("stock_items").getList(1, SKU_FETCH_LIMIT, { + filter: `is_active = true && (${skuClauses.join(" || ")})`, + fields: STOCK_FIELDS, + }); + add(narrow.items); + } + + // 3. The size pull — the rows of the right size, however few words + // they share with the line. + const sizeClauses = sizeNeedleClauses(size); + if (sizeClauses.length > 0) { + const res = await pb.collection("stock_items").getList(1, SIZE_FETCH_LIMIT, { + filter: `is_active = true && (${sizeClauses.join(" || ")})`, + fields: STOCK_FIELDS, + }); + add(res.items); + } + + // 4. The kind pull — only when the word pull did not already cover + // the kind word (it usually does). + const kindWord = size.kind ? KIND_WORDS[size.kind] : ""; + if (kindWord && !tokens.includes(kindWord)) { + const v = escapeForPbString(kindWord); + const res = await pb.collection("stock_items").getList(1, SIZE_FETCH_LIMIT, { + filter: `is_active = true && (name ~ "${v}" || description ~ "${v}")`, + fields: STOCK_FIELDS, + }); + add(res.items); + } + + return rowsById; +} + +/** + * The filter clauses that pull the rows of the line's own size. + * + * `name ~ "M16" || description ~ "M16" || sku ~ "M16"` for a metric + * thread, `… ~ "150mm"` for a dimension. + */ +function sizeNeedleClauses(size: ItemSize): string[] { + const clauses: string[] = []; + const needles: string[] = []; + if (size.thread) needles.push(size.thread); + for (const dim of size.dims) needles.push(`${dim.value}${dim.unit}`); + for (const needle of needles.slice(0, 2)) { + const v = escapeForPbString(needle); + clauses.push(`name ~ "${v}"`, `description ~ "${v}"`, `sku ~ "${v}"`); + } + return clauses; +} + +// ─── The candidate's own size ───────────────────────────────────────── + +/** + * The size of a POS row. + * + * The name, the description and the SKU are parsed separately and + * merged attribute by attribute (the name wins) — a terse name + * ("BOLT M16") must not hide the length in the description + * ("M16X60 GR8.8 ZP"). + */ +export function parseCandidateSize(row: { + name?: string; + description?: string; + sku?: string; +}): ItemSize { + const parts = [ + parseItemSize(row.name || ""), + parseItemSize(row.description || ""), + parseItemSize(row.sku || ""), + ]; + const first = (pick: (s: ItemSize) => T | null): T | null => { + for (const p of parts) { + const v = pick(p); + if (v !== null && v !== undefined) return v; + } + return null; + }; + const dims: ItemSize["dims"] = []; + for (const p of parts) { + for (const dim of p.dims) { + if (!dims.some((d) => d.value === dim.value && d.unit === dim.unit)) dims.push(dim); + } + } + return { + kind: first((s) => s.kind), + variant: first((s) => s.variant), + thread: first((s) => s.thread), + length: first((s) => s.length), + grade: first((s) => s.grade), + finish: first((s) => s.finish), + dims, + }; +} + +// ─── The shortlist ──────────────────────────────────────────────────── + +/** + * The size-aware shortlist for one line. + * + * - an exact/normalised SKU hit always wins (score 1); + * - then the size comparison — same kind + thread + length first, + * then the same kind + thread with the nearest length, then the + * same thread; + * - then the word score, as before. + * + * A line with no size information keeps today's word-only behaviour. + */ +export async function shortlistPosItems( + pb: AdminPb, + line: { description: string; sku?: string; notes?: string }, + limit = POS_PICK_CANDIDATES, + supplierNameById?: Map, +): Promise { + const lineSize = parseItemSize(line.description || ""); + const wantedSku = normaliseSku((line.sku || "").replace(/[\\"]/g, "").trim()); + + const rowsById = await pullCandidates(pb, line, lineSize); + if (rowsById.size === 0) return []; + + const lineTokens = tokenizeIntel(line.description || ""); + const scored = [...rowsById.values()].map((row) => { + const skuHit = wantedSku !== "" && normaliseSku(row.sku || "") === wantedSku; + const score = skuHit + ? 1 + : Math.max( + combinedScore(lineTokens, tokenizeIntel(row.name || "")), + combinedScore(lineTokens, tokenizeIntel(row.description || "")), + ); + const size = parseCandidateSize(row); + const sizeCompare = compareSizes(lineSize, size); + return { row, score, skuHit, size, sizeCompare }; + }); + + const kept = scored + .filter((r) => r.skuHit || r.score >= MIN_SCORE || r.sizeCompare.rank <= SIZE_RANK_THREAD) + .sort( + (a, b) => + Number(b.skuHit) - Number(a.skuHit) || + a.sizeCompare.rank - b.sizeCompare.rank || + b.score - a.score, + ) + .slice(0, limit); + + if (kept.length === 0) return []; + + const suppliers = supplierNameById ?? (await loadSuppliersById(pb)); + return kept.map(({ row, score, size, sizeCompare }) => ({ + id: row.id, + name: row.name || "", + sku: row.sku || "", + description: row.description || "", + notes: row.notes || "", + unit: row.unit || "", + cost_price: row.cost_price || 0, + selling_price: row.selling_price || 0, + markup: row.markup || 0, + supplier_id: row.supplier || "", + supplier_name: suppliers.get(row.supplier || "") || "", + score: +score.toFixed(3), + current_qty: row.current_qty || 0, + min_qty: row.min_qty ?? undefined, + size, + sizeCompare, + })); +} diff --git a/src/lib/server/quote-assistant-shortlist.test.ts b/src/lib/server/quote-assistant-shortlist.test.ts index e46587f..7cdac4d 100644 --- a/src/lib/server/quote-assistant-shortlist.test.ts +++ b/src/lib/server/quote-assistant-shortlist.test.ts @@ -43,7 +43,8 @@ describe("shortlistStockItems", () => { it("filters active rows and ORs name/description/sku per informative token", async () => { const { pb, getList } = makePb([NUT]); await shortlistStockItems(pb, { description: "M18 hex nut grade 8" }); - expect(getList).toHaveBeenCalledTimes(1); + // The word pull and the size pull (the line names the thread M18). + expect(getList).toHaveBeenCalledTimes(2); const opts = getList.mock.calls[0][2]; expect(opts.filter!.startsWith("is_active = true && (")).toBe(true); for (const tok of ["hex", "nut", "grade"]) { @@ -51,8 +52,19 @@ describe("shortlistStockItems", () => { expect(opts.filter).toContain(`description ~ "${tok}"`); expect(opts.filter).toContain(`sku ~ "${tok}"`); } - expect(opts.fields).toBe("id,sku,name,description,notes,unit,cost_price,selling_price,markup,supplier"); - expect(getList.mock.calls[0][1]).toBe(500); + expect(opts.fields).toBe( + "id,sku,name,description,notes,unit,cost_price,selling_price,markup,supplier,current_qty,min_qty", + ); + expect(getList.mock.calls[0][1]).toBe(300); + }); + + it("also pulls the rows of the line's own size", async () => { + const { pb, getList } = makePb([NUT]); + await shortlistStockItems(pb, { description: "M18 hex nut grade 8" }); + const sizePull = getList.mock.calls[1][2].filter!; + expect(sizePull).toContain('name ~ "M18"'); + expect(sizePull).toContain('description ~ "M18"'); + expect(sizePull).toContain('sku ~ "M18"'); }); it("returns only rows that match the description", async () => { @@ -127,10 +139,14 @@ describe("shortlistStockItems", () => { expect(out.map((r) => r.id).filter((id) => id === "sku1")).toHaveLength(1); }); - it("makes no narrow query without a SKU", async () => { + it("makes no narrow SKU query without a SKU", async () => { const { pb, getList } = makePb([NUT]); await shortlistStockItems(pb, { description: "M18 hex nut grade 8" }); - expect(getList).toHaveBeenCalledTimes(1); + // The word pull and the size pull — but no narrow SKU pull. + expect(getList).toHaveBeenCalledTimes(2); + for (const call of getList.mock.calls) { + expect(call[2].filter).not.toContain('sku = "'); + } }); it("uses a supplier map passed in instead of loading suppliers", async () => { diff --git a/src/lib/server/quote-assistant-shortlist.ts b/src/lib/server/quote-assistant-shortlist.ts index 3b4d057..a50cc2e 100644 --- a/src/lib/server/quote-assistant-shortlist.ts +++ b/src/lib/server/quote-assistant-shortlist.ts @@ -1,134 +1,42 @@ /** - * quote-assistant-shortlist — deterministic (no AI) candidate lookup of POS - * items (`stock_items`) for one quote line. A later step sends these - * candidates to the AI to choose from. + * quote-assistant-shortlist — the POS candidate lookup for one quote + * line. * - * Candidate pull mirrors item-intel-service's stock section: PB `~` OR'd over - * the top informative tokens, then in-app Jaccard/overlap scoring. Differences: - * only active items, name + description + sku are searched, and an exact - * (normalised) SKU hit always wins with score 1. + * The work now lives in `pos-match.ts` — the shared, SIZE-AWARE POS + * matcher that the quote assistant, the order fill and the price check + * all use. This module keeps the old name and signature so the quote + * assistant (and its tests) keep working, and gains the size-aware + * ranking for free. * - * The broad pull is capped and unsorted, so with a SKU on the line a second - * narrow query (exact / normalised SKU) makes sure the SKU row is never - * crowded out; both result sets are merged by id before scoring. + * What the size-aware ranking buys us: for "BOLT HEX M16X60 GR8.8 ZP" + * the old word-only shortlist put the M22x65 first, because every bolt + * shares the words "BOLT 8.8 ZP". The size pull now brings the M16 + * rows into the pool and the size comparison puts them first. */ -import type { getAdminPocketBase } from "@/lib/server/pocketbase-admin"; import { - combinedScore, - informativeTokens, - loadSuppliersById, - tokenizeIntel, -} from "@/lib/server/item-intel-service"; -import { escapeForPbString, normaliseSku } from "@/lib/server/price-list-match"; + shortlistPosItems, + type AdminPb, + type PosCandidate, + type StockCandidate, +} from "@/lib/server/pos-match"; -export interface StockCandidate { - id: string; - name: string; - sku: string; - description: string; - notes: string; - unit: string; - cost_price: number; - selling_price: number; - markup: number; - supplier_id: string; - supplier_name: string; - score: number; -} - -export type AdminPb = Awaited>; - -const FETCH_LIMIT = 500; -const SKU_FETCH_LIMIT = 50; -const STOCK_FIELDS = "id,sku,name,description,notes,unit,cost_price,selling_price,markup,supplier"; -const MIN_SCORE = 0.2; -const MAX_TOKENS = 4; - -type StockRow = { - id: string; - sku?: string; - name?: string; - description?: string; - notes?: string; - unit?: string; - cost_price?: number; - selling_price?: number; - markup?: number; - supplier?: string; -}; +export type { AdminPb, StockCandidate } from "@/lib/server/pos-match"; /** - * `supplierNameById`: pass a preloaded supplier map when shortlisting many - * lines in a row; when omitted the suppliers are loaded here. + * `supplierNameById`: pass a preloaded supplier map when shortlisting + * many lines in a row; when omitted the suppliers are loaded here. */ export async function shortlistStockItems( pb: AdminPb, - line: { description: string; sku?: string }, + line: { description: string; sku?: string; notes?: string }, limit = 8, supplierNameById?: Map, ): Promise { - const tokens = informativeTokens(line.description || "", MAX_TOKENS); - // `"` and `\` can't be escaped reliably in a PB filter string and never - // matter to a SKU match (normaliseSku drops them anyway). - const rawSku = (line.sku || "").replace(/[\\"]/g, "").trim(); - const wantedSku = normaliseSku(rawSku); - - const clauses: string[] = []; - for (const t of tokens) { - const v = escapeForPbString(t); - clauses.push(`name ~ "${v}"`, `description ~ "${v}"`, `sku ~ "${v}"`); - } - if (rawSku) clauses.push(`sku ~ "${escapeForPbString(rawSku)}"`); - if (clauses.length === 0) return []; - - const res = await pb.collection("stock_items").getList(1, FETCH_LIMIT, { - filter: `is_active = true && (${clauses.join(" || ")})`, - fields: STOCK_FIELDS, - }); - - const rowsById = new Map(); - if (rawSku) { - const skuClauses = [`sku = "${escapeForPbString(rawSku)}"`]; - if (wantedSku) skuClauses.push(`sku ~ "${escapeForPbString(wantedSku)}"`); - const narrow = await pb.collection("stock_items").getList(1, SKU_FETCH_LIMIT, { - filter: `is_active = true && (${skuClauses.join(" || ")})`, - fields: STOCK_FIELDS, - }); - for (const row of narrow.items) rowsById.set(row.id, row); - } - for (const row of res.items) if (!rowsById.has(row.id)) rowsById.set(row.id, row); - - const lineTokens = tokenizeIntel(line.description || ""); - const scored = [...rowsById.values()] - .map((row) => { - const skuHit = wantedSku !== "" && normaliseSku(row.sku) === wantedSku; - const score = skuHit - ? 1 - : Math.max( - combinedScore(lineTokens, tokenizeIntel(row.name || "")), - combinedScore(lineTokens, tokenizeIntel(row.description || "")), - ); - return { row, score, skuHit }; - }) - .filter((r) => r.skuHit || r.score >= MIN_SCORE) - .sort((a, b) => b.score - a.score || Number(b.skuHit) - Number(a.skuHit)) - .slice(0, limit); - - if (scored.length === 0) return []; - - const suppliers = supplierNameById ?? (await loadSuppliersById(pb)); - return scored.map(({ row, score }) => ({ - id: row.id, - name: row.name || "", - sku: row.sku || "", - description: row.description || "", - notes: row.notes || "", - unit: row.unit || "", - cost_price: row.cost_price || 0, - selling_price: row.selling_price || 0, - markup: row.markup || 0, - supplier_id: row.supplier || "", - supplier_name: suppliers.get(row.supplier || "") || "", - score: +score.toFixed(3), - })); + const candidates: PosCandidate[] = await shortlistPosItems( + pb, + line, + limit, + supplierNameById, + ); + return candidates; } diff --git a/src/lib/types.ts b/src/lib/types.ts index 6ffe0b0..aaac228 100644 --- a/src/lib/types.ts +++ b/src/lib/types.ts @@ -2343,7 +2343,27 @@ export interface ItemHistoryResponse { * used when no anchor was supplied or the anchor no * longer resolves). */ -export type ItemIntelMatchKind = "stock_item" | "sku" | "fuzzy"; +export type ItemIntelMatchKind = "stock_item" | "sku" | "fuzzy" | "pos_pick"; + +/** The AI's pick for a line — the POS row it chose and how sure it was. + * + * Stored on the intel blob (`order_items.ai_item_insights.pos_pick`) + * so the price check and the popover can say "this is the item the AI + * picked" and, when it was only the closest, what differs. + */ +export interface ItemIntelPosPick { + /** PB id of the `stock_items` row the AI chose. */ + stock_item_id: string; + /** "exact" = every size attribute the line states matches. + * "closest" = the nearest item, but something differs. */ + match: "exact" | "closest"; + /** What differs between the line and the picked item, in plain words. */ + differs?: string[]; + /** Why the AI picked it. */ + reason?: string; + /** The model that made the pick. */ + model?: string; +} /** Optional pin for the intel computation. * @@ -2553,6 +2573,9 @@ export interface ItemIntel { /** How the primary stock match was found. Absent on blobs written * before anchoring existed — treat absent as "fuzzy". */ match_kind?: ItemIntelMatchKind; + /** The AI's POS pick for this line, when there was one. Absent when + * the intel was computed without the AI (fuzzy only). */ + pos_pick?: ItemIntelPosPick; } // ─────────────────────────────────────────────────────────────────────────────