/** * item-intel-service — deterministic, description-fuzzy intel for any * quote_item or order_item. Powers the ⓘ popover (Option A — replaces * the older `ItemHistoryPopover` content). * * ## What it answers for one item * * - "Do we have it in stock?" → stock_list (fuzzy stock_items) * - "Did we buy it before, and from whom?" → supplier_history * (last order per unique * supplier, via * stock_purchase_items ↔ * stock_purchases) * - "Is it on our price list?" → price_list (fuzzy price_list, * ranked by overlapScore on * the denormalised search_text) * - "Have we quoted it before?" → recent_quote (fuzzy quote_items) * - "Have we ordered it before?" → recent_order (fuzzy order_items) * * ## Matching strategy (description-fuzzy, no SKU gate) * * New items don't have a SKU. The previous SKU-first services * (`getItemHistory`, `populateQuoteItemInsights`) returned empty for * every section whenever no SKU match could be recovered via the * description-fuzzy SKU recovery. This service runs the description * fuzzy match against each section's collection directly, so brand * new items with only a free-text description still get intel. * * The candidate pull for stock_items, quote_items, and order_items * uses PB's `~` operator with an OR of the top informative tokens. * In-app Jaccard + overlapScore then rank the candidates. Price list * uses the same OR but on a multi-column candidate filter * (description + search_text + sku + sku_normalised) since its * `search_text` field is the denormalised token soup the Catalogue * tab already relies on. * * ## Per-section limits (the popover must stay scannable) * * stock_list → 3 rows * supplier_history → one row per unique supplier (latest purchase) * price_list → 5 rows * recent_quote → 2 rows * recent_order → 2 rows * * Empty sections are `[]`, never padded. * * ## Stock display vs purchase-cap decoupling * * Stock list displays the top 3 by fuzzy score. But the supplier * dedup needs purchase history from ALL stock_items that fuzzy-match * (not just the top 3), otherwise legitimate suppliers we bought * from get hidden behind a low stock score. The SPI walk uses a * wider cap (10) and runs in parallel. * * ## Auth + writes * * Uses admin PB client. Writes `quote_items.ai_item_insights` or * `order_items.ai_item_insights` directly. Idempotent — safe to * re-run. The MCP `populate_insights` tool delegates here. */ import { getAdminPocketBase } from "@/lib/server/pocketbase-admin"; import { jaccard } from "@/lib/server/order-quote-match"; import type { ItemIntel, ItemIntelPriceListRow, ItemIntelQuoteRow, ItemIntelOrderRow, ItemIntelStockRow, ItemIntelSupplierRow, ItemIntelTarget, } from "@/lib/types"; import { ITEM_INTEL_LIMITS } from "@/lib/types"; // ─── Pure scoring helpers ──────────────────────────────────────────── /** Tokenizer tuned for item-intel fuzzy matching. * * Differs from `order-quote-match.tokenize` in three ways that * matter for catalogue fuzziness: * * 1. Keeps short numeric tokens ("50", "20L", "M20") — these * carry product identity (sizes, capacities, model numbers). * Dropping them made every SAE oil tie at overlap=1.0 and * ruined price_list matching against the AFRIQ SKUs. * 2. Smaller stopword set — only tokens that are pure noise * across every catalogue. No "mm" / "qty" filter because * "qty" doesn't appear in our descriptions and "mm" only * does so meaningfully (e.g. "M6 X 30MM" — keep it). * 3. Allows single-character numeric tokens ("L", "5") so * compound unit strings like "20LT" or "5L" tokenise as * one unit token (the regex matches the whole run, not * per-character). */ const INTEL_STOPWORDS = new Set([ "the", "and", "with", "for", "of", "to", "a", "an", "in", "on", "at", "by", "from", "as", "is", "or", "x", ]); function tokenizeIntel(s: string): Set { const out = new Set(); if (!s) return out; for (const tok of String(s).toLowerCase().match(/[a-z0-9]+/g) || []) { if (INTEL_STOPWORDS.has(tok)) continue; out.add(tok); } return out; } /** Overlap coefficient: |A ∩ B| / min(|A|, |B|). * Better than Jaccard for "short query against a rich candidate" — * doesn't penalise extra tokens on the candidate side. Used for * ranking against the price_list `search_text` blob. */ function overlapScore(a: Set, b: Set): number { if (a.size === 0 || b.size === 0) return 0; let inter = 0; for (const t of a) if (b.has(t)) inter++; return inter / Math.min(a.size, b.size); } /** Days between `iso` and `now`. Negative clamped to 0. */ function daysAgo(iso: string | null | undefined): number { if (!iso) return 0; const t = new Date(iso).getTime(); if (Number.isNaN(t)) return 0; const diffMs = Date.now() - t; return diffMs <= 0 ? 0 : Math.floor(diffMs / 86_400_000); } /** Drop stopwords + short tokens. */ const INFORMATIVE_MIN_LEN = 3; /** Pick the top N tokens by length desc (most distinctive first). */ function informativeTokens(s: string, max = 4): string[] { const toks = tokenizeIntel(s || ""); return [...toks] .filter((t) => !INTEL_STOPWORDS.has(t) && t.length >= INFORMATIVE_MIN_LEN) .sort((a, b) => b.length - a.length) .slice(0, max); } /** PocketBase `~` is case-insensitive substring match. OR'ing the top * informative tokens keeps the candidate set broad while letting * the in-app Jaccard/overlap rank with precision. */ function buildPbContainsOR(field: string, targetDesc: string, maxTokens = 4): string { const toks = informativeTokens(targetDesc, maxTokens); if (toks.length === 0) return ""; return toks.map((t) => `${field} ~ "${t}"`).join(" || "); } /** Combined Jaccard + overlap score, capped at 1. */ function combinedScore(a: Set, b: Set): number { return Math.max(jaccard(a, b), overlapScore(a, b)); } // ─── PB I/O helpers ─────────────────────────────────────────────────── type PbListResult = { items: T[]; totalItems?: number }; async function safeGetList( pb: ReturnType extends never ? never : Awaited>, collection: string, opts: { max?: number; filter?: string; sort?: string; fields?: string; expand?: string }, fallback: PbListResult = { items: [] }, ): Promise> { try { return (await pb.collection(collection).getList(1, opts.max ?? 300, opts)) as PbListResult; } catch (err) { const status = (err as { status?: number })?.status; if (status === 400 || status === 404) { console.warn(`[item-intel] ${collection} returned ${status} — degrading`); return fallback; } throw err; } } async function loadSuppliersById(pb: Awaited>): Promise> { try { const rows = await pb .collection("suppliers") .getFullList<{ id: string; suppliername: string }>({ fields: "id,suppliername", batch: 500 }); return new Map(rows.map((s) => [s.id, s.suppliername])); } catch { return new Map(); } } // ─── The 5 sections ────────────────────────────────────────────────── async function buildStock( pb: Awaited>, target: { id: string; description: string }, supplierNameById: Map, ): Promise<{ matches: ItemIntelStockRow[]; purchaseHistory: Array<{ stock_item_id: string; stock_item_sku: string; stock_item_name: string; supplier_id: string; supplier_name: string; cost_price: number; quantity: number; purchased_at: string; days_ago: number; purchase_number: string; }> }> { const filter = buildPbContainsOR("description", target.description); if (!filter) return { matches: [], purchaseHistory: [] }; const candRows = await safeGetList<{ id: string; sku: string; name: string; description: string; cost_price: number; selling_price: number; current_qty: number; min_qty: number | null; supplier: string; }>(pb, "stock_items", { filter, fields: "id,sku,name,description,cost_price,selling_price,current_qty,min_qty,supplier", max: 500, }); const targetTokens = tokenizeIntel(target.description); const rankedAll = candRows.items .map((si) => ({ si, score: combinedScore(targetTokens, tokenizeIntel(si.description || "")) })) .filter((r) => r.score >= 0.2) .sort((a, b) => b.score - a.score); const forDisplay = rankedAll.slice(0, ITEM_INTEL_LIMITS.stock); const forPurchases = rankedAll.slice(0, 10); const matches: ItemIntelStockRow[] = forDisplay.map(({ si, score }) => { const current_qty = si.current_qty || 0; const min_qty = si.min_qty ?? null; let status: ItemIntelStockRow["status"] = "out"; if (current_qty > 0) status = min_qty != null && current_qty <= min_qty ? "low" : "in_stock"; return { stock_item_id: si.id, score: +score.toFixed(3), name: si.name || "", sku: si.sku || "", description: si.description || "", qty: current_qty, min_qty, cost_price: si.cost_price || 0, selling_price: si.selling_price || 0, supplier_id: si.supplier || "", supplier_name: supplierNameById.get(si.supplier) || si.supplier || "", status, }; }); const purchaseHistory: Array<{ stock_item_id: string; stock_item_sku: string; stock_item_name: string; supplier_id: string; supplier_name: string; cost_price: number; quantity: number; purchased_at: string; days_ago: number; purchase_number: string; }> = (await Promise.all( forPurchases.map(async ({ si }) => { try { const spiRows = await pb .collection("stock_purchase_items") .getFullList<{ id: string; cost_price: number; quantity: number; expand?: { purchase?: { id: string; purchase_number: string; supplier: string; created: string } }; }>({ filter: `stock_item = "${si.id}"`, expand: "purchase", fields: "id,cost_price,quantity,expand.purchase.id,expand.purchase.purchase_number,expand.purchase.supplier,expand.purchase.created", }); const out: typeof purchaseHistory = []; for (const spi of spiRows) { const p = spi.expand?.purchase; if (!p) continue; out.push({ stock_item_id: si.id, stock_item_sku: si.sku || "", stock_item_name: si.name || "", supplier_id: p.supplier || "", supplier_name: supplierNameById.get(p.supplier) || p.supplier || "", cost_price: spi.cost_price || 0, quantity: spi.quantity || 0, purchased_at: p.created, days_ago: daysAgo(p.created), purchase_number: p.purchase_number || "", }); } return out; } catch (err) { console.warn(`[item-intel] spi join failed for ${si.id}:`, (err as { status?: number })?.status); return []; } }), )).flat(); purchaseHistory.sort((a, b) => (b.purchased_at || "").localeCompare(a.purchased_at || "")); return { matches, purchaseHistory }; } /** Last order at every unique supplier. Latest purchase wins per * supplier_id; sort by purchased_at desc. The stock_item_* fields * are taken from the most recent purchase row for that supplier — * earlier rows for the same supplier carry older stock_items. */ function buildLastSupplierPrice(purchaseHistory: Array<{ supplier_id: string; supplier_name: string; cost_price: number; purchased_at: string; days_ago: number; purchase_number: string; stock_item_id: string; stock_item_sku: string; stock_item_name: string; }>): ItemIntelSupplierRow[] { const bySupplier = new Map(); for (const r of purchaseHistory) { if (!r.supplier_id) continue; const prev = bySupplier.get(r.supplier_id); if (!prev) { bySupplier.set(r.supplier_id, { supplier_id: r.supplier_id, supplier_name: r.supplier_name, last_cost: r.cost_price, last_bought_at: r.purchased_at, days_ago: r.days_ago, last_purchase_number: r.purchase_number, times_bought: 1, stock_item_id: r.stock_item_id, stock_item_sku: r.stock_item_sku, stock_item_name: r.stock_item_name, }); } else { prev.times_bought += 1; if (new Date(r.purchased_at).getTime() > new Date(prev.last_bought_at).getTime()) { prev.last_cost = r.cost_price; prev.last_bought_at = r.purchased_at; prev.days_ago = r.days_ago; prev.last_purchase_number = r.purchase_number; prev.stock_item_id = r.stock_item_id; prev.stock_item_sku = r.stock_item_sku; prev.stock_item_name = r.stock_item_name; } } } return [...bySupplier.values()].sort((a, b) => (b.last_bought_at || "").localeCompare(a.last_bought_at || ""), ); } async function buildPriceList( pb: Awaited>, target: { description: string }, ): Promise { const toks = informativeTokens(target.description, 4); if (toks.length === 0) return []; // OR informative tokens across every text column the repo already // keeps searchable. `search_text` is the denormalised token soup // built by `buildPriceListSearchText` (src/lib/price-list-search.ts) // — it's how the Catalogue tab does fuzzy hits and the cheapest // way to catch items where the catalogue wording diverges from the // operator's wording (e.g. "SAE 50 - 20L" → row "SAE 50 SF/CF4 - 20 LT"). const cols = ["description", "search_text", "sku", "sku_normalised"]; const clauses = cols.flatMap((c) => toks.map((t) => `${c} ~ "${t}"`)); const filter = `(${clauses.join(" || ")})`; const candRows = await safeGetList<{ id: string; sku: string; sku_normalised: string; description: string; brand: string; category: string; cost_price: number; selling_price: number; supplier: string; source_file: string; updated: string; search_text: string; }>(pb, "price_list", { filter, fields: "id,sku,sku_normalised,description,brand,category,cost_price,selling_price,supplier,source_file,updated,search_text", max: 300, }); // Rank with overlapScore against the concatenated text columns — // for short queries against a rich denormalised blob, overlap is // the right measure (it doesn't penalise extra tokens the way // Jaccard does). const targetTokens = tokenizeIntel(target.description); const ranked: Array<{ row: typeof candRows.items[number]; score: number }> = []; for (const r of candRows.items) { const haystack = [r.description, r.search_text, r.sku, r.sku_normalised, r.brand, r.category, r.supplier, r.source_file] .filter(Boolean).join(" "); const score = overlapScore(targetTokens, tokenizeIntel(haystack)); if (score >= 0.6) ranked.push({ row: r, score }); } ranked.sort((a, b) => b.score - a.score); return ranked.slice(0, ITEM_INTEL_LIMITS.priceList).map(({ row: r, score }) => ({ score: +score.toFixed(3), sku: r.sku || "", description: r.description || "", brand: r.brand || "", category: r.category || "", cost_price: r.cost_price || 0, selling_price: r.selling_price || 0, supplier: r.supplier || "", source_file: r.source_file || "", updated: r.updated, })); } async function buildQuoteHistory( pb: Awaited>, target: { id: string; description: string }, supplierNameById: Map, ): Promise { const filter = buildPbContainsOR("description", target.description); if (!filter) return []; const candRows = await safeGetList<{ id: string; description: string; cost_price: number; selling_price: number; quantity: number; created: string; quote: string; supplier: string; expand?: { quote?: { quote_number: string; status: string; customer_name?: string; company?: string; expand?: { company?: { id: string; company?: string } }; }; }; }>(pb, "quote_items", { filter: `id != "${target.id}" && (${filter})`, expand: "quote.company", fields: "id,description,cost_price,selling_price,quantity,created,quote,supplier,expand.quote.id,expand.quote.quote_number,expand.quote.status,expand.quote.company,expand.quote.expand.company.id,expand.quote.expand.company.company", max: 300, sort: "-created", }); const targetTokens = tokenizeIntel(target.description); const ranked = candRows.items .map((q) => ({ q, score: combinedScore(targetTokens, tokenizeIntel(q.description || "")) })) .filter((r) => r.score >= 0.5) .sort((a, b) => b.score - a.score) .slice(0, ITEM_INTEL_LIMITS.recentQuote); return ranked.map(({ q, score }) => { // Resolve the customer/company FKs to human-readable names so the // popover doesn't show opaque ids like `vvns7ji6qeao9xk`. Prefer // the expanded company name (from the `companies` collection); // fall back to the customer name field on the quote, then the raw // FK id, then an em-dash. const companyName = q.expand?.quote?.expand?.company?.company || ""; const customerName = q.expand?.quote?.customer_name || ""; return { score: +score.toFixed(3), quote_item_id: q.id, quote_id: q.quote, quote_number: q.expand?.quote?.quote_number || "", quote_status: q.expand?.quote?.status || "", customer_name: companyName || customerName || q.expand?.quote?.company || "", company: companyName || q.expand?.quote?.company || "", description: q.description || "", cost_price: q.cost_price || 0, selling_price: q.selling_price || 0, quantity: q.quantity || 0, quoted_at: q.created, days_ago: daysAgo(q.created), supplier_id: q.supplier || "", supplier_name: q.supplier ? supplierNameById.get(q.supplier) || "" : "", }; }); } async function buildOrderHistory( pb: Awaited>, target: { description: string }, supplierNameById: Map, ): Promise { const filter = buildPbContainsOR("description", target.description); if (!filter) return []; const candRows = await safeGetList<{ id: string; description: string; cost_price: number; selling_price: number; quantity: number; item_status: string; supplier: string; updated: string; order: string; expand?: { order?: { order_number: string; company?: string } }; }>(pb, "order_items", { filter, expand: "order", fields: "id,description,cost_price,selling_price,quantity,item_status,supplier,updated,order,expand.order.id,expand.order.order_number,expand.order.company", max: 300, sort: "-updated", }); const targetTokens = tokenizeIntel(target.description); const ranked = candRows.items .map((o) => ({ o, score: combinedScore(targetTokens, tokenizeIntel(o.description || "")) })) .filter((r) => r.score >= 0.5) .sort((a, b) => b.score - a.score) .slice(0, ITEM_INTEL_LIMITS.recentOrder); return ranked.map(({ o, score }) => ({ score: +score.toFixed(3), order_item_id: o.id, order_id: o.order, order_number: o.expand?.order?.order_number || "", description: o.description || "", cost_price: o.cost_price || 0, selling_price: o.selling_price || 0, quantity: o.quantity || 0, item_status: o.item_status || "", supplier_id: o.supplier || "", supplier_name: supplierNameById.get(o.supplier) || o.supplier || "", updated: o.updated, days_ago: daysAgo(o.updated), })); } // ─── Summary ───────────────────────────────────────────────────────── function composeSummary(input: { matches: ItemIntelStockRow[]; supplierHistory: ItemIntelSupplierRow[]; recent_quote: ItemIntelQuoteRow[]; recent_order: ItemIntelOrderRow[]; price_list: ItemIntelPriceListRow[]; }): string { const bits: string[] = []; const topStock = input.matches[0]; if (topStock && topStock.qty > 0) { bits.push(`in stock ${topStock.qty} (${topStock.supplier_name || "—"}, cost R${topStock.cost_price.toFixed(2)})`); } else if (topStock) { bits.push(`out of stock (last supplier ${topStock.supplier_name || "—"})`); } if (input.supplierHistory.length === 1) { const s = input.supplierHistory[0]; bits.push(`bought from ${s.supplier_name} ${s.days_ago}d ago @ R${s.last_cost.toFixed(2)} (×${s.times_bought})`); } else if (input.supplierHistory.length > 1) { bits.push(`bought from ${input.supplierHistory.length} suppliers — top: ${input.supplierHistory[0].supplier_name} R${input.supplierHistory[0].last_cost.toFixed(2)} ${input.supplierHistory[0].days_ago}d ago`); } const lq = input.recent_quote[0]; if (lq) bits.push(`last quoted R${lq.selling_price.toFixed(2)} ${lq.days_ago}d ago (${lq.quote_number})`); const lo = input.recent_order[0]; if (lo) bits.push(`last order ${lo.days_ago}d ago (${lo.order_number}, ${lo.item_status})`); if (input.price_list.length > 0) bits.push(`on price list (${input.price_list.length})`); return bits.join(" · ") || "No history yet for this item."; } // ─── Public entry points ───────────────────────────────────────────── /** * Compute and write the Item Intel payload for one quote_item or * order_item. Collection is the PB collection name (e.g. * `"quote_items"`). The intel is written to `.ai_item_insights`. * * `write: false` runs the whole algorithm without persisting — use * for the live preview path. */ export async function populateItemIntel( itemId: string, collection: "quote_items" | "order_items", options: { write?: boolean } = {}, ): Promise { const write = options.write !== false; const pb = await getAdminPocketBase(); const target = await pb.collection(collection).getOne<{ id: string; sku: string; description: string; quantity: number; supplier: string; }>(itemId, { fields: "id,sku,description,quantity,supplier" }); const supplierNameById = await loadSuppliersById(pb); const generated_at = new Date().toISOString(); const [stockRaw, price_list, recent_quote, recent_order] = await Promise.all([ buildStock(pb, target, supplierNameById), buildPriceList(pb, target), buildQuoteHistory(pb, target, supplierNameById), buildOrderHistory(pb, target, supplierNameById), ]); const supplier_history = buildLastSupplierPrice(stockRaw.purchaseHistory); const item: ItemIntelTarget = { id: itemId, collection, description: target.description || "", sku: target.sku || "", quantity: target.quantity || 0, supplier: target.supplier || "", supplier_name: target.supplier ? supplierNameById.get(target.supplier) || "" : "", }; const summary = composeSummary({ matches: stockRaw.matches, supplierHistory: supplier_history, recent_quote, recent_order, price_list, }); const intel: ItemIntel = { item, summary, generated_at, stock_list: stockRaw.matches, supplier_history, price_list, recent_quote, recent_order, }; if (write) { await pb.collection(collection).update(itemId, { ai_item_insights: intel }); } return intel; } /** * Populate Item Intel for every active line on a quote or order. * Parallel-safe within one parent; the caller controls cross-parent * batching to avoid hammering PB. */ export async function populateIntelForParent( parentId: string, collection: "quote_items" | "order_items", parentCollection: "quotes" | "orders", options: { write?: boolean } = {}, ): Promise<{ parent_id: string; total: number; populated: number; no_match: number }> { const pb = await getAdminPocketBase(); const parentField = parentCollection === "quotes" ? "quote" : "order"; const items = await pb.collection(collection).getFullList<{ id: string }>({ filter: `${parentField} = "${parentId}" && (moved_to_quote = null || moved_to_quote = "")`, fields: "id", }); const results = await Promise.all( items.map((it) => populateItemIntel(it.id, collection, options).then( (intel) => ({ id: it.id, found: intel.stock_list.length > 0 || intel.price_list.length > 0 || intel.recent_quote.length > 0 || intel.recent_order.length > 0 || intel.supplier_history.length > 0, }), () => ({ id: it.id, found: false }), )), ); return { parent_id: parentId, total: items.length, populated: results.filter((r) => r.found).length, no_match: results.filter((r) => !r.found).length, }; }