WFDASH-WEBAPP/scripts/smoke/item-intel.mjs
Dave 3a59fa104c feat(item-intel): add new item intel popover and backend workflows
This commit replaces the legacy item history and AI insights system with a standardized, fuzzy-matched item intel workflow:
- Update AiItemHistoryTrigger component to support both quote and order items via the `collection` prop, replacing the old `insights` prop with `intel`
- Add full `ItemIntel` type definitions to `src/lib/types.ts` for the new standardized insight payload
- Create new REST API endpoints for fetching and computing item intel for `quote_items` and `order_items`
- Rewrite the MCP insight endpoint to use the new shared `item-intel-service` instead of legacy individual insight functions
- Reorder quotes page tabs to place Delivery Note quotes alongside New quotes for better daily operator workflow
- Add item intel triggers to quote process and order process pages
- Remove deprecated document-level `AiInsightsPanel` components from quote and order detail pages
- Add bulk backfill endpoint for populating intel across existing active items
2026-08-11 02:57:57 +02:00

533 lines
No EOL
20 KiB
JavaScript
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

/**
* item-intel.mjs — live smoke for the item-intel service.
*
* Runs the description-fuzzy, item-level intel algorithm against the
* 3 quote_items from Q202608094 and prints the resulting JSON. Does
* NOT write to PB unless WRITE_INTEL=1 is set in the env.
*
* node scripts/smoke/item-intel.mjs # dry-run, prints JSON
* WRITE_INTEL=1 node scripts/smoke/item-intel.mjs # also persists ai_item_insights
*
* The algorithm here is the spec for the future
* `src/lib/server/item-intel-service.ts`. Once the JSON looks right,
* port this file to TS + add the ItemIntel types to
* `src/lib/types.ts`.
*/
import { readFileSync } from "node:fs";
import { resolve } from "node:path";
import PocketBase from "pocketbase";
// ── env (inline loader so we don't depend on tsx) ───────────────────
function loadEnvFile(name) {
try {
const raw = readFileSync(resolve(process.cwd(), name), "utf8");
for (const line of raw.split("\n")) {
const t = line.trim();
if (!t || t.startsWith("#")) continue;
const m = t.match(/^([A-Z0-9_]+)\s*=\s*(.+)$/i);
if (m && !process.env[m[1]]) {
process.env[m[1]] = m[2].replace(/^["']|["']$/g, "").trim();
}
}
} catch {}
}
loadEnvFile(".env");
loadEnvFile(".env.local");
const PB_URL =
process.env.NEXT_PUBLIC_POCKETBASE_URL || "http://127.0.0.1:8090";
const EMAIL = process.env.PB_ADMIN_EMAIL;
const PASS = process.env.PB_ADMIN_PASSWORD;
if (!EMAIL || !PASS) {
console.error("PB_ADMIN_EMAIL / PB_ADMIN_PASSWORD missing in .env.local");
process.exit(1);
}
const WRITE = process.env.WRITE_INTEL === "1";
// ── fuzzy helpers (mirror src/lib/server/order-quote-match.ts) ───────
const STOPWORDS = new Set([
"the", "and", "with", "for", "of", "to", "a", "an", "in", "on", "at",
"by", "from", "as", "is", "or", "x", "-", "/", "&",
]);
function tokenize(s) {
const out = new Set();
if (!s) return out;
for (const tok of String(s).toLowerCase().match(/[a-z0-9]+/g) || []) {
out.add(tok);
}
return out;
}
function informativeTokens(s, max = 4) {
const toks = tokenize(s);
const keep = [...toks]
.filter((t) => !STOPWORDS.has(t) && t.length >= 3)
.sort((a, b) => b.length - a.length)
.slice(0, max);
return keep;
}
function jaccard(a, b) {
if (a.size === 0 || b.size === 0) return 0;
let inter = 0;
for (const t of a) if (b.has(t)) inter++;
return inter / (a.size + b.size - inter);
}
function overlapScore(a, b) {
if (a.size === 0 || b.size === 0) return 0;
let inter = 0;
for (const t of a) if (b.has(t)) inter++;
return inter / Math.min(a.size, b.size);
}
function combinedScore(a, b) {
return Math.max(jaccard(a, b), overlapScore(a, b));
}
/**
* PB `~` is case-insensitive contains. We OR the top informative tokens
* so the candidate set stays broad (a description that says "black"
* still pulls stock rows labelled "BLK"/"ZPL"), then rank in-app with
* Jaccard + overlap. The in-app scorer is where precision lives.
*/
function buildPbContainsFilter(field, targetDesc, maxTokens = 4) {
const toks = informativeTokens(targetDesc, maxTokens);
if (toks.length === 0) return "";
return toks.map((t) => `${field} ~ "${t}"`).join(" || ");
}
function rankByDescription(targetDesc, candidates, opts = {}) {
const { threshold = 0.25, useJaccardOnly = false } = opts;
const targetTokens = tokenize(targetDesc);
if (targetTokens.size === 0) return [];
const scored = [];
for (const c of candidates) {
const desc = c.description || "";
if (!desc) continue;
const ct = tokenize(desc);
const j = jaccard(targetTokens, ct);
const o = overlapScore(targetTokens, ct);
const score = useJaccardOnly ? j : Math.max(j, o);
if (score >= threshold) scored.push({ item: c, score });
}
scored.sort((a, b) => b.score - a.score);
return scored;
}
function daysAgo(iso) {
if (!iso) return 0;
const t = new Date(iso).getTime();
if (Number.isNaN(t)) return 0;
return Math.max(0, Math.floor((Date.now() - t) / 86_400_000));
}
// ── PB helpers ──────────────────────────────────────────────────────
async function safeGetList(pb, collection, opts, fallback = { items: [] }) {
try {
return await pb.collection(collection).getList(1, opts.max || 300, opts);
} catch (err) {
const status = err?.status;
if (status === 400 || status === 404) {
console.warn(`[safeGetList] ${collection} returned ${status} — degrading`);
return fallback;
}
throw err;
}
}
async function loadSuppliersById(pb) {
const rows = await pb
.collection("suppliers")
.getFullList({ fields: "id,suppliername", batch: 500 });
return new Map(rows.map((s) => [s.id, s.suppliername]));
}
// ── the 5 sections (description-fuzzy, no SKU gate) ────────────────
/** 1. STOCK: fuzzy stock_items → for each, stock_purchase_items join purchases. */
async function buildStock(pb, target, supplierNameById) {
const filter = buildPbContainsFilter("description", target.description);
if (!filter) return { matches: [], purchase_history: [] };
const candRows = await safeGetList(pb, "stock_items", {
filter,
fields: "id,sku,name,description,cost_price,selling_price,current_qty,min_qty,supplier,markup,is_active",
max: 500,
});
const rankedAll = rankByDescription(target.description, candRows.items, { threshold: 0.2 });
// Two limits, two reasons:
// - matches cap (3): what the popover displays under "Stock List"
// - purchase-cap (10): how many stock_items we walk for purchase
// history so the supplier dedup sees the full supplier pool.
// The supplier_history section is the "call these N suppliers"
// list — truncating this low would hide legitimate suppliers
// we bought from (their stock_item just scored below the stock
// display cutoff).
const stockItemsForDisplay = rankedAll.slice(0, 3);
const stockItemsForPurchases = rankedAll.slice(0, 10);
const matches = stockItemsForDisplay.map(({ item: si, score }) => buildStockMatchRow(si, score, supplierNameById));
const purchase_history = (await Promise.all(
stockItemsForPurchases.map(({ item: si }) => fetchStockPurchaseHistory(pb, si.id, supplierNameById))
)).flat();
purchase_history.sort((a, b) => (b.purchased_at || "").localeCompare(a.purchased_at || ""));
return { matches, purchase_history };
}
function buildStockMatchRow(si, score, supplierNameById) {
const current_qty = si.current_qty || 0;
const min_qty = si.min_qty ?? null;
let status = "out";
if (current_qty > 0) status = min_qty != null && current_qty <= min_qty ? "low" : "in_stock";
return {
stock_item_id: si.id,
score: +score.toFixed(3),
name: si.name || "",
sku: si.sku || "",
description: si.description || "",
current_qty,
min_qty,
cost_price: si.cost_price || 0,
selling_price: si.selling_price || 0,
supplier_id: si.supplier || "",
supplier_name: supplierNameById.get(si.supplier) || si.supplier || "",
status,
};
}
async function fetchStockPurchaseHistory(pb, stockItemId, supplierNameById) {
try {
const spiRows = await pb
.collection("stock_purchase_items")
.getFullList({
filter: `stock_item = "${stockItemId}"`,
expand: "purchase",
fields:
"id,purchase,stock_item,cost_price,quantity,quantity_received,notes,status," +
"expand.purchase.id,expand.purchase.purchase_number,expand.purchase.supplier,expand.purchase.created,expand.purchase.status",
});
const out = [];
for (const spi of spiRows) {
const p = spi.expand?.purchase;
if (!p) continue;
out.push({
stock_item_id: stockItemId,
stock_purchase_item_id: spi.id,
purchase_id: p.id,
purchase_number: p.purchase_number || "",
purchased_at: p.created,
days_ago: daysAgo(p.created),
cost_price: spi.cost_price || 0,
quantity: spi.quantity || 0,
quantity_received: spi.quantity_received || 0,
supplier_id: p.supplier || "",
supplier_name: supplierNameById.get(p.supplier) || p.supplier || "",
status: p.status || "",
});
}
return out;
} catch (err) {
console.warn(`[stock] spi join failed for ${stockItemId}:`, err?.status || err?.message);
return [];
}
}
/** Derive multi-supplier "last supplier price" from purchase_history. */
function buildLastSupplierPrice(purchase_history) {
const bySupplier = new Map();
for (const r of purchase_history) {
if (!r.supplier_id) continue;
const prev = bySupplier.get(r.supplier_id);
if (!prev) {
bySupplier.set(r.supplier_id, {
supplier_id: r.supplier_id,
supplier_name: r.supplier_name,
last_cost: r.cost_price,
last_bought_at: r.purchased_at,
days_ago: r.days_ago,
times_bought: 1,
last_purchase_number: r.purchase_number,
});
} else {
prev.times_bought += 1;
if (
new Date(r.purchased_at).getTime() > new Date(prev.last_bought_at).getTime()
) {
prev.last_cost = r.cost_price;
prev.last_bought_at = r.purchased_at;
prev.days_ago = r.days_ago;
prev.last_purchase_number = r.purchase_number;
}
}
}
return [...bySupplier.values()].sort((a, b) =>
(b.last_bought_at || "").localeCompare(a.last_bought_at || "")
);
}
/** 2. PRICE LIST: fuzzy price_list across description + search_text + sku. */
async function buildPriceList(pb, target, supplierNameById) {
const toks = informativeTokens(target.description, 4);
if (toks.length === 0) return [];
// OR informative tokens across every text column the repo already
// keeps searchable. `search_text` is the denormalised token soup
// built by `buildPriceListSearchText` — it's how the Catalogue tab
// does fuzzy hits, and the cheapest way to catch items where the
// catalogue wording diverges from the operator's wording
// (e.g. "SAE 50 - 20L" → row "SAE 50 SF/CF4 - 20 LT").
const cols = ["description", "search_text", "sku", "sku_normalised"];
const clauses = cols.flatMap((c) => toks.map((t) => `${c} ~ "${t}"`));
const filter = `(${clauses.join(" || ")})`;
const candRows = await safeGetList(pb, "price_list", {
filter,
fields:
"id,sku,sku_normalised,description,brand,category,subcategory,cost_price,selling_price,supplier,source_file,unit,stock_qty,updated",
max: 300,
});
// Rank with overlapScore — "all the operator's distinctive tokens
// appear somewhere in the row's token soup". Threshold 0.6 = at
// least 60% of target tokens present (e.g. for 3 tokens → 2 of 3).
// Drop the jaccard gate: for short queries against a rich
// denormalised blob, overlap is the right measure.
const targetTokens = tokenize(target.description);
const ranked = [];
for (const r of candRows.items) {
const haystack = [r.description, r.search_text, r.sku, r.sku_normalised, r.brand, r.category, r.subcategory, r.supplier, r.source_file].filter(Boolean).join(" ");
const hayTokens = tokenize(haystack);
const o = overlapScore(targetTokens, hayTokens);
if (o >= 0.6) ranked.push({ row: r, score: o });
}
ranked.sort((a, b) => b.score - a.score);
return ranked.slice(0, 5).map(({ row: r, score }) => ({
score: +score.toFixed(3),
sku: r.sku || "",
description: r.description || "",
brand: r.brand || "",
category: r.category || "",
cost_price: r.cost_price || 0,
selling_price: r.selling_price || 0,
supplier: r.supplier || "",
source_file: r.source_file || "",
updated: r.updated,
}));
}
/** 3. QUOTE HISTORY: fuzzy quote_items by description, expand quote. */
async function buildQuoteHistory(pb, target) {
const filter = buildPbContainsFilter("description", target.description);
if (!filter) return [];
const candRows = await safeGetList(pb, "quote_items", {
filter: `id != "${target.id}" && (${filter})`,
expand: "quote",
fields:
"id,description,sku,cost_price,selling_price,quantity,created,quote," +
"expand.quote.id,expand.quote.quote_number,expand.quote.status,expand.quote.customer_name,expand.quote.company",
max: 300,
sort: "-created",
});
const ranked = rankByDescription(target.description, candRows.items, { threshold: 0.25 }).slice(0, 2);
return ranked.map(({ item: q, score }) => ({
score: +score.toFixed(3),
quote_item_id: q.id,
quote_id: q.quote,
quote_number: q.expand?.quote?.quote_number || "",
quote_status: q.expand?.quote?.status || "",
customer_name: q.expand?.quote?.customer_name || "",
company: q.expand?.quote?.company || "",
description: q.description || "",
cost_price: q.cost_price || 0,
selling_price: q.selling_price || 0,
quantity: q.quantity || 0,
quoted_at: q.created,
days_ago: daysAgo(q.created),
}));
}
/** 4. ORDER HISTORY: fuzzy order_items by description, expand order. */
async function buildOrderHistory(pb, target, supplierNameById) {
const filter = buildPbContainsFilter("description", target.description);
if (!filter) return [];
const candRows = await safeGetList(pb, "order_items", {
filter,
expand: "order",
fields:
"id,description,sku,cost_price,selling_price,quantity,item_status,supplier,updated,order," +
"expand.order.id,expand.order.order_number,expand.order.company",
max: 300,
sort: "-updated",
});
const ranked = rankByDescription(target.description, candRows.items, { threshold: 0.25 }).slice(0, 2);
return ranked.map(({ item: o, score }) => ({
score: +score.toFixed(3),
order_item_id: o.id,
order_id: o.order,
order_number: o.expand?.order?.order_number || "",
description: o.description || "",
cost_price: o.cost_price || 0,
selling_price: o.selling_price || 0,
quantity: o.quantity || 0,
item_status: o.item_status || "",
supplier_id: o.supplier || "",
supplier_name: supplierNameById.get(o.supplier) || o.supplier || "",
updated: o.updated,
days_ago: daysAgo(o.updated),
}));
}
/** Compose the short summary. */
function composeSummary({ stock, last_supplier_price, quote_history, order_history, price_list }) {
const bits = [];
const topStock = stock.matches?.[0];
if (topStock && topStock.current_qty > 0) {
bits.push(
`in stock ${topStock.current_qty} (${topStock.supplier_name || "—"}, cost R${topStock.cost_price.toFixed(2)})`
);
} else if (topStock) {
bits.push(`out of stock (last supplier ${topStock.supplier_name || "—"})`);
}
if (last_supplier_price.length === 1) {
const s = last_supplier_price[0];
bits.push(
`bought from ${s.supplier_name} ${s.days_ago}d ago @ R${s.last_cost.toFixed(2)} (×${s.times_bought})`
);
} else if (last_supplier_price.length > 1) {
bits.push(`bought from ${last_supplier_price.length} suppliers — top: ${last_supplier_price[0].supplier_name} R${last_supplier_price[0].last_cost.toFixed(2)} ${last_supplier_price[0].days_ago}d ago`);
}
const lq = quote_history[0];
if (lq) bits.push(`last quoted R${lq.selling_price.toFixed(2)} ${lq.days_ago}d ago (${lq.quote_number})`);
const lo = order_history[0];
if (lo) bits.push(`last order ${lo.days_ago}d ago (${lo.order_number}, ${lo.item_status})`);
if (price_list.length > 0) bits.push(`on price list (${price_list.length})`);
return bits.join(" · ") || "No history yet for this item.";
}
// ── main service function (mirrors future populateItemIntel) ────────
async function populateItemIntel(pb, itemId, collection, { write = false } = {}) {
const target = await pb.collection(collection).getOne(itemId, {
fields: "id,sku,description,cost_price,selling_price,quantity",
});
const supplierNameById = await loadSuppliersById(pb);
const generated_at = new Date().toISOString();
const [stock, price_list, quote_history, order_history] = await Promise.all([
buildStock(pb, target, supplierNameById),
buildPriceList(pb, target, supplierNameById),
buildQuoteHistory(pb, target),
buildOrderHistory(pb, target, supplierNameById),
]);
// ── reshape to the operator-facing vertical layout ──────────────
// Each section is an array. Empty array when nothing found.
// One row per actionable record. No score noise (the rank only
// matters to the algo; the operator wants the list).
const stock_list = stock.matches.map((m) => ({
sku: m.sku,
name: m.name,
description: m.description,
qty: m.current_qty,
cost_price: m.cost_price,
selling_price: m.selling_price,
supplier: m.supplier_name,
status: m.status,
}));
// supplier_history = one row per unique supplier (latest purchase wins).
// Source: the raw stock_purchase_items join; the derivation already
// lives in `buildLastSupplierPrice` and is also used by the summary.
const supplier_history = buildLastSupplierPrice(stock.purchase_history).map((s) => ({
supplier: s.supplier_name,
last_cost: s.last_cost,
last_bought_at: s.last_bought_at,
days_ago: s.days_ago,
times_bought: s.times_bought,
last_purchase_number: s.last_purchase_number,
}));
const recent_quote = quote_history.map((q) => ({
quote_number: q.quote_number,
quote_status: q.quote_status,
quoted_at: q.quoted_at,
days_ago: q.days_ago,
quantity: q.quantity,
cost_price: q.cost_price,
selling_price: q.selling_price,
description: q.description,
}));
const recent_order = order_history.map((o) => ({
order_number: o.order_number,
updated: o.updated,
days_ago: o.days_ago,
quantity: o.quantity,
cost_price: o.cost_price,
selling_price: o.selling_price,
item_status: o.item_status,
supplier: o.supplier_name,
description: o.description,
}));
const summary = composeSummary({
stock: { matches: stock.matches, purchase_history: stock.purchase_history },
last_supplier_price: buildLastSupplierPrice(stock.purchase_history),
quote_history,
order_history,
price_list,
});
const intel = {
item: {
id: itemId,
collection,
description: target.description || "",
sku: target.sku || "",
quantity: target.quantity || 0,
},
summary,
generated_at,
stock_list,
supplier_history,
price_list,
recent_quote,
recent_order,
};
if (write) {
await pb.collection(collection).update(itemId, { ai_item_insights: intel });
}
return intel;
}
// ── runner ──────────────────────────────────────────────────────────
const TARGETS = [
{ id: "rar7m1d0m6jgeb5", desc: "M20 x 30 set screw black" },
{ id: "0zbkoxfdlz0t7mh", desc: "SAE 50 - 20L" },
{ id: "h5szkwdxs1u2quk", desc: "Silicone sealant clear" },
];
async function main() {
console.log(`PB: ${PB_URL}`);
console.log(`Write: ${WRITE ? "YES (WRITE_INTEL=1)" : "no (dry-run)"}`);
console.log(`Targets: ${TARGETS.length} quote_items\n`);
const pb = new PocketBase(PB_URL);
pb.autoCancellation(false);
await pb.collection("_superusers").authWithPassword(EMAIL, PASS);
console.log("Auth OK\n");
const out = [];
for (const t of TARGETS) {
const t0 = Date.now();
console.log(`── ${t.desc} (${t.id}) ──`);
try {
const intel = await populateItemIntel(pb, t.id, "quote_items", { write: WRITE });
const ms = Date.now() - t0;
console.log(` ${ms}ms · stock_list=${intel.stock_list.length} · supplier_history=${intel.supplier_history.length} · price_list=${intel.price_list.length} · recent_quote=${intel.recent_quote.length} · recent_order=${intel.recent_order.length}`);
console.log(` summary: ${intel.summary}`);
out.push(intel);
} catch (err) {
console.error(` FAILED:`, err?.stack || err?.message || err);
out.push({ id: t.id, error: String(err?.message || err) });
}
console.log();
}
// full JSON
console.log("════════════════════════════════════════════════════════");
console.log("FULL JSON");
console.log("════════════════════════════════════════════════════════");
console.log(JSON.stringify(out, null, 2));
}
main().catch((err) => {
console.error("FATAL:", err);
process.exit(1);
});