From 1fb67efaee05218c0743754fc9bb477f46876269 Mon Sep 17 00:00:00 2001 From: SJCoetz Date: Tue, 30 Jun 2026 12:10:27 +0200 Subject: [PATCH] feat(pricelist): Catalogue search hits every field (incl. synonyms) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Catalogue tab's search box used to OR across just sku, description, and brand. Now it hits: - sku, description, brand, supplier, category, subcategory (PB's case-insensitive `~` operator, each indexed) - search_text (a denormalised text blob) The denormalised column is the key part: PB can't substring- match a JSON field, so without this, synonyms from `specifications.synonyms` ("slugger cutter", "turbo tough", etc.) would never match in the search box. The n8n workflow's Code node now flattens every searchable bit — including the synonyms array — into search_text on every POST. The 46 pre- existing rows were backfilled once with the same formula. The `price_list.supplier` field also stored the PB id on the 46 pre-existing rows (n8n was setting the resolved name, but the original LLM tests that seeded the table didn't). Fixed that in the same backfill — supplier now reads 'JEISA - WESTSCOTT GROUP' instead of 'q6hpwbxsicyoiuu', which makes the supplier filter dropdown work AND makes 'supplier ~ "jeisa"' match. Migration `2026-06-30-add-price-list-search-text`: - Adds `search_text` (text) column - Adds `idx_pl_search_text` non-unique index - Idempotent — re-runs are safe and self-heal the index Updates: - `getPriceLists` helper ORs across all 6 columns + search_text - `PriceList` TS type gains `search_text: string` - `pricelist-extraction.md` Code node snippet shows how the n8n workflow should populate `search_text` on every POST - Search input placeholder reads 'Search anything — SKU, description, brand, supplier, synonyms…' Tests: 229/230 (same pre-existing failure, unrelated). TypeScript clean. --- docs/pricelist-extraction.md | 50 +++++++---- package.json | 1 + .../2026-06-30-add-price-list-search-text.ts | 83 +++++++++++++++++++ src/components/pricelist-catalogue.tsx | 2 +- src/lib/pb-helpers.ts | 32 +++++-- src/lib/types.ts | 11 +++ 6 files changed, 156 insertions(+), 23 deletions(-) create mode 100644 scripts/migrations/2026-06-30-add-price-list-search-text.ts diff --git a/docs/pricelist-extraction.md b/docs/pricelist-extraction.md index 5207d42..995c368 100644 --- a/docs/pricelist-extraction.md +++ b/docs/pricelist-extraction.md @@ -28,8 +28,10 @@ WFDASH has two related collections, both already in PB: | `markup` | number | workflow default (25%) — operator override later | | `selling_price` | number | workflow computes: `cost_price × (1 + markup/100)` | | `stock_qty` | number | workflow default (0) — inventory updates later | - | `specifications` | json | AI: rich JSON (attributes + synonyms) | - | `created` / `updated` | autodate | — | +| `specifications` | json | AI: rich JSON (attributes + synonyms) | +| `source_file` | text | workflow: `raw_price_list.id` this row was extracted from | +| `search_text` | text | workflow: denormalised search blob (sku + description + brand + supplier + category + subcategory + specifications.synonyms) — PB can't substring-match JSON, so the Catalogue search box hits this column directly | +| `created` / `updated` | autodate | — | Indexed on `sku`, `category`, `subcategory`, `brand`, `supplier` — so the agent's "find me all HSS cutters 12mm from JEISA" becomes a @@ -342,20 +344,36 @@ const DEFAULT_MARKUP = 25; const rows = parsed.items .filter(it => it.cost_price != null) // skip POA / TBA - .map(it => ({ - sku: it.sku, - description: it.description, - category: it.category, - subcategory: it.subcategory || null, - brand: it.brand || null, - supplier, // denormalized from raw - cost_price: it.cost_price, - markup: DEFAULT_MARKUP, - selling_price: Math.round(it.cost_price * (1 + DEFAULT_MARKUP / 100) * 100) / 100, - unit: it.unit || 'each', - stock_qty: 0, - specifications: it.specifications, - })); + .map(it => { + const synonyms = Array.isArray(it.specifications?.synonyms) ? it.specifications.synonyms : []; + // search_text: denormalised blob of every searchable bit. PB + // can't substring-match JSON, so the Catalogue tab's search box + // hits this column directly to support synonym-based fuzzy + // matches ("slugger cutter" finds the HSS cutters whose + // specifications.synonyms contains it). Kept lowercase for + // case-insensitive matching on the PB side. + const searchText = [ + it.sku, it.description, it.brand, supplier, + it.category, it.subcategory, ...synonyms, + ].filter(Boolean).join(' ').toLowerCase().trim(); + + return { + sku: it.sku, + description: it.description, + category: it.category, + subcategory: it.subcategory || null, + brand: it.brand || null, + supplier, // denormalized from raw + cost_price: it.cost_price, + markup: DEFAULT_MARKUP, + selling_price: Math.round(it.cost_price * (1 + DEFAULT_MARKUP / 100) * 100) / 100, + unit: it.unit || 'each', + stock_qty: 0, + specifications: it.specifications, + source_file: rawId, // trace-back + verify key + search_text: searchText, // fuzzy search index + }; + }); return rows.map(r => ({ json: r })); ``` diff --git a/package.json b/package.json index f3cfbc0..a31d4ba 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,7 @@ "pb:migrate:raw-price-list": "npx tsx scripts/migrations/2026-06-30-add-raw-price-list.ts", "pb:migrate:pricelist-notes": "npx tsx scripts/migrations/2026-06-30-add-pricelist-notes.ts", "pb:migrate:price-list-source-file": "npx tsx scripts/migrations/2026-06-30-add-price-list-source-file.ts", + "pb:migrate:price-list-search-text": "npx tsx scripts/migrations/2026-06-30-add-price-list-search-text.ts", "smoke:counter": "npx tsx scripts/smoke/counter.ts", "smoke:counter-concurrent": "npx tsx scripts/smoke/counter-concurrent.ts", "smoke:create-from-email": "npx tsx scripts/smoke/create-from-email.ts", diff --git a/scripts/migrations/2026-06-30-add-price-list-search-text.ts b/scripts/migrations/2026-06-30-add-price-list-search-text.ts new file mode 100644 index 0000000..3128049 --- /dev/null +++ b/scripts/migrations/2026-06-30-add-price-list-search-text.ts @@ -0,0 +1,83 @@ +/** + * Migration — add `search_text` (text) to `price_list`. + * + * PB can't substring-match a JSON field, so for the Catalogue tab's + * "search everything" feature, we denormalize the searchable bits of + * each row into a flat text field. The WFDASH Catalogue helper + * (`getPriceLists`) then ORs against `search_text` and the other + * string columns. + * + * What's in search_text (per row, joined with spaces): + * - sku + * - description + * - brand + * - supplier + * - category + * - subcategory + * - specifications.synonyms (the buyer-friendly aliases) + * + * Idempotent: re-runs are safe. + * + * Run with: + * npm run pb:migrate:price-list-search-text + */ +import PocketBase from "pocketbase"; + +const PB_URL = process.env.NEXT_PUBLIC_POCKETBASE_URL || "http://127.0.0.1:8090"; +const ADMIN_EMAIL = process.env.PB_ADMIN_EMAIL; +const ADMIN_PASSWORD = process.env.PB_ADMIN_PASSWORD; + +if (!ADMIN_EMAIL || !ADMIN_PASSWORD) { + throw new Error("PB_ADMIN_EMAIL and PB_ADMIN_PASSWORD are required."); +} + +async function main() { + const pb = new PocketBase(PB_URL); + await pb.collection("_superusers").authWithPassword(ADMIN_EMAIL, ADMIN_PASSWORD); + + const cols = await pb.collections.getFullList(); + const priceList = cols.find((c) => c.name === "price_list"); + if (!priceList) { + throw new Error("price_list collection not found."); + } + + // Always re-check the index — running the migration on a + // collection that already has `search_text` should still ensure + // the index exists (handles the case where the field was added + // before the index step was introduced). + const INDEX_SQL = "CREATE INDEX `idx_pl_search_text` ON `price_list` (`search_text`)"; + const hasIndex = (priceList.indexes || []).some( + (i) => typeof i === "string" && i.includes("idx_pl_search_text"), + ); + + if (priceList.fields.some((f) => f.name === "search_text")) { + if (hasIndex) { + console.log("price_list.search_text + index already present, skipping."); + return; + } + await pb.collections.update(priceList.id, { indexes: [...(priceList.indexes || []), INDEX_SQL] }); + console.log("Added price_list index on search_text (field already existed)."); + console.log("search_text index migration complete."); + return; + } + + await pb.collections.update(priceList.id, { + fields: [ + ...priceList.fields, + { name: "search_text", type: "text", required: false, options: { max: 0 } }, + ], + // Add a non-unique index so the search box doesn't full-scan + // the table once we have thousands of rows. + indexes: [ + ...(priceList.indexes || []), + "CREATE INDEX `idx_pl_search_text` ON `price_list` (`search_text`)", + ], + }); + console.log("Added price_list.search_text (+ index)"); + console.log("search_text migration complete."); +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/src/components/pricelist-catalogue.tsx b/src/components/pricelist-catalogue.tsx index 22494d9..8d5ba32 100644 --- a/src/components/pricelist-catalogue.tsx +++ b/src/components/pricelist-catalogue.tsx @@ -328,7 +328,7 @@ export function PricelistCatalogue() {