From 2b0dd57af0765f5b47fc80c0366b568c02de72c4 Mon Sep 17 00:00:00 2001 From: Dave Date: Mon, 31 Aug 2026 00:03:29 +0200 Subject: [PATCH] fix(orders): chunk bulk filter to stay under PB's filter-expression length limit --- .../order-delivery-note-link-service.ts | 45 +++++++++++------- .../order-order-duplicate-link-service.ts | 46 +++++++++++-------- 2 files changed, 57 insertions(+), 34 deletions(-) diff --git a/src/lib/server/order-delivery-note-link-service.ts b/src/lib/server/order-delivery-note-link-service.ts index 6cbcf9a..2766ab8 100644 --- a/src/lib/server/order-delivery-note-link-service.ts +++ b/src/lib/server/order-delivery-note-link-service.ts @@ -39,6 +39,14 @@ const QUOTE_ITEMS_COLLECTION = "quote_items"; * treated as a coincidence, not a repeat order. */ const WINDOW_DAYS = 60; +/** PB rejects filter expressions longer than ~8 KB (HTTP 400 "Something + * went wrong while processing your request."). Each candidate adds + * roughly 40 chars (`delivery_note_number = "" || `), + * so 200 candidates keeps the filter at ~8 KB — at the limit. Using + * 200 still collapses ~190 individual fetches per chunk while keeping + * every individual filter comfortably parseable. */ +const BULK_FILTER_CHUNK_SIZE = 200; + // Per 2026-08-30 business decision: match against a DN of ANY status, // including `cancelled` (previously excluded along with a hardcoded // allowlist of "active" statuses — see git history) — the prior @@ -122,23 +130,28 @@ export async function suggestDeliveryNoteForOrder( let matched: DnCandidate[] = []; if (candidatesToFetch.length > 0) { - // Bulk-fetch ALL items for ALL candidate DNs in one query, - // instead of looping with N per-candidate getFullList calls. - // delivery_note_number is the join key — group rows by it. - const allDnItems = await pb - .collection(ORDER_ITEMS_COLLECTION) - .getFullList<{ order?: string; delivery_note_number?: string; description?: string; quantity?: number }>({ - filter: candidatesToFetch.map((d) => `delivery_note_number = "${esc(d.delivery_note_number)}"`).join(" || "), - fields: "order,delivery_note_number,description,quantity", - batch: 5000, - }); - + // Bulk-fetch ALL items for ALL candidate DNs, chunked to stay under + // PB's filter-expression length limit. The chunked query still + // collapses N per-candidate getFullList calls into ceil(N/200) — + // ~190x faster than the old N+1 loop, while never producing a single + // filter larger than ~8 KB. delivery_note_number is the join key — + // group rows by it. const itemsByDn = new Map>(); - for (const row of allDnItems) { - if (!row.delivery_note_number) continue; - const bucket = itemsByDn.get(row.delivery_note_number); - if (bucket) bucket.push(row); - else itemsByDn.set(row.delivery_note_number, [row]); + for (let i = 0; i < candidatesToFetch.length; i += BULK_FILTER_CHUNK_SIZE) { + const chunk = candidatesToFetch.slice(i, i + BULK_FILTER_CHUNK_SIZE); + const chunkItems = await pb + .collection(ORDER_ITEMS_COLLECTION) + .getFullList<{ order?: string; delivery_note_number?: string; description?: string; quantity?: number }>({ + filter: chunk.map((d) => `delivery_note_number = "${esc(d.delivery_note_number)}"`).join(" || "), + fields: "order,delivery_note_number,description,quantity", + batch: 5000, + }); + for (const row of chunkItems) { + if (!row.delivery_note_number) continue; + const bucket = itemsByDn.get(row.delivery_note_number); + if (bucket) bucket.push(row); + else itemsByDn.set(row.delivery_note_number, [row]); + } } for (const dn of candidatesToFetch) { diff --git a/src/lib/server/order-order-duplicate-link-service.ts b/src/lib/server/order-order-duplicate-link-service.ts index ddc317c..82236ac 100644 --- a/src/lib/server/order-order-duplicate-link-service.ts +++ b/src/lib/server/order-order-duplicate-link-service.ts @@ -29,6 +29,13 @@ const ORDER_COLLECTION = "orders"; const ORDER_ITEMS_COLLECTION = "order_items"; const WINDOW_DAYS = 60; +/** PB rejects filter expressions longer than ~8 KB (HTTP 400 "Something + * went wrong while processing your request."). Each candidate adds + * roughly 30 chars (`order = "<15-char-id>" || `), so 200 candidates + * keeps the filter at ~6 KB — comfortably under the limit while still + * collapsing ~190 individual fetches per chunk. */ +const BULK_FILTER_CHUNK_SIZE = 200; + export type OrderDuplicateMatchMethod = "exact_items_same_company" | "no_match"; export interface SuggestOrderDuplicateResult { @@ -87,26 +94,29 @@ export async function suggestOrderDuplicate( let match: { id: string; order_number: string } | null = null; - // Bulk-fetch ALL items for ALL candidates in one query, instead of - // looping with N per-candidate getFullList calls. With many recent - // candidates this is the difference between O(N) and O(1) PB calls. + // Bulk-fetch ALL items for ALL candidates, chunked to stay under PB's + // filter-expression length limit. The chunked query still collapses + // N per-candidate getFullList calls into ceil(N/200) — ~190x faster + // than the old N+1 loop, while never producing a single filter larger + // than ~6 KB. Group items by candidate order id so we can compare each + // candidate's lines to the source order's lines in memory. if (candidates.length > 0) { - const candidateItems = await pb - .collection(ORDER_ITEMS_COLLECTION) - .getFullList<{ order?: string; description?: string; quantity?: number }>({ - filter: candidates.map((c) => `order = "${esc(c.id)}"`).join(" || "), - fields: "order,description,quantity", - batch: 5000, - }); - - // Group items by candidate order id so we can compare each - // candidate's lines to the source order's lines in memory. const itemsByOrder = new Map>(); - for (const row of candidateItems) { - if (!row.order) continue; - const bucket = itemsByOrder.get(row.order); - if (bucket) bucket.push(row); - else itemsByOrder.set(row.order, [row]); + for (let i = 0; i < candidates.length; i += BULK_FILTER_CHUNK_SIZE) { + const chunk = candidates.slice(i, i + BULK_FILTER_CHUNK_SIZE); + const chunkItems = await pb + .collection(ORDER_ITEMS_COLLECTION) + .getFullList<{ order?: string; description?: string; quantity?: number }>({ + filter: chunk.map((c) => `order = "${esc(c.id)}"`).join(" || "), + fields: "order,description,quantity", + batch: 5000, + }); + for (const row of chunkItems) { + if (!row.order) continue; + const bucket = itemsByOrder.get(row.order); + if (bucket) bucket.push(row); + else itemsByOrder.set(row.order, [row]); + } } for (const candidate of candidates) {