David/WFDASH-WEBAPP
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(orders): chunk bulk filter to stay under PB's filter-expression length limit

Browse source
This commit is contained in:
Dave 2026-08-31 00:03:29 +02:00
parent 926be6c57b
commit 2b0dd57af0
2 changed files with 57 additions and 34 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

45
src/lib/server/order-delivery-note-link-service.ts
View file

@ -39,6 +39,14 @@ const QUOTE_ITEMS_COLLECTION = "quote_items";
* treated as a coincidence, not a repeat order. */
const WINDOW_DAYS = 60;
/** PB rejects filter expressions longer than ~8 KB (HTTP 400 "Something
* went wrong while processing your request."). Each candidate adds
* roughly 40 chars (`delivery_note_number = "<up-to-20-char-no>" || `),
* so 200 candidates keeps the filter at ~8 KB — at the limit. Using
* 200 still collapses ~190 individual fetches per chunk while keeping
* every individual filter comfortably parseable. */
const BULK_FILTER_CHUNK_SIZE = 200;
// Per 2026-08-30 business decision: match against a DN of ANY status,
// including `cancelled` (previously excluded along with a hardcoded
// allowlist of "active" statuses — see git history) — the prior
@ -122,23 +130,28 @@ export async function suggestDeliveryNoteForOrder(
let matched: DnCandidate[] = [];
if (candidatesToFetch.length > 0) {
// Bulk-fetch ALL items for ALL candidate DNs in one query,
// instead of looping with N per-candidate getFullList calls.
// delivery_note_number is the join key — group rows by it.
const allDnItems = await pb
.collection(ORDER_ITEMS_COLLECTION)
.getFullList<{ order?: string; delivery_note_number?: string; description?: string; quantity?: number }>({
filter: candidatesToFetch.map((d) => `delivery_note_number = "${esc(d.delivery_note_number)}"`).join(" || "),
fields: "order,delivery_note_number,description,quantity",
batch: 5000,
});
// Bulk-fetch ALL items for ALL candidate DNs, chunked to stay under
// PB's filter-expression length limit. The chunked query still
// collapses N per-candidate getFullList calls into ceil(N/200) —
// ~190x faster than the old N+1 loop, while never producing a single
// filter larger than ~8 KB. delivery_note_number is the join key —
// group rows by it.
const itemsByDn = new Map<string, Array<{ order?: string; description?: string; quantity?: number }>>();
for (const row of allDnItems) {
if (!row.delivery_note_number) continue;
const bucket = itemsByDn.get(row.delivery_note_number);
if (bucket) bucket.push(row);
else itemsByDn.set(row.delivery_note_number, [row]);
for (let i = 0; i < candidatesToFetch.length; i += BULK_FILTER_CHUNK_SIZE) {
const chunk = candidatesToFetch.slice(i, i + BULK_FILTER_CHUNK_SIZE);
const chunkItems = await pb
.collection(ORDER_ITEMS_COLLECTION)
.getFullList<{ order?: string; delivery_note_number?: string; description?: string; quantity?: number }>({
filter: chunk.map((d) => `delivery_note_number = "${esc(d.delivery_note_number)}"`).join(" || "),
fields: "order,delivery_note_number,description,quantity",
batch: 5000,
});
for (const row of chunkItems) {
if (!row.delivery_note_number) continue;
const bucket = itemsByDn.get(row.delivery_note_number);
if (bucket) bucket.push(row);
else itemsByDn.set(row.delivery_note_number, [row]);
}
}
for (const dn of candidatesToFetch) {

46
src/lib/server/order-order-duplicate-link-service.ts
View file

@ -29,6 +29,13 @@ const ORDER_COLLECTION = "orders";
const ORDER_ITEMS_COLLECTION = "order_items";
const WINDOW_DAYS = 60;
/** PB rejects filter expressions longer than ~8 KB (HTTP 400 "Something
* went wrong while processing your request."). Each candidate adds
* roughly 30 chars (`order = "<15-char-id>" || `), so 200 candidates
* keeps the filter at ~6 KB — comfortably under the limit while still
* collapsing ~190 individual fetches per chunk. */
const BULK_FILTER_CHUNK_SIZE = 200;
export type OrderDuplicateMatchMethod = "exact_items_same_company" | "no_match";
export interface SuggestOrderDuplicateResult {
@ -87,26 +94,29 @@ export async function suggestOrderDuplicate(
let match: { id: string; order_number: string } | null = null;
// Bulk-fetch ALL items for ALL candidates in one query, instead of
// looping with N per-candidate getFullList calls. With many recent
// candidates this is the difference between O(N) and O(1) PB calls.
// Bulk-fetch ALL items for ALL candidates, chunked to stay under PB's
// filter-expression length limit. The chunked query still collapses
// N per-candidate getFullList calls into ceil(N/200) — ~190x faster
// than the old N+1 loop, while never producing a single filter larger
// than ~6 KB. Group items by candidate order id so we can compare each
// candidate's lines to the source order's lines in memory.
if (candidates.length > 0) {
const candidateItems = await pb
.collection(ORDER_ITEMS_COLLECTION)
.getFullList<{ order?: string; description?: string; quantity?: number }>({
filter: candidates.map((c) => `order = "${esc(c.id)}"`).join(" || "),
fields: "order,description,quantity",
batch: 5000,
});
// Group items by candidate order id so we can compare each
// candidate's lines to the source order's lines in memory.
const itemsByOrder = new Map<string, Array<{ description?: string; quantity?: number }>>();
for (const row of candidateItems) {
if (!row.order) continue;
const bucket = itemsByOrder.get(row.order);
if (bucket) bucket.push(row);
else itemsByOrder.set(row.order, [row]);
for (let i = 0; i < candidates.length; i += BULK_FILTER_CHUNK_SIZE) {
const chunk = candidates.slice(i, i + BULK_FILTER_CHUNK_SIZE);
const chunkItems = await pb
.collection(ORDER_ITEMS_COLLECTION)
.getFullList<{ order?: string; description?: string; quantity?: number }>({
filter: chunk.map((c) => `order = "${esc(c.id)}"`).join(" || "),
fields: "order,description,quantity",
batch: 5000,
});
for (const row of chunkItems) {
if (!row.order) continue;
const bucket = itemsByOrder.get(row.order);
if (bucket) bucket.push(row);
else itemsByOrder.set(row.order, [row]);
}
}
for (const candidate of candidates) {