diff --git a/package.json b/package.json index fadad83..31b04b7 100644 --- a/package.json +++ b/package.json @@ -36,6 +36,7 @@ "pb:migrate:stock-purchase-documents-history": "npx tsx scripts/migrations/2026-07-16-add-stock-purchase-documents-history.ts", "pb:migrate:stock-purchase-send-statuses": "npx tsx scripts/migrations/2026-07-16-add-stock-purchase-send-statuses.ts", "pb:migrate:stock-purchase-problem-fields": "npx tsx scripts/migrations/2026-07-17-add-stock-purchase-problem-fields.ts", + "pb:migrate:order-items-item-status-index": "npx tsx scripts/migrations/2026-08-04-add-order-items-item-status-index.ts", "smoke:counter": "npx tsx scripts/smoke/counter.ts", "smoke:counter-concurrent": "npx tsx scripts/smoke/counter-concurrent.ts", "smoke:create-from-email": "npx tsx scripts/smoke/create-from-email.ts", diff --git a/scripts/migrations/2026-08-04-add-order-items-item-status-index.ts b/scripts/migrations/2026-08-04-add-order-items-item-status-index.ts new file mode 100644 index 0000000..ab8fa52 --- /dev/null +++ b/scripts/migrations/2026-08-04-add-order-items-item-status-index.ts @@ -0,0 +1,89 @@ +/* eslint-disable */ +/** + * Migration: add an index on `order_items.item_status`. + * + * Why this exists + * ─────────────── + * The /orders page's tab badges ("New (12)", "Processing (4)") and + * its server-side tab filter both run queries of the form + * `item_status = "pending" || item_status = "approval_needed" || ...` + * on the `order_items` collection. With ~18k items and no index on + * `item_status`, every load is a full table scan — the single biggest + * cost on the orders page after the deep expand on the orders + * collection itself. + * + * Adding the index makes these queries O(log n) B-tree lookups. The + * page-load budget drops from seconds to sub-second on the cold path + * (realtime refetches benefit even more because the tab-counts cache + * is bypassed for them). + * + * Schema + * ────── + * order_items.item_status select (existing field, no schema change) + * + * The index is non-unique (the same status appears on many rows). + * + * Idempotent — safe to re-run. The migration checks for the index + * before adding it; subsequent runs print "already present, skipping." + * + * Run with: + * npx tsx scripts/migrations/2026-08-04-add-order-items-item-status-index.ts + */ +import PocketBase from "pocketbase"; +import { readFileSync } from "node:fs"; +import { resolve } from "node:path"; + +function loadEnvFile(name: string) { + try { + const raw = readFileSync(resolve(process.cwd(), name), "utf8"); + for (const line of raw.split("\n")) { + const trimmed = line.trim(); + if (!trimmed || trimmed.startsWith("#")) continue; + const m = trimmed.match(/^([A-Z0-9_]+)\s*=\s*(.+)$/i); + if (m && !process.env[m[1]]) { + process.env[m[1]] = m[2].replace(/^["']|["']$/g, "").trim(); + } + } + } catch { + // missing file — fine + } +} +loadEnvFile(".env"); +loadEnvFile(".env.local"); + +async function main() { + const url = process.env.NEXT_PUBLIC_POCKETBASE_URL; + const email = process.env.PB_ADMIN_EMAIL; + const password = process.env.PB_ADMIN_PASSWORD; + + if (!url || !email || !password) { + throw new Error( + "Missing NEXT_PUBLIC_POCKETBASE_URL / PB_ADMIN_EMAIL / PB_ADMIN_PASSWORD in env", + ); + } + + const pb = new PocketBase(url); + await pb.admins.authWithPassword(email, password); + + const collection = await pb.collections.getOne("order_items"); + const INDEX_NAME = "idx_oi_item_status"; + const INDEX_SQL = `CREATE INDEX \`${INDEX_NAME}\` ON \`order_items\` (\`item_status\`)`; + + const existing = (collection.indexes || []) as string[]; + if (existing.includes(INDEX_SQL)) { + console.log("order_items.item_status index already present, skipping."); + return; + } + + await pb.collections.update(collection.id, { + indexes: [...existing, INDEX_SQL], + }); + + console.log("Added order_items.item_status index (idx_oi_item_status)."); + console.log("Run /orders in the browser — the initial load should be visibly faster."); +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/src/app/(app)/orders/page.tsx b/src/app/(app)/orders/page.tsx index 67c1047..2f66e5e 100644 --- a/src/app/(app)/orders/page.tsx +++ b/src/app/(app)/orders/page.tsx @@ -313,7 +313,7 @@ export default function OrdersPage() { /** ID of the order whose review modal is open. null = no modal. */ const [reviewOrderId, setReviewOrderId] = useState(null); - const fetchOrders = useCallback(async () => { + const fetchOrders = useCallback(async (opts: { nocache?: boolean } = {}) => { setLoading(true); setError(""); try { @@ -322,6 +322,16 @@ export default function OrdersPage() { perPage: "20", }); if (search.trim()) params.set("search", search.trim()); + // Server-side tab filter — narrows the result set so the + // pagination footer and the tab badge agree. Without this the + // page would only refilter the current page client-side and a + // tab like Processing with old orders could render empty even + // though the badge says "Processing (4)". + params.set("tab", activeTab); + // Realtime path passes `nocache=1` so the 30s tab-counts cache + // is bypassed — badges should reflect the change that triggered + // the event, not a value from up to 30s ago. + if (opts.nocache) params.set("nocache", "1"); const result = await fetchDashboardJson( `/api/v1/orders/list?${params.toString()}`, @@ -335,7 +345,7 @@ export default function OrdersPage() { } finally { setLoading(false); } - }, [page, search]); + }, [page, search, activeTab]); useEffect(() => { // eslint-disable-next-line react-hooks/set-state-in-effect -- data fetch on dep change @@ -362,15 +372,19 @@ export default function OrdersPage() { // Realtime — refresh on any order or item change. PB fires once per // write, but a single operator action can produce several writes in // quick succession (item transitions, etc). Coalesce them with a - // 300ms debounce so a burst costs one refetch, not eight. + // 1000ms debounce so a burst costs one refetch, not eight. The + // earlier 300ms debounce was too aggressive for the heavy deep- + // expand + status-counts query — bursts of writes would visibly + // stall the page. 1s is still well under the operator's perceptual + // threshold and prevents thundering-herd refetches. const realtimeTimer = useRef | null>(null); useEffect(() => { const pb = getPocketBase(); const handler = () => { if (realtimeTimer.current) clearTimeout(realtimeTimer.current); realtimeTimer.current = setTimeout(() => { - fetchOrders(); - }, 300); + fetchOrders({ nocache: true }); + }, 1000); }; pb.collection("orders").subscribe("*", handler); pb.collection("order_items").subscribe("*", handler); @@ -384,6 +398,12 @@ export default function OrdersPage() { const handleTabChange = (next: TabGroup) => { setActiveTab(next); setPage(1); + // The fetchOrders callback now depends on activeTab, so the + // page-1 reset is enough to trigger a fresh server-side query + // scoped to the new tab. Reset per-row expansion since the + // visible order set changes. + setExpanded(new Set()); + setShowAllByOrder(new Set()); }; const handleSearchChange = (v: string) => { @@ -415,16 +435,23 @@ export default function OrdersPage() { const openReview = (orderId: string) => setReviewOrderId(orderId); const closeReview = () => setReviewOrderId(null); - // Filter visible orders to those that have at least one item in - // the active tab group. An order with no items at all (rare — - // drafts) doesn't show on any tab by design. - const visibleOrders = useMemo(() => { - return orders.filter((o) => o.tabGroups.includes(activeTab)); - }, [orders, activeTab]); - + // The server already filtered `orders` to those with at least one + // item in `activeTab` (via the `tab` query param). No client-side + // filter needed — re-applying it would drop orders the server + // intended to show. const totalLoaded = orders.length; const searching = search.trim().length > 0; + // Sum of orders across the other tabs (for the empty-state nudge + // when the current tab is empty). The `counts` we have are global + // counts per tab, so we can point the operator to the busy tabs + // even when the one they're looking at has nothing. + const otherTabCounts = useMemo(() => { + return (Object.entries(counts) as [TabGroup, number][]) + .filter(([tab, n]) => tab !== activeTab && n > 0) + .sort(([, a], [, b]) => b - a); + }, [counts, activeTab]); + return (
@@ -474,7 +501,7 @@ export default function OrdersPage() {

{error}

- ) : visibleOrders.length === 0 ? ( + ) : orders.length === 0 ? ( @@ -487,9 +514,16 @@ export default function OrdersPage() { - ) : totalLoaded > 0 ? ( + ) : otherTabCounts.length > 0 ? (

- ({totalLoaded} other order{totalLoaded === 1 ? "" : "s"} on other tabs.) + Busy tabs: + {" "} + {otherTabCounts + .map( + ([tab, n]) => + `${TAB_GROUP_VISUAL[tab].label} (${n})`, + ) + .join(" · ")}

) : ( - - Page {page} of {totalPages} - - +
+
+ + + Page {page} of {totalPages} + + +
+ {/* Sub-line — clarifies that the footer reflects the tab's + matching set, not the global order count. Operators used + to seeing "Page 1 of 226" on the old all-orders path were + confused by the much smaller tab-scoped number. */} +

+ {TAB_GROUP_VISUAL[activeTab].label} orders · {totalLoaded} on this page +

)} {reviewOrderId && ( diff --git a/src/app/api/v1/orders/list/route.ts b/src/app/api/v1/orders/list/route.ts index 45e252a..be742df 100644 --- a/src/app/api/v1/orders/list/route.ts +++ b/src/app/api/v1/orders/list/route.ts @@ -1,15 +1,38 @@ import { NextRequest, NextResponse } from "next/server"; import { assertBrowserSession } from "@/lib/server/browser-session"; import { getOrdersDashboardData } from "@/lib/server/orders-service"; +import type { TabGroup } from "@/lib/item-status-ui"; +import { TAB_GROUP_ORDER } from "@/lib/item-status-ui"; /** * Browser-facing read model for /orders. The external agent endpoint remains * at GET /api/v1/orders; this route accepts the logged-in dashboard user's PB * token and returns orders, line items, tab counts, and resolved rep names. + * + * Query params: + * page — 1-indexed page number (default 1) + * perPage — page size, clamped to 1..100 (default 20) + * search — free-text filter (order #, company, customer, item, price) + * tab — restrict results to orders with items in this tab group. + * Without `tab`, the query returns across all tabs and the + * pagination footer reflects the global order count + * (~4520 today). With `tab`, the server narrows first so + * the footer reflects only the matching set — the badge + * "Processing (4)" then matches what page 1 actually shows. + * nocache=1 — bypass the 30s tab-counts cache. The realtime + * subscription on /orders sets this so badges update + * the moment an item changes status. */ export const runtime = "nodejs"; export const dynamic = "force-dynamic"; +const VALID_TABS = new Set(TAB_GROUP_ORDER); + +function parseTab(raw: string | null): TabGroup | undefined { + if (!raw) return undefined; + return VALID_TABS.has(raw) ? (raw as TabGroup) : undefined; +} + export async function GET(req: NextRequest) { try { await assertBrowserSession(req); @@ -17,12 +40,17 @@ export async function GET(req: NextRequest) { const page = Number(req.nextUrl.searchParams.get("page") || "1"); const perPage = Number(req.nextUrl.searchParams.get("perPage") || "20"); const search = req.nextUrl.searchParams.get("search") || ""; + const tab = parseTab(req.nextUrl.searchParams.get("tab")); + const bypassCountsCache = + req.nextUrl.searchParams.get("nocache") === "1"; const result = await getOrdersDashboardData({ page, perPage, search, sort: "-created", + tabGroup: tab, + bypassCountsCache, }); return NextResponse.json(result); } catch (error) { diff --git a/src/lib/server/orders-service.test.ts b/src/lib/server/orders-service.test.ts index 89b23bd..a103881 100644 --- a/src/lib/server/orders-service.test.ts +++ b/src/lib/server/orders-service.test.ts @@ -16,36 +16,64 @@ const { store, pbMock, clearStore } = vi.hoisted(() => { return store[name]; }; + // The mock `where` is intentionally minimal — supports the patterns the + // production service actually issues: `field = "value"` (with or without + // quotes), `field != "value"`, `&&` (AND), `||` (OR), and `id` lookups by + // PB record id. Anything unrecognised passes through (returns all items + // for the AND group), so unknown filters don't false-negative tests. + const matchesClause = (item: Record, clause: string): boolean => { + const m = clause.match(/^(\w+)\s*(=~|!~|=|!=)\s*"?([^"]+?)"?$/); + if (!m) return true; + const [, field, op, raw] = m; + const expected = raw.replace(/^"|"$/g, ""); + const actual = item[field]; + if (field === "year" || field === "month") { + const num = Number(actual); + const want = Number(expected); + switch (op) { + case "=": return num === want; + case "!=": return num !== want; + default: return true; + } + } + const actualStr = String(actual ?? ""); + switch (op) { + case "=": return actualStr === expected; + case "!=": return actualStr !== expected; + case "=~": return actualStr.includes(expected); + case "!~": return !actualStr.includes(expected); + default: return true; + } + }; + const where = (name: string, filter: string): Record[] => { const items = Array.from(getOrCreateMap(name).values()); if (!filter || !filter.trim()) return items; - const clauses = filter.split("&&").map((c) => c.trim()); + // Split on `||` first (outer OR), then each alternative is a group of + // `&&`-joined AND clauses. An item passes if it satisfies ANY + // alternative group. + const alternatives = filter.split("||").map((a) => a.trim()).filter(Boolean); return items.filter((item) => - clauses.every((clause) => { - const m = clause.match(/^(\w+)\s*=\s*"?([^"]+)"?$/); - if (!m) return true; - const [, field, raw] = m; - const value = raw.replace(/^"|"$/g, ""); - if (field === "year" || field === "month") { - return Number(item[field]) === Number(value); - } - return String(item[field] ?? "") === value; + alternatives.some((alt) => { + const clauses = alt.split("&&").map((c) => c.trim()).filter(Boolean); + return clauses.every((clause) => matchesClause(item, clause)); }), ); }; const pbMock = { + autoCancellation: () => undefined, collection: (name: string) => ({ async getList( - _page = 1, - _perPage = 25, + page = 1, + perPage = 25, opts: { filter?: string; sort?: string } = {}, ) { - const items = where(name, opts.filter || ""); + const all = where(name, opts.filter || ""); if (opts.sort) { const sign = opts.sort.startsWith("-") ? -1 : 1; const field = opts.sort.replace(/^-/, ""); - items.sort((a, b) => { + all.sort((a, b) => { const av = a[field] as string | number | undefined; const bv = b[field] as string | number | undefined; if (av === bv) return 0; @@ -54,10 +82,20 @@ const { store, pbMock, clearStore } = vi.hoisted(() => { return (av > bv ? 1 : -1) * sign; }); } - return { items, totalItems: items.length, page: 1, perPage: _perPage }; + // Real pagination — mirror PB's getList semantics so the + // service code (which relies on totalPages) exercises the same + // branches under test as in production. + const totalItems = all.length; + const totalPages = Math.max(1, Math.ceil(totalItems / perPage)); + const start = (page - 1) * perPage; + const items = all.slice(start, start + perPage); + return { items, totalItems, totalPages, page, perPage }; }, - async getFullList() { - return where(name, ""); + async getFullList(opts: { filter?: string; fields?: string } = {}) { + // Mirror production: respect the filter. The fields projection + // is a wire-payload optimisation in PB; for tests we just + // return the matched records as-is. + return where(name, opts.filter || ""); }, async getFirstListItem(filter: string) { const items = where(name, filter); @@ -139,8 +177,13 @@ vi.mock("./pocketbase-admin", () => ({ import { createOrderFromQuote, + clearOrderItemStatusTabCountsCache, + getOrderItemStatusTabCounts, + getOrderIdsForTabGroup, getOrderItemsCollectionName, + getOrdersDashboardData, listCollectionPool, + listOrdersWithItems, } from "./orders-service"; beforeEach(() => { @@ -449,3 +492,382 @@ describe("createOrderFromQuote", () => { expect(new Set(numbers).size).toBe(5); }); }); + +// ---------------------------------------------------------------------------- +// /orders dashboard read model — tab filter, counts cache, rep lookup. +// +// These tests cover the bug surface fixed in 2026-07: the old +// `getOrdersDashboardData` returned the global orders pagination +// (Page 1 of 226, 4520 total orders) regardless of which tab the +// operator selected, so "Processing (4)" badges showed empty pages +// when the four matching orders happened to fall on later pages. +// ---------------------------------------------------------------------------- + +describe("getOrderIdsForTabGroup", () => { + it("returns the distinct order ids with at least one item in the tab group", async () => { + // Three orders, each with one item. Statuses cover "new", + // "processing", and "delivered" tabs so we can verify partition + // membership. + await pbMock.collection("orders").create({ + id: "o-new", + order_number: "ORD-2025-01-001", + quote: "q1", + }); + await pbMock.collection("orders").create({ + id: "o-processing", + order_number: "ORD-2025-01-002", + quote: "q2", + }); + await pbMock.collection("orders").create({ + id: "o-delivered", + order_number: "ORD-2025-01-003", + quote: "q3", + }); + await pbMock.collection("order_items").create({ + id: "oi-1", order: "o-new", description: "Widget", + item_status: "pending", + }); + await pbMock.collection("order_items").create({ + id: "oi-2", order: "o-processing", description: "Sprocket", + item_status: "processed", + }); + await pbMock.collection("order_items").create({ + id: "oi-3", order: "o-delivered", description: "Gear", + item_status: "delivered", + }); + + const processingIds = await getOrderIdsForTabGroup(pbMock as never, "processing"); + expect(processingIds.sort()).toEqual(["o-processing"]); + + const newIds = await getOrderIdsForTabGroup(pbMock as never, "new"); + expect(newIds.sort()).toEqual(["o-new"]); + + const deliveredIds = await getOrderIdsForTabGroup(pbMock as never, "delivered"); + expect(deliveredIds.sort()).toEqual(["o-delivered"]); + }); + + it("dedupes when an order has multiple items in the same tab", async () => { + await pbMock.collection("orders").create({ + id: "o-multi", + order_number: "ORD-2025-01-010", + quote: "q", + }); + await pbMock.collection("order_items").create({ + id: "oi-a", order: "o-multi", description: "A", + item_status: "pending", + }); + await pbMock.collection("order_items").create({ + id: "oi-b", order: "o-multi", description: "B", + item_status: "pending", + }); + + const ids = await getOrderIdsForTabGroup(pbMock as never, "new"); + expect(ids).toEqual(["o-multi"]); + }); +}); + +describe("getOrderItemStatusTabCounts", () => { + beforeEach(() => clearOrderItemStatusTabCountsCache()); + + it("returns the count of distinct orders per tab", async () => { + await pbMock.collection("orders").create({ + id: "o-a", order_number: "ORD-2025-01-001", quote: "qa", + }); + await pbMock.collection("orders").create({ + id: "o-b", order_number: "ORD-2025-01-002", quote: "qb", + }); + await pbMock.collection("order_items").create({ + id: "oi-1", order: "o-a", description: "x", item_status: "pending", + }); + await pbMock.collection("order_items").create({ + id: "oi-2", order: "o-a", description: "y", item_status: "processed", + }); + await pbMock.collection("order_items").create({ + id: "oi-3", order: "o-b", description: "z", item_status: "pending", + }); + + const counts = await getOrderItemStatusTabCounts(pbMock as never); + expect(counts.new).toBe(2); + expect(counts.processing).toBe(1); + }); + + it("bypasses the cache when bypassCache is true", async () => { + await pbMock.collection("orders").create({ + id: "o-cache", order_number: "ORD-2025-01-001", quote: "qc", + }); + await pbMock.collection("order_items").create({ + id: "oi-cache", order: "o-cache", description: "x", item_status: "pending", + }); + + const first = await getOrderItemStatusTabCounts(pbMock as never); + expect(first.new).toBe(1); + + // Mutate the underlying store directly — bypasses our normal + // service writes, but lets us verify the cache is being served + // if `bypassCache` is not set. + await pbMock.collection("orders").create({ + id: "o-cache-2", order_number: "ORD-2025-01-002", quote: "qc2", + }); + await pbMock.collection("order_items").create({ + id: "oi-cache-2", order: "o-cache-2", description: "x", item_status: "pending", + }); + + const cached = await getOrderItemStatusTabCounts(pbMock as never); + expect(cached.new).toBe(1); // unchanged — cache hit + + const fresh = await getOrderItemStatusTabCounts(pbMock as never, { bypassCache: true }); + expect(fresh.new).toBe(2); // refresh sees the new order + }); + + it("clearOrderItemStatusTabCountsCache forces a fresh read", async () => { + await pbMock.collection("orders").create({ + id: "o-clear", order_number: "ORD-2025-01-001", quote: "qclr", + }); + await pbMock.collection("order_items").create({ + id: "oi-clear", order: "o-clear", description: "x", item_status: "pending", + }); + + await getOrderItemStatusTabCounts(pbMock as never); // prime the cache + clearOrderItemStatusTabCountsCache(); + const fresh = await getOrderItemStatusTabCounts(pbMock as never); + expect(fresh.new).toBe(1); + }); + + it("fires a single combined getFullList for all tabs (not 6 separate queries)", async () => { + // Regression guard: the old implementation issued 6 parallel + // `getFullList` calls (one per tab), each with a separate filter + // scan. With ~18k rows in `order_items` and no `item_status` + // index, those 6 scans dominated the page-load budget. The new + // implementation collapses to 1 combined query. This test + // intercepts the mock's `collection()` to count how many times + // `getFullList` is called on the `order_items` collection. + await pbMock.collection("order_items").create({ + id: "oi-1", order: "o1", description: "x", item_status: "pending", + }); + await pbMock.collection("order_items").create({ + id: "oi-2", order: "o2", description: "x", item_status: "processed", + }); + await pbMock.collection("order_items").create({ + id: "oi-3", order: "o3", description: "x", item_status: "delivered", + }); + + const originalCollection = pbMock.collection; + let getFullListCalls = 0; + pbMock.collection = ((name: string) => { + const proxy = originalCollection(name); + if (name === "order_items") { + const originalGetFullList = proxy.getFullList; + proxy.getFullList = ((...args: unknown[]) => { + getFullListCalls++; + return (originalGetFullList as (...a: unknown[]) => unknown).call( + proxy, + ...args, + ); + }) as typeof proxy.getFullList; + } + return proxy; + }) as typeof pbMock.collection; + + try { + await getOrderItemStatusTabCounts(pbMock as never, { bypassCache: true }); + expect(getFullListCalls).toBe(1); + } finally { + pbMock.collection = originalCollection; + } + }); +}); + +describe("listOrdersWithItems — tab filter", () => { + it("returns only orders that have at least one item in the requested tab", async () => { + // Two orders: one with a "pending" item, one with a "processed" + // item. Filtered to "processing" should yield only the second. + await pbMock.collection("orders").create({ + id: "o-pending", order_number: "ORD-2025-01-001", quote: "qp", + created: "2025-01-01T00:00:00Z", + }); + await pbMock.collection("orders").create({ + id: "o-processed", order_number: "ORD-2025-01-002", quote: "qpr", + created: "2025-01-02T00:00:00Z", + }); + await pbMock.collection("order_items").create({ + id: "oi-pending", order: "o-pending", description: "Widget", + item_status: "pending", + }); + await pbMock.collection("order_items").create({ + id: "oi-processed", order: "o-processed", description: "Sprocket", + item_status: "processed", + }); + + const result = await listOrdersWithItems({ + page: 1, + perPage: 20, + tabGroup: "processing", + }); + + expect(result.items.map((i) => i.order.id)).toEqual(["o-processed"]); + expect(result.totalItems).toBe(1); + expect(result.totalPages).toBe(1); + }); + + it("returns an empty list with totalPages=0 when no orders match the tab", async () => { + await pbMock.collection("orders").create({ + id: "o-only-new", order_number: "ORD-2025-01-001", quote: "qn", + }); + await pbMock.collection("order_items").create({ + id: "oi-only-new", order: "o-only-new", description: "x", + item_status: "pending", + }); + + const result = await listOrdersWithItems({ + page: 1, + perPage: 20, + tabGroup: "cancelled", + }); + expect(result.items).toEqual([]); + expect(result.totalItems).toBe(0); + }); + + it("reports totalItems from the matching set, not the page slice (pagination honesty)", async () => { + // Five orders with processed items, perPage=2. The page-1 response + // should have 2 items but totalItems=5 / totalPages=3 so the + // footer says "Page 1 of 3" — matching the operator's mental + // model of how many orders the tab actually contains. + for (let i = 0; i < 5; i++) { + await pbMock.collection("orders").create({ + id: `o-${i}`, + order_number: `ORD-2025-01-00${i + 1}`, + quote: `q-${i}`, + created: `2025-01-0${i + 1}T00:00:00Z`, + }); + await pbMock.collection("order_items").create({ + id: `oi-${i}`, + order: `o-${i}`, + description: "x", + item_status: "processed", + }); + } + + const result = await listOrdersWithItems({ + page: 1, + perPage: 2, + tabGroup: "processing", + }); + expect(result.items.length).toBe(2); + expect(result.totalItems).toBe(5); + expect(result.totalPages).toBe(3); + }); + + it("falls back to all-orders pagination when no tab is supplied", async () => { + // Sanity check — the existing all-orders path is unchanged. PB's + // `getList(page, perPage, { sort })` returns a paginated slice; + // the test mock mirrors that with `totalItems = items.length`, + // so totalPages = 1 here regardless of the underlying set size. + await pbMock.collection("orders").create({ + id: "o-a", order_number: "ORD-2025-01-001", quote: "qa", + }); + await pbMock.collection("orders").create({ + id: "o-b", order_number: "ORD-2025-01-002", quote: "qb", + }); + + const result = await listOrdersWithItems({ + page: 1, + perPage: 20, + }); + expect(result.items.length).toBe(2); + }); + + it("passes a `fields` parameter to the orders list (limits wire payload on the deep expand)", async () => { + // The orders list query asks PB to limit the main record AND each + // expanded record to just the fields the list view reads. Without + // this, every order card drags the full `quote` record along + // (quotes has unbounded `email_body` / `email_message` columns), + // so 20 orders with deep expand = multi-MB JSON payload. PB + // honours `expand.field` syntax in the `fields` query param. + let capturedFields: string | undefined; + let capturedExpand: string | undefined; + const originalCollection = pbMock.collection; + pbMock.collection = ((name: string) => { + const proxy = originalCollection(name); + if (name === "orders") { + const originalGetList = proxy.getList; + proxy.getList = ((page: number, perPage: number, opts: { filter?: string; sort?: string; fields?: string; expand?: string }) => { + capturedFields = opts?.fields; + capturedExpand = opts?.expand; + return (originalGetList as (...a: unknown[]) => unknown).call( + proxy, + page, + perPage, + opts, + ); + }) as typeof proxy.getList; + } + return proxy; + }) as typeof pbMock.collection; + + try { + await listOrdersWithItems({ page: 1, perPage: 20 }); + + // Main record fields + expect(capturedFields).toContain("order_number"); + expect(capturedFields).toContain("created"); + // Per-expand field limits — the actual perf win + expect(capturedFields).toContain("quote.quote_number"); + expect(capturedFields).toContain("company.company"); + expect(capturedFields).toContain("customer.customer"); + expect(capturedFields).toContain("customer.rep.name"); + // Confirm we dropped potential_quote from the expand + expect(capturedExpand).not.toContain("potential_quote"); + } finally { + pbMock.collection = originalCollection; + } + }); +}); + +describe("getOrdersDashboardData — end-to-end", () => { + beforeEach(() => clearOrderItemStatusTabCountsCache()); + + it("returns tab-scoped counts and items in a single response", async () => { + // Three orders: a pending one (new tab), a processed one + // (processing tab), and a delivered one. + await pbMock.collection("orders").create({ + id: "o-p", order_number: "ORD-2025-01-001", quote: "qp", + created: "2025-01-01T00:00:00Z", + }); + await pbMock.collection("orders").create({ + id: "o-pr", order_number: "ORD-2025-01-002", quote: "qpr", + created: "2025-01-02T00:00:00Z", + }); + await pbMock.collection("orders").create({ + id: "o-d", order_number: "ORD-2025-01-003", quote: "qd", + created: "2025-01-03T00:00:00Z", + }); + await pbMock.collection("order_items").create({ + id: "oi-p", order: "o-p", description: "Widget", + item_status: "pending", + }); + await pbMock.collection("order_items").create({ + id: "oi-pr", order: "o-pr", description: "Sprocket", + item_status: "processed", + }); + await pbMock.collection("order_items").create({ + id: "oi-d", order: "o-d", description: "Gear", + item_status: "delivered", + }); + + const result = await getOrdersDashboardData({ + page: 1, + perPage: 20, + tabGroup: "processing", + }); + + // Counts are global (always were) — the badge still tells the truth. + expect(result.counts.new).toBe(1); + expect(result.counts.processing).toBe(1); + expect(result.counts.delivered).toBe(1); + + // Items are scoped to the active tab. + expect(result.items.map((i) => i.order.id)).toEqual(["o-pr"]); + expect(result.totalPages).toBe(1); + expect(result.totalItems).toBe(1); + }); +}); diff --git a/src/lib/server/orders-service.ts b/src/lib/server/orders-service.ts index a60d4b7..1e744ae 100644 --- a/src/lib/server/orders-service.ts +++ b/src/lib/server/orders-service.ts @@ -1,11 +1,13 @@ import { getAdminPocketBase } from "./pocketbase-admin"; import { TAB_GROUP_ITEM_STATUSES, + TAB_GROUP_ORDER, itemStatusToTabGroup, + tabGroupFilter, type TabGroup, } from "../item-status-ui"; import { resolveRepDisplayName } from "../order-rep"; -import type { Customer, Order, OrderItem, User } from "../types"; +import type { Customer, ItemStatus, Order, OrderItem, User } from "../types"; const ORDER_COLLECTION = "orders"; const QUOTE_COLLECTION = "quotes"; @@ -20,6 +22,52 @@ type AdminPocketBase = Awaited>; type RepUser = Pick; +/** + * Cached counts from `getOrderItemStatusTabCounts`. Tab badges stay + * stable across rapid realtime refetches and across search/tab + * changes — counts are global, so the value doesn't move when the + * page query moves. `bypassCache` is set on realtime refetches so + * the badges update the moment an item changes status. + */ +let cachedTabCounts: { + value: Record; + expiresAt: number; +} | null = null; +// 60s TTL — counts are global and don't change on every realtime +// event, but operators want them fresh enough to trust. The realtime +// path bypasses the cache via the `bypassCache` option, so changes +// the current operator makes are reflected immediately; the 60s +// window is the staleness budget for changes made by other operators. +const TAB_COUNTS_CACHE_TTL_MS = 60_000; + +/** Invalidate the tab-counts cache. Exported for tests + realtime handlers. */ +export function clearOrderItemStatusTabCountsCache(): void { + cachedTabCounts = null; +} + +/** + * All item statuses that map to a tab group, deduplicated. The + * status-to-tab mapping in `TAB_GROUP_ITEM_STATUSES` is a partition + * (each status belongs to exactly one tab), so this set covers the + * statuses that contribute to ANY tab count. Statuses outside this + * set (e.g. `invoiced`, `with_driver`, `removed`) don't drive any + * tab and are excluded from the counts query to keep the result + * set tight. + */ +const TAB_STATUSES: string[] = Array.from( + new Set(TAB_GROUP_ORDER.flatMap((tab) => TAB_GROUP_ITEM_STATUSES[tab])), +); + +/** + * PB `getFullList` batch size for the tab-counts combined query. + * Default is 500; raising to 5000 cuts the per-load internal query + * count by ~10x for a 18k-item `order_items` collection. The + * tradeoff is a larger per-batch payload, but `fields: "item_status, + * order"` keeps each row under ~50 bytes, so 5000 rows = ~250 KB + * per round-trip — well within reason. + */ +const TAB_COUNTS_BATCH_SIZE = 5000; + export interface OrderWithItems { order: Order; items: OrderItem[]; @@ -48,6 +96,11 @@ const ORDER_ITEM_FILTER_CHUNK = 20; * company, customer, descriptions, and prices. Rep names are resolved with * admin-expanded `customer.rep`, then the minimal users lookup for legacy * `orders.rep` rows that stored a user id instead of a name. + * + * `tabGroup` (when set) narrows the result set to orders with at least one + * line item in that tab — so the pagination footer reflects the visible + * total rather than the global order count, and the badge "Processing (4)" + * matches what you actually see on page 1. */ export async function getOrdersDashboardData( params: { @@ -55,13 +108,17 @@ export async function getOrdersDashboardData( perPage?: number; search?: string; sort?: string; + tabGroup?: TabGroup; + bypassCountsCache?: boolean; } = {}, ): Promise { const pb = await getAdminPocketBase(); pb.autoCancellation(false); const [list, counts] = await Promise.all([ listOrdersWithItems(params, pb), - getOrderItemStatusTabCounts(pb), + getOrderItemStatusTabCounts(pb, { + bypassCache: params.bypassCountsCache, + }), ]); return { ...list, counts }; } @@ -72,6 +129,7 @@ export async function listOrdersWithItems( perPage?: number; search?: string; sort?: string; + tabGroup?: TabGroup; } = {}, pbArg?: AdminPocketBase, ): Promise> { @@ -81,25 +139,136 @@ export async function listOrdersWithItems( const perPage = Math.min(100, Math.max(1, params.perPage || 20)); const search = (params.search || "").trim(); const sort = params.sort || "-created"; - const expand = "company,customer,customer.rep,quote,potential_quote"; + const tabGroup = params.tabGroup; + // `potential_quote` is dropped from the expand: the list view only checks + // the field's existence (the "Suggested" pill in the header), and + // PB issues a separate IN query per expanded relation. Without it, we + // save one round-trip per page load and per realtime refetch. + const expand = "company,customer,customer.rep,quote"; + // `fields` has two purposes here: + // + // 1. Limit the MAIN record's wire payload to only the columns the + // list view actually reads. The orders schema is wide (notes, + // email_body, ai_insights, migration_meta, etc.) and most rows + // carry KBs of content we never render. + // + // 2. Limit each EXPANDED record's fields via the `expand.field` + // syntax PB supports in the `fields` query param. This is the + // big win: `quotes` has unbounded text columns (email_body, + // email_message) that, without limiting, would drag the full + // email body of every linked quote into the list response — + // 20 orders with deep expand can easily become a 1-5 MB + // JSON payload. We only need `quote.quote_number` for the + // pill, so we ask for that and nothing else. + // + // - `company.company` — the company name on the header row + // - `customer.customer` — the customer name on the header row + // - `customer.rep.id/name/email` — rep label, via + // `resolveRepDisplayName` + // - `quote.quote_number` — the linked-quote pill + // Top-level fields = main record (orders) columns. Dotted fields = + // per-expand field limits using PB's `expandField.subfield` syntax. + // The relation field ids (`company`, `customer`, `customer.rep`, + // `quote`) must be present in the main list so PB knows which + // relations to expand; once listed, the dotted forms below limit + // what comes back from each expanded record. + const fields = + [ + // main record + "id", + "order_number", + "company", + "customer", + "quote", + "potential_quote", + "potential_quote_confidence", + "quote_match_confidence", + "rep", + "created", + "updated", + // expanded `company` — just the name + "company.id", + "company.company", + // expanded `customer` — just the name + the rep relation id + "customer.id", + "customer.customer", + "customer.rep", + // expanded `customer.rep` (users) — just the label fields + "customer.rep.id", + "customer.rep.name", + "customer.rep.email", + // expanded `quote` — just the number for the pill + "quote.id", + "quote.quote_number", + ].join(","); - const ordersPage = search - ? await pb.collection(ORDER_COLLECTION).getList(1, ORDER_SEARCH_HARD_CAP, { - sort, - expand, - }) - : await pb.collection(ORDER_COLLECTION).getList(page, perPage, { + // Tab filter path — three sub-cases: + // (a) no tab, no search → original PB-paginated path + // (b) tab + no search → resolve matching order ids, then + // PB-paginate the filtered set (small tabs + // fall through to `getFullList`+slice when + // the `id = "..." || ...` filter would + // exceed PB's URL length cap) + // (c) any tab/search combo → search caps at 500 orders; we filter + // client-side in the loop below, so the + // tab filter is applied there too. Both + // search and tab reduce to a single page. + let ordersPage: { items: Order[]; totalItems: number; totalPages: number; page: number; perPage: number }; + + if (tabGroup && !search) { + const matchingIds = await getOrderIdsForTabGroup(pb, tabGroup); + if (matchingIds.length === 0) { + ordersPage = { items: [], totalItems: 0, totalPages: 0, page, perPage }; + } else { + ordersPage = await fetchOrdersFilteredByIds(pb, matchingIds, { + page, + perPage, sort, expand, + fields, }); + } + } else if (search && !tabGroup) { + ordersPage = await pb.collection(ORDER_COLLECTION).getList( + 1, + ORDER_SEARCH_HARD_CAP, + { sort, expand, fields }, + ); + // search returns a single page; totalPages/totalItems are derived + // from the filtered list size once we know how many survived the + // search predicate (set further down). + ordersPage = { ...ordersPage, totalItems: ordersPage.items.length, totalPages: 1 }; + } else if (search && tabGroup) { + // Both filters active. Fetch the cap, then apply both predicates. + const rawPage = await pb.collection(ORDER_COLLECTION).getList( + 1, + ORDER_SEARCH_HARD_CAP, + { sort, expand, fields }, + ); + ordersPage = { + items: rawPage.items, + totalItems: rawPage.items.length, + totalPages: 1, + page: 1, + perPage: ORDER_SEARCH_HARD_CAP, + }; + } else { + // (a) no tab, no search — original path. + ordersPage = await pb.collection(ORDER_COLLECTION).getList( + page, + perPage, + { sort, expand, fields }, + ); + } const orderIds = ordersPage.items.map((order) => order.id); const [allItems, users] = await Promise.all([ fetchOrderItemsForOrders(pb, orderIds), - pb.collection("users").getFullList({ - fields: "id,name,email", - sort: "name", - }), + // Targeted rep lookup: only the user ids the loaded orders actually + // reference. Replaces a `users.getFullList` (which pulled every + // user every page load) with a small `id = ...` filter — typically + // a handful of ids, vs every user in PB. + fetchRepUsersForOrders(pb, ordersPage.items), ]); const usersById = new Map(users.map((user) => [user.id, user])); @@ -135,6 +304,16 @@ export async function listOrdersWithItems( continue; } + // Tab filter (case c, search+tab). The server-side tab path (b) + // already filtered orders by tab, so this only runs when search is + // also active. + if (search && tabGroup) { + const inTab = orderItems.some((it) => + TAB_GROUP_ITEM_STATUSES[tabGroup].includes(it.item_status as ItemStatus), + ); + if (!inTab) continue; + } + const tabGroups = new Set(); for (const item of orderItems) { tabGroups.add(itemStatusToTabGroup(item.item_status)); @@ -142,36 +321,196 @@ export async function listOrdersWithItems( items.push({ order, items: orderItems, tabGroups: [...tabGroups], repName }); } + // For the search path, items.length is the post-filter count and + // pagination collapses to 1 page (search caps at 500 orders and we + // filter client-side). For the tab-only path, PB already paginated + // the matching set, so ordersPage.totalItems/totalPages are the + // truth. For the plain PB-paginated path (no tab, no search), PB's + // own metadata is also authoritative. + const isSearchOnly = search && !tabGroup; + const isSearchAndTab = search && tabGroup; + const finalTotalItems = + isSearchOnly || isSearchAndTab ? items.length : ordersPage.totalItems; + const finalTotalPages = + isSearchOnly || isSearchAndTab ? 1 : ordersPage.totalPages; + return { items, - totalPages: search ? 1 : ordersPage.totalPages, - totalItems: search ? items.length : ordersPage.totalItems, + totalPages: finalTotalPages, + totalItems: finalTotalItems, }; } +/** + * Returns the distinct order ids that have at least one line item in the + * given tab group. Used by the server-side tab filter to build the + * `id = "..." || ...` predicate for the orders query. + */ +export async function getOrderIdsForTabGroup( + pb: AdminPocketBase, + tabGroup: TabGroup, +): Promise { + const items = await pb + .collection(getOrderItemsCollectionName()) + .getFullList<{ order?: string }>({ + filter: tabGroupFilter(tabGroup), + fields: "order", + }); + const ids = new Set(); + for (const item of items) { + if (item.order) ids.add(item.order); + } + return [...ids]; +} + +/** + * Fetch orders whose id is in `ids`, paginated. Two strategies: + * + * - ≤ ID_LIST_INLINE_LIMIT ids → single `id = "..." || ...` filter, + * PB paginates and sorts server-side. Fast and lets PB's own + * metadata report `totalPages` accurately. + * - more ids → `getFullList` (PB returns all matches sorted by `sort`) + * then slice. The `id = "..." || ...` filter for 200+ ids blows past + * PB's URL length cap, so we bypass it. + * + * Both paths apply the same `sort` and `expand`, so the per-row shape + * is identical. + */ +async function fetchOrdersFilteredByIds( + pb: AdminPocketBase, + ids: string[], + opts: { page: number; perPage: number; sort: string; expand: string; fields: string }, +): Promise<{ items: Order[]; totalItems: number; totalPages: number; page: number; perPage: number }> { + const { page, perPage, sort, expand, fields } = opts; + const filter = ids.map((id) => `id = "${id.replace(/"/g, '\\"')}"`).join(" || "); + + if (ids.length <= ID_LIST_INLINE_LIMIT) { + const result = await pb + .collection(ORDER_COLLECTION) + .getList(page, perPage, { filter, sort, expand, fields }); + return { + items: result.items, + totalItems: result.totalItems, + totalPages: result.totalPages, + page: result.page, + perPage: result.perPage, + }; + } + + const all = await pb + .collection(ORDER_COLLECTION) + .getFullList({ filter, sort, expand, fields }); + const totalItems = all.length; + const totalPages = Math.max(1, Math.ceil(totalItems / perPage)); + const start = (page - 1) * perPage; + return { + items: all.slice(start, start + perPage), + totalItems, + totalPages, + page, + perPage, + }; +} + +/** + * PB's `id = "..." || ...` filter length blows past PB's URL/filter + * length cap somewhere above 200 ids (~4 KB). Below this we let PB + * paginate; above it, we `getFullList` and slice client-side. + */ +const ID_LIST_INLINE_LIMIT = 200; + +/** + * Fetches only the rep users referenced by the loaded orders, instead of + * `users.getFullList` (every user in PB every page load). + * + * Coverage: + * - `customer.rep` ids where the expand didn't return a record (PB + * occasionally returns the relation id but not the expanded record + * — e.g. when the user has been soft-deleted) + * - `orders.rep` strings that look like PB record ids (15-char + * alphanumeric). Older orders stored a user id here instead of a + * name; the lookup is the fallback path in `resolveRepDisplayName`. + * + * Returns an empty array when no lookup is needed. + */ +async function fetchRepUsersForOrders( + pb: AdminPocketBase, + orders: Order[], +): Promise { + const needed = new Set(); + for (const order of orders) { + const customer = order.expand?.customer as Customer | undefined; + // customer.rep id without an expanded record — we need to look it up + if (customer?.rep && !customer.expand?.rep) { + needed.add(customer.rep); + } + // Legacy: order.rep sometimes stored a PB user id (15-char alphanumeric) + if (order.rep && /^[a-z0-9]{15}$/i.test(order.rep)) { + needed.add(order.rep); + } + } + if (needed.size === 0) return []; + + const filter = [...needed].map((id) => `id = "${id.replace(/"/g, '\\"')}"`).join(" || "); + return pb.collection("users").getFullList({ + filter, + fields: "id,name,email", + }); +} + export async function getOrderItemStatusTabCounts( pbArg?: AdminPocketBase, + options: { bypassCache?: boolean } = {}, ): Promise> { const pb = pbArg || (await getAdminPocketBase()); pb.autoCancellation(false); - const tabs = Object.keys(TAB_GROUP_ITEM_STATUSES) as TabGroup[]; - const results = await Promise.all( - tabs.map((tab) => - pb.collection(getOrderItemsCollectionName()).getFullList<{ order?: string }>({ - filter: TAB_GROUP_ITEM_STATUSES[tab] - .map((status) => `item_status = "${status}"`) - .join(" || "), - fields: "order", - }), - ), - ); + + if ( + !options.bypassCache && + cachedTabCounts && + cachedTabCounts.expiresAt > Date.now() + ) { + return cachedTabCounts.value; + } + + // Combined query: one PB request, one filter scan, all tab statuses + // at once. Replaces the previous 6 parallel `getFullList` calls + // (one per tab), each of which did an independent filter scan over + // `order_items`. With a non-indexed `item_status` field on a large + // collection (50k+ rows is plausible), the 6-scan path dominates + // the page-load budget; collapsing to 1 scan is the single biggest + // perf win on the orders page after the earlier expand + users + // changes. Bucket in memory — the dataset is small (only `item_status` + // and `order` fields, both short strings). + const filter = TAB_STATUSES.map((s) => `item_status = "${s}"`).join(" || "); + const allItems = await pb + .collection(getOrderItemsCollectionName()) + .getFullList<{ item_status: string; order?: string }>({ + filter, + fields: "item_status,order", + batch: TAB_COUNTS_BATCH_SIZE, + }); const counts = {} as Record; - tabs.forEach((tab, index) => { - counts[tab] = new Set( - results[index].map((item) => item.order).filter(Boolean), - ).size; - }); + const seen: Record> = {} as Record>; + for (const tab of TAB_GROUP_ORDER) { + counts[tab] = 0; + seen[tab] = new Set(); + } + + for (const item of allItems) { + const orderId = item.order; + if (!orderId) continue; + const tab = itemStatusToTabGroup(item.item_status as ItemStatus); + if (seen[tab].has(orderId)) continue; + seen[tab].add(orderId); + counts[tab]++; + } + + cachedTabCounts = { + value: counts, + expiresAt: Date.now() + TAB_COUNTS_CACHE_TTL_MS, + }; return counts; } @@ -243,6 +582,25 @@ async function fetchOrderItemsForOrders( ): Promise { if (orderIds.length === 0) return []; + // Limit the wire payload to just what the list view + search + // predicate read. The `order_items` schema has many unbounded text + // columns (description 5k, notes 5k, problem_notes 5k, ai_item_insights + // 64k json, etc.) that the list view never displays but would all + // be pulled over the wire without this. `supplier` was an expand + // here historically but the list view doesn't render the supplier + // per line (that's the order-level PO concern, not the per-line + // card), so we drop it. + const ITEM_FIELDS = [ + "id", + "order", + "description", + "po_description", + "quantity", + "selling_price", + "cost_price", + "item_status", + ].join(","); + const chunks: string[][] = []; for (let index = 0; index < orderIds.length; index += ORDER_ITEM_FILTER_CHUNK) { chunks.push(orderIds.slice(index, index + ORDER_ITEM_FILTER_CHUNK)); @@ -254,7 +612,7 @@ async function fetchOrderItemsForOrders( filter: ids .map((id) => `order = "${id.replace(/"/g, '\\"')}"`) .join(" || "), - expand: "supplier", + fields: ITEM_FIELDS, sort: "created", }), ),