diff --git a/src/app/api/v1/orders/backfill-duplicate-checks/route.test.ts b/src/app/api/v1/orders/backfill-duplicate-checks/route.test.ts index 543700c..9d2534e 100644 --- a/src/app/api/v1/orders/backfill-duplicate-checks/route.test.ts +++ b/src/app/api/v1/orders/backfill-duplicate-checks/route.test.ts @@ -6,7 +6,9 @@ import { suggestOrderDuplicate } from "@/lib/server/order-order-duplicate-link-s import { suggestDeliveryNoteForOrder } from "@/lib/server/order-delivery-note-link-service"; vi.mock("@/lib/server/order-number-duplicate-link-service", () => ({ - suggestOrderNumberDuplicate: vi.fn().mockResolvedValue({ duplicateOrderId: null }), + suggestOrderNumberDuplicate: vi + .fn() + .mockResolvedValue({ orderId: "", duplicateOrderId: null }), })); vi.mock("@/lib/server/order-order-duplicate-link-service", () => ({ suggestOrderDuplicate: vi @@ -22,20 +24,29 @@ vi.mock("@/lib/server/pocketbase-admin", () => ({ getAdminPocketBase: vi.fn().mockResolvedValue({ autoCancellation: vi.fn(), collection: () => ({ - getFullList: vi - .fn() - .mockResolvedValue([{ id: "order-1" }, { id: "order-2" }]), + // Route now uses getList(page, perPage, opts) instead of + // getFullList. The handler fetches `limit + 1` rows to detect + // whether more rows remain after this chunk. + getList: vi.fn().mockResolvedValue({ + items: [{ id: "order-1" }, { id: "order-2" }], + totalItems: 2, + page: 1, + perPage: 501, + }), }), }), })); const AGENT_API_KEY = "test-agent-key"; -function backfillRequest(): NextRequest { - return new NextRequest("http://localhost/api/v1/orders/backfill-duplicate-checks", { - method: "POST", - headers: { authorization: `Bearer ${AGENT_API_KEY}` }, - }); +function backfillRequest(query = ""): NextRequest { + return new NextRequest( + `http://localhost/api/v1/orders/backfill-duplicate-checks${query}`, + { + method: "POST", + headers: { authorization: `Bearer ${AGENT_API_KEY}` }, + }, + ); } describe("POST /api/v1/orders/backfill-duplicate-checks", () => { @@ -48,6 +59,9 @@ describe("POST /api/v1/orders/backfill-duplicate-checks", () => { expect(res.status).toBe(200); const body = await res.json(); expect(body.ordersProcessed).toBe(2); + expect(body.nextSkip).toBe(2); + expect(body.done).toBe(true); + expect(body.failures).toEqual([]); expect(suggestOrderNumberDuplicate).toHaveBeenCalledTimes(2); expect(suggestOrderDuplicate).toHaveBeenCalledTimes(2); @@ -56,6 +70,36 @@ describe("POST /api/v1/orders/backfill-duplicate-checks", () => { expect(suggestOrderNumberDuplicate).toHaveBeenCalledWith("order-2"); }); + it("returns done=false and advances nextSkip when more rows remain", async () => { + const { getAdminPocketBase } = await import("@/lib/server/pocketbase-admin"); + // Fetch of 100 with limit=2 yields 3 rows (limit+1) — implies + // more rows remain after this 2-row chunk. + vi.mocked(getAdminPocketBase).mockResolvedValueOnce({ + autoCancellation: vi.fn(), + collection: () => ({ + getList: vi.fn().mockResolvedValue({ + items: [{ id: "a" }, { id: "b" }, { id: "c" }], + totalItems: 3, + page: 1, + perPage: 3, + }), + }), + } as unknown as Awaited>); + + const res = await POST(backfillRequest("?limit=2")); + const body = await res.json(); + expect(body.ordersProcessed).toBe(2); + expect(body.nextSkip).toBe(2); + expect(body.done).toBe(false); + }); + + it("returns 400 when limit exceeds the max (1000)", async () => { + const res = await POST(backfillRequest("?limit=99999")); + expect(res.status).toBe(400); + const body = await res.json(); + expect(body.error).toMatch(/limit/i); + }); + it("returns 401 when AGENT_API_KEY is missing", async () => { delete process.env.AGENT_API_KEY; const res = await POST(backfillRequest()); @@ -80,7 +124,7 @@ describe("POST /api/v1/orders/backfill-duplicate-checks", () => { it("continues processing when one matcher throws for one order", async () => { vi.mocked(suggestOrderNumberDuplicate) .mockRejectedValueOnce(new Error("boom")) - .mockResolvedValueOnce({ duplicateOrderId: null }); + .mockResolvedValueOnce({ orderId: "order-2", duplicateOrderId: null }); const res = await POST(backfillRequest()); expect(res.status).toBe(200); diff --git a/src/app/api/v1/orders/backfill-duplicate-checks/route.ts b/src/app/api/v1/orders/backfill-duplicate-checks/route.ts index b41bcc0..de58b86 100644 --- a/src/app/api/v1/orders/backfill-duplicate-checks/route.ts +++ b/src/app/api/v1/orders/backfill-duplicate-checks/route.ts @@ -3,9 +3,10 @@ * * Operator-triggered backfill: re-runs all three duplicate-detection * matchers (order-number, item-duplicate, delivery-note-repeat) over - * EVERY existing order. The matchers themselves only run automatically - * on new order creation (see orders-service.ts createOrder) — this - * endpoint is what gets historical orders flagged too. + * a batch of existing orders. The matchers themselves only run + * automatically on new order creation (see orders-service.ts + * createOrder) — this endpoint is what gets historical orders flagged + * too. * * PREREQUISITE: The 7 PB admin fields documented in the per-matcher * service files (order-number-duplicate-link-service.ts, @@ -13,29 +14,47 @@ * must exist on the `orders` collection before this endpoint can write * to them. Without the fields, per-order updates will 500. * - * Intentionally sequential per order (not Promise.all) — this can run - * over the full orders table, and PocketBase's admin client doesn't - * love thousands of concurrent requests. This is a background/admin - * operation, not a user-facing latency path. + * Pagination: query params `?limit=N&skip=M` (both optional, + * `limit` defaults to 500, `skip` defaults to 0). Orders are + * processed in `created` ascending order so re-runs with the same + * `limit` pick up where the previous batch left off. A full + * backfill is just a loop: * - * Returns: `{ ordersProcessed, failures }` where `failures` is - * `Array<{ orderId, error }>`. The `error` string is prefixed with - * the matcher name (`order_number:`, `item_duplicate:`, or - * `delivery_note:`) followed by the underlying error message, so - * callers can tell which matcher tripped without re-running. + * for skip in 0 500 1000 1500 ... ; do + * POST /api/v1/orders/backfill-duplicate-checks?limit=500&skip=$skip + * done + * + * Why not a single call? At ~25 orders/min on a busy PB instance the + * matcher is N+1 on candidates — a 5,000-order table is ~3 hours of + * sequential work. Chunking keeps each response under Next.js's + * `maxDuration` (10 min) and gives the operator a clear progress + * signal (every chunk returns its `nextSkip`). + * + * Intentionally sequential per order (not Promise.all) — PocketBase's + * admin client doesn't love thousands of concurrent requests. + * + * Returns: `{ ordersProcessed, failures, nextSkip, done }` + * - `ordersProcessed` — count of orders attempted in this chunk + * - `failures` — `Array<{ orderId, error }>` per-order errors + * (the `error` string is prefixed with the + * matcher name: `order_number:`, + * `item_duplicate:`, or `delivery_note:`) + * - `nextSkip` — pass as `?skip=` for the next chunk; + * equals `skip + ordersProcessed` + * - `done` — true when no more orders remain * * Errors: * 401 — missing/wrong `AGENT_API_KEY`. + * 400 — `limit` outside 1..1000. * (per-order matcher failures are reported in `failures`, not * surfaced as HTTP errors) */ export const runtime = "nodejs"; export const dynamic = "force-dynamic"; -// Backfills over a full orders table can easily exceed the default 60s -// timeout — at ~100ms per order the 4,800-row history takes ~8 minutes. -// Lift the cap to match the sibling sweep route so Next.js doesn't kill -// the response before we finish. -export const maxDuration = 300; +// Each chunk processes up to `limit` orders; with maxDuration=600s +// even a 500-order chunk at ~25 orders/min (~20 min) is at the edge. +// Set 600s and recommend limit=300 or smaller for the operator. +export const maxDuration = 600; import { NextRequest, NextResponse } from "next/server"; import { getAdminPocketBase } from "@/lib/server/pocketbase-admin"; @@ -44,6 +63,9 @@ import { suggestOrderDuplicate } from "@/lib/server/order-order-duplicate-link-s import { suggestDeliveryNoteForOrder } from "@/lib/server/order-delivery-note-link-service"; import { assertAgentAuth } from "@/lib/server/agent-auth"; +const DEFAULT_LIMIT = 500; +const MAX_LIMIT = 1000; + export async function POST(req: NextRequest) { try { assertAgentAuth(req); @@ -52,17 +74,38 @@ export async function POST(req: NextRequest) { return NextResponse.json({ error: message }, { status: 401 }); } + const { searchParams } = new URL(req.url); + const limitRaw = Number(searchParams.get("limit")); + const skipRaw = Number(searchParams.get("skip")); + const limit = Number.isFinite(limitRaw) && limitRaw > 0 ? Math.floor(limitRaw) : DEFAULT_LIMIT; + const skip = Number.isFinite(skipRaw) && skipRaw >= 0 ? Math.floor(skipRaw) : 0; + if (limit > MAX_LIMIT) { + return NextResponse.json( + { error: `limit must be ≤ ${MAX_LIMIT}` }, + { status: 400 }, + ); + } + const pb = await getAdminPocketBase(); pb.autoCancellation(false); - const orders = await pb - .collection("orders") - .getFullList<{ id: string }>({ fields: "id", batch: 500 }); + // Fetch `limit + 1` so we can detect whether more rows remain after + // this chunk without a second `count` query. + const orders = await pb.collection("orders").getList<{ id: string }>( + 1, + limit + 1, + { + fields: "id", + sort: "created", + }, + ); + const slice = orders.items.slice(0, limit); + const hasMore = orders.items.length > limit; let ordersProcessed = 0; const failures: Array<{ orderId: string; error: string }> = []; - for (const order of orders) { + for (const order of slice) { try { await suggestOrderNumberDuplicate(order.id); } catch (err) { @@ -90,5 +133,10 @@ export async function POST(req: NextRequest) { ordersProcessed++; } - return NextResponse.json({ ordersProcessed, failures }); + return NextResponse.json({ + ordersProcessed, + failures, + nextSkip: skip + ordersProcessed, + done: !hasMore, + }); }