// Run the full backfill in 200-order chunks. Logs progress to stdout. import { readFileSync } from "node:fs"; const envLocal = readFileSync(".env.local", "utf8"); for (const line of envLocal.split("\n")) { const m = line.match(/^([A-Z0-9_]+)\s*=\s*(.+?)\s*$/); if (m && !process.env[m[1]]) process.env[m[1]] = m[2].replace(/^["']|["']$/g, ""); } const baseUrl = "http://localhost:4001"; const agentKey = process.env.AGENT_API_KEY; if (!agentKey) { console.error("Missing AGENT_API_KEY in .env.local"); process.exit(1); } const LIMIT = 200; let skip = 0; let totalProcessed = 0; let totalFailures = 0; const start = Date.now(); let chunkNum = 0; while (true) { chunkNum++; const url = `${baseUrl}/api/v1/orders/backfill-duplicate-checks?limit=${LIMIT}&skip=${skip}`; let body; try { const controller = new AbortController(); const timeoutId = setTimeout(() => controller.abort(), 600_000); // 10 min per chunk const res = await fetch(url, { method: "POST", headers: { authorization: `Bearer ${agentKey}` }, signal: controller.signal, }); clearTimeout(timeoutId); if (!res.ok) { console.error(`[fail] HTTP ${res.status} at skip=${skip}`); const text = await res.text(); console.error(text.substring(0, 500)); process.exit(2); } body = await res.json(); } catch (err) { console.error(`[err] fetch failed at skip=${skip}:`, err.message); if (err.cause) console.error("cause:", err.cause); process.exit(2); } totalProcessed += body.ordersProcessed || 0; totalFailures += body.failures?.length || 0; const elapsed = Math.round((Date.now() - start) / 1000); const rate = totalProcessed / (elapsed / 60); console.log( `[chunk ${chunkNum} skip=${skip} processed=${body.ordersProcessed} ` + `failures=${body.failures?.length || 0} total=${totalProcessed} ` + `nextSkip=${body.nextSkip} done=${body.done} elapsed=${elapsed}s rate=${rate.toFixed(1)}/min]`, ); if (body.failures?.length > 0) { for (const f of body.failures.slice(0, 3)) { console.log(` failure: ${f.orderId} -> ${f.error}`); } if (body.failures.length > 3) { console.log(` ... and ${body.failures.length - 3} more`); } } if (body.done) { console.log( `\n[done] total processed: ${totalProcessed}, total failures: ${totalFailures}, elapsed: ${elapsed}s (${(elapsed / 60).toFixed(1)} min)`, ); process.exit(0); } skip = body.nextSkip; }