diff --git a/src/app/api/purchase-orders/upload/route.test.ts b/src/app/api/purchase-orders/upload/route.test.ts index b398b80..4218709 100644 --- a/src/app/api/purchase-orders/upload/route.test.ts +++ b/src/app/api/purchase-orders/upload/route.test.ts @@ -1,7 +1,9 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; import { MAX_FILE_BYTES } from "@/lib/purchase-order-schema"; -// Mock pbAdmin so the route doesn't hit a real PocketBase instance. +// Mock pbAdmin. Two methods on the collection are used: getFullList (dedup) +// and create (new batch). Both are mocked below. +const getFullListMock = vi.fn(); const createMock = vi.fn(); vi.mock("@/lib/pb", () => ({ @@ -10,7 +12,10 @@ vi.mock("@/lib/pb", () => ({ if (name !== "data_import_batches") { throw new Error(`unexpected collection in test: ${name}`); } - return { create: createMock }; + return { + create: createMock, + getFullList: getFullListMock, + }; }, })), })); @@ -24,13 +29,20 @@ function makeRequest(form: FormData): Request { }); } -function pdf(bytes = 128, type = "application/pdf"): File { - return new File([new Uint8Array(bytes)], "po-1234.pdf", { type }); +function pdf(bytes = 128, type = "application/pdf", name = "po-1234.pdf"): File { + return new File([new Uint8Array(bytes)], name, { type }); +} + +function daysAgoIso(days: number): string { + return new Date(Date.now() - days * 86400_000).toISOString(); } describe("POST /api/purchase-orders/upload", () => { beforeEach(() => { createMock.mockReset(); + getFullListMock.mockReset(); + // Default: no prior batches (dedup misses). + getFullListMock.mockResolvedValue([]); }); it("returns 400 when the body is not multipart form data", async () => { @@ -44,6 +56,7 @@ describe("POST /api/purchase-orders/upload", () => { expect(res.status).toBe(400); expect(await res.json()).toEqual({ error: "invalid_form_data" }); expect(createMock).not.toHaveBeenCalled(); + expect(getFullListMock).not.toHaveBeenCalled(); }); it("returns 400 when no file field is present", async () => { @@ -51,6 +64,7 @@ describe("POST /api/purchase-orders/upload", () => { expect(res.status).toBe(400); expect(await res.json()).toEqual({ error: "missing_file" }); expect(createMock).not.toHaveBeenCalled(); + expect(getFullListMock).not.toHaveBeenCalled(); }); it("returns 400 for an empty file", async () => { @@ -63,9 +77,6 @@ describe("POST /api/purchase-orders/upload", () => { it("returns 400 when the file exceeds the size cap", async () => { const file = pdf(); - // Avoid allocating 20 MB just to trip the guard. The faked `size` only - // survives if the form is handed to the route directly — a real Request - // re-parses the multipart body and rebuilds the File. Object.defineProperty(file, "size", { value: MAX_FILE_BYTES + 1 }); const form = new FormData(); form.append("file", file); @@ -111,6 +122,71 @@ describe("POST /api/purchase-orders/upload", () => { expect(payload.original_file).toBeInstanceOf(File); expect(payload.uploaded_by).toBe("web-upload"); expect(typeof payload.date_uploaded).toBe("string"); + // New: file_hash is now a 64-char lowercase hex SHA-256. + expect(payload.file_hash).toMatch(/^[a-f0-9]{64}$/); + }); + + it("computes the same hash for byte-identical files (so re-uploads dedup)", async () => { + createMock.mockResolvedValueOnce({ id: "batchfirst" }); + createMock.mockResolvedValueOnce({ id: "batchsecond" }); + const form1 = new FormData(); + form1.append("file", pdf(64)); + const form2 = new FormData(); + form2.append("file", pdf(64)); + + const r1 = await POST(makeRequest(form1)); + expect(r1.status).toBe(202); + const r2 = await POST(makeRequest(form2)); + // Second upload: no dedup match (mocked getFullList returns []), + // so a new batch is queued — but with the SAME file_hash. + expect(r2.status).toBe(202); + expect(createMock.mock.calls[0][0].file_hash).toBe( + createMock.mock.calls[1][0].file_hash, + ); + }); + + it("returns 200 with duplicate=true when a recent processed batch has the same hash", async () => { + getFullListMock.mockResolvedValueOnce([ + { + id: "priorbatch1", + processed_record_id: "priorpo1", + }, + ]); + const form = new FormData(); + form.append("file", pdf()); + + const res = await POST(makeRequest(form)); + + expect(res.status).toBe(200); + const body = await res.json(); + expect(body).toEqual({ + duplicate: true, + status: "duplicate", + existing_batch_id: "priorbatch1", + existing_po_id: "priorpo1", + filename: "po-1234.pdf", + }); + expect(createMock).not.toHaveBeenCalled(); + + // Verify the dedup query used the hash + status=processed + date cutoff. + const filter = getFullListMock.mock.calls[0][0].filter; + expect(filter).toContain('status = "processed"'); + expect(filter).toMatch(/file_hash = "[a-f0-9]{64}"/); + expect(filter).toContain("date_uploaded >="); + }); + + it("falls through to create when the dedup query throws", async () => { + getFullListMock.mockRejectedValueOnce(new Error("pb dedup down")); + const errSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + createMock.mockResolvedValueOnce({ id: "abcdefghijklmno" }); + const form = new FormData(); + form.append("file", pdf()); + + const res = await POST(makeRequest(form)); + + expect(res.status).toBe(202); + expect(createMock).toHaveBeenCalledTimes(1); + errSpy.mockRestore(); }); it("does not call n8n synchronously", async () => { @@ -134,7 +210,11 @@ describe("POST /api/purchase-orders/upload", () => { const res = await POST(makeRequest(form)); expect(res.status).toBe(500); - expect(await res.json()).toEqual({ error: "batch_create_failed" }); + const body = await res.json(); + expect(body.error).toBe("batch_create_failed"); + expect(body).toHaveProperty("pb_status"); + expect(body).toHaveProperty("pb_message"); + expect(body).toHaveProperty("pb_data"); errSpy.mockRestore(); }); }); diff --git a/src/app/api/purchase-orders/upload/route.ts b/src/app/api/purchase-orders/upload/route.ts index ae7fb86..9cd272e 100644 --- a/src/app/api/purchase-orders/upload/route.ts +++ b/src/app/api/purchase-orders/upload/route.ts @@ -1,22 +1,30 @@ import { NextResponse } from "next/server"; +import { createHash } from "node:crypto"; import { pbAdmin } from "@/lib/pb"; import { ALLOWED_MIME, MAX_FILE_BYTES } from "@/lib/purchase-order-schema"; export const dynamic = "force-dynamic"; export const runtime = "nodejs"; +// Dedup window for byte-identical re-uploads. Anything older than this is +// ignored — we trust the AI extraction + the unique index on +// purchase_orders.purchase_order_number for those. +const DEDUP_WINDOW_DAYS = 30; + /** * POST /api/purchase-orders/upload * * Accepts multipart/form-data with a single `file` field (PDF or image). * - * Writes the file to data_import_batches with status='pending', - * batch_type='purchase_orders', and returns the batch_id. n8n's - * 5-minute cron will pick it up, download the file from PB, extract - * PO data via AI Agent, and callback to /api/purchase-orders/process-batch. + * Hash dedup: before creating a new batch row, we SHA-256 the bytes and + * look for a recent (last 30 days) `data_import_batches` row with the + * same hash that already reached `status=processed`. If we find one, we + * short-circuit with 200 and `{ duplicate: true, existing_batch_id, ... }` + * — no new batch row, no AI extraction, no wasted tokens. The browser can + * still poll the status endpoint to confirm. * - * The browser polls /api/data-import-batches/[id]/status to render - * the import state to the user. + * The file is also written to PB for the cron to pick up; we store the + * hash on the batch row so future re-uploads can dedup against it. */ export async function POST(request: Request) { const start = Date.now(); @@ -48,7 +56,59 @@ export async function POST(request: Request) { ); } + // Hash the file bytes. Cheap enough to do before the PB round-trip + // (20 MB worst case, but in practice POs are a few hundred KB to ~2 MB). + let file_hash: string; + try { + const bytes = Buffer.from(await file.arrayBuffer()); + file_hash = createHash("sha256").update(bytes).digest("hex"); + } catch (err) { + console.error("[po-upload] hash_failed", { err: String(err) }); + return NextResponse.json( + { error: "hash_failed" }, + { status: 500 }, + ); + } + const pb = await pbAdmin(); + + // Hash dedup: look for a recent processed batch with the same hash. + // Uses the idx_batches_file_hash index on (file_hash, status). + let existing: Array<{ id: string; processed_record_id?: string }> = []; + try { + const cutoff = new Date(Date.now() - DEDUP_WINDOW_DAYS * 86400_000).toISOString(); + existing = await pb.collection("data_import_batches").getFullList({ + filter: `file_hash = "${file_hash}" && status = "processed" && date_uploaded >= "${cutoff}"`, + sort: "-date_uploaded", + fields: "id,processed_record_id", + }); + } catch (err) { + // Dedup query failing is non-fatal — log and proceed to create the + // batch. Worst case is we miss the dedup and the cron will catch the + // duplicate by po_number downstream. + console.warn("[po-upload] dedup_query_failed", { err: String(err) }); + } + + if (existing.length > 0) { + const prior = existing[0]; + console.log("[po-upload] duplicate_by_hash", { + file_hash, + existing_batch_id: prior.id, + existing_po_id: prior.processed_record_id ?? null, + duration_ms: Date.now() - start, + }); + return NextResponse.json( + { + duplicate: true, + status: "duplicate", + existing_batch_id: prior.id, + existing_po_id: prior.processed_record_id ?? null, + filename: file.name, + }, + { status: 200 }, + ); + } + let batch: { id: string }; try { batch = await pb.collection("data_import_batches").create({ @@ -59,6 +119,7 @@ export async function POST(request: Request) { uploaded_by: "web-upload", status: "pending", attempts: 0, + file_hash, }); } catch (err) { // PocketBase's ClientResponseError carries the per-field validation @@ -89,6 +150,7 @@ export async function POST(request: Request) { filename: file.name, size: file.size, contentType: file.type, + file_hash, duration_ms: Date.now() - start, });