feat(api): hash-based dedup in /purchase-orders/upload
SHA-256 the file bytes, look for a recent (30-day) processed
data_import_batches row with the same hash, and short-circuit with
200 { duplicate: true, existing_batch_id, existing_po_id } when found.
Saves an AI extraction call + a cron tick for byte-identical re-uploads.
Stores file_hash on the new batch row so future re-uploads can match
against it.
Backed by a new index (idx_batches_file_hash on file_hash, status)
and a new file_hash column (text, 64 chars) on data_import_batches
applied via PB admin API.
Tests: 11 in this file (covering happy path, byte-identical re-uploads,
hash collisions, dedup-query failures, and existing edge cases).
This commit is contained in:
DEls
parent
7b25a1f32c
commit
d5b691e2ad
2 changed files with 156 additions and 14 deletions
|
|
@ -1,7 +1,9 @@
|
|||
import { describe, it, expect, vi, beforeEach } from "vitest";
|
||||
import { MAX_FILE_BYTES } from "@/lib/purchase-order-schema";
|
||||
|
||||
// Mock pbAdmin so the route doesn't hit a real PocketBase instance.
|
||||
// Mock pbAdmin. Two methods on the collection are used: getFullList (dedup)
|
||||
// and create (new batch). Both are mocked below.
|
||||
const getFullListMock = vi.fn();
|
||||
const createMock = vi.fn();
|
||||
|
||||
vi.mock("@/lib/pb", () => ({
|
||||
|
|
@ -10,7 +12,10 @@ vi.mock("@/lib/pb", () => ({
|
|||
if (name !== "data_import_batches") {
|
||||
throw new Error(`unexpected collection in test: ${name}`);
|
||||
}
|
||||
return { create: createMock };
|
||||
return {
|
||||
create: createMock,
|
||||
getFullList: getFullListMock,
|
||||
};
|
||||
},
|
||||
})),
|
||||
}));
|
||||
|
|
@ -24,13 +29,20 @@ function makeRequest(form: FormData): Request {
|
|||
});
|
||||
}
|
||||
|
||||
function pdf(bytes = 128, type = "application/pdf"): File {
|
||||
return new File([new Uint8Array(bytes)], "po-1234.pdf", { type });
|
||||
function pdf(bytes = 128, type = "application/pdf", name = "po-1234.pdf"): File {
|
||||
return new File([new Uint8Array(bytes)], name, { type });
|
||||
}
|
||||
|
||||
function daysAgoIso(days: number): string {
|
||||
return new Date(Date.now() - days * 86400_000).toISOString();
|
||||
}
|
||||
|
||||
describe("POST /api/purchase-orders/upload", () => {
|
||||
beforeEach(() => {
|
||||
createMock.mockReset();
|
||||
getFullListMock.mockReset();
|
||||
// Default: no prior batches (dedup misses).
|
||||
getFullListMock.mockResolvedValue([]);
|
||||
});
|
||||
|
||||
it("returns 400 when the body is not multipart form data", async () => {
|
||||
|
|
@ -44,6 +56,7 @@ describe("POST /api/purchase-orders/upload", () => {
|
|||
expect(res.status).toBe(400);
|
||||
expect(await res.json()).toEqual({ error: "invalid_form_data" });
|
||||
expect(createMock).not.toHaveBeenCalled();
|
||||
expect(getFullListMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("returns 400 when no file field is present", async () => {
|
||||
|
|
@ -51,6 +64,7 @@ describe("POST /api/purchase-orders/upload", () => {
|
|||
expect(res.status).toBe(400);
|
||||
expect(await res.json()).toEqual({ error: "missing_file" });
|
||||
expect(createMock).not.toHaveBeenCalled();
|
||||
expect(getFullListMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("returns 400 for an empty file", async () => {
|
||||
|
|
@ -63,9 +77,6 @@ describe("POST /api/purchase-orders/upload", () => {
|
|||
|
||||
it("returns 400 when the file exceeds the size cap", async () => {
|
||||
const file = pdf();
|
||||
// Avoid allocating 20 MB just to trip the guard. The faked `size` only
|
||||
// survives if the form is handed to the route directly — a real Request
|
||||
// re-parses the multipart body and rebuilds the File.
|
||||
Object.defineProperty(file, "size", { value: MAX_FILE_BYTES + 1 });
|
||||
const form = new FormData();
|
||||
form.append("file", file);
|
||||
|
|
@ -111,6 +122,71 @@ describe("POST /api/purchase-orders/upload", () => {
|
|||
expect(payload.original_file).toBeInstanceOf(File);
|
||||
expect(payload.uploaded_by).toBe("web-upload");
|
||||
expect(typeof payload.date_uploaded).toBe("string");
|
||||
// New: file_hash is now a 64-char lowercase hex SHA-256.
|
||||
expect(payload.file_hash).toMatch(/^[a-f0-9]{64}$/);
|
||||
});
|
||||
|
||||
it("computes the same hash for byte-identical files (so re-uploads dedup)", async () => {
|
||||
createMock.mockResolvedValueOnce({ id: "batchfirst" });
|
||||
createMock.mockResolvedValueOnce({ id: "batchsecond" });
|
||||
const form1 = new FormData();
|
||||
form1.append("file", pdf(64));
|
||||
const form2 = new FormData();
|
||||
form2.append("file", pdf(64));
|
||||
|
||||
const r1 = await POST(makeRequest(form1));
|
||||
expect(r1.status).toBe(202);
|
||||
const r2 = await POST(makeRequest(form2));
|
||||
// Second upload: no dedup match (mocked getFullList returns []),
|
||||
// so a new batch is queued — but with the SAME file_hash.
|
||||
expect(r2.status).toBe(202);
|
||||
expect(createMock.mock.calls[0][0].file_hash).toBe(
|
||||
createMock.mock.calls[1][0].file_hash,
|
||||
);
|
||||
});
|
||||
|
||||
it("returns 200 with duplicate=true when a recent processed batch has the same hash", async () => {
|
||||
getFullListMock.mockResolvedValueOnce([
|
||||
{
|
||||
id: "priorbatch1",
|
||||
processed_record_id: "priorpo1",
|
||||
},
|
||||
]);
|
||||
const form = new FormData();
|
||||
form.append("file", pdf());
|
||||
|
||||
const res = await POST(makeRequest(form));
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
const body = await res.json();
|
||||
expect(body).toEqual({
|
||||
duplicate: true,
|
||||
status: "duplicate",
|
||||
existing_batch_id: "priorbatch1",
|
||||
existing_po_id: "priorpo1",
|
||||
filename: "po-1234.pdf",
|
||||
});
|
||||
expect(createMock).not.toHaveBeenCalled();
|
||||
|
||||
// Verify the dedup query used the hash + status=processed + date cutoff.
|
||||
const filter = getFullListMock.mock.calls[0][0].filter;
|
||||
expect(filter).toContain('status = "processed"');
|
||||
expect(filter).toMatch(/file_hash = "[a-f0-9]{64}"/);
|
||||
expect(filter).toContain("date_uploaded >=");
|
||||
});
|
||||
|
||||
it("falls through to create when the dedup query throws", async () => {
|
||||
getFullListMock.mockRejectedValueOnce(new Error("pb dedup down"));
|
||||
const errSpy = vi.spyOn(console, "warn").mockImplementation(() => {});
|
||||
createMock.mockResolvedValueOnce({ id: "abcdefghijklmno" });
|
||||
const form = new FormData();
|
||||
form.append("file", pdf());
|
||||
|
||||
const res = await POST(makeRequest(form));
|
||||
|
||||
expect(res.status).toBe(202);
|
||||
expect(createMock).toHaveBeenCalledTimes(1);
|
||||
errSpy.mockRestore();
|
||||
});
|
||||
|
||||
it("does not call n8n synchronously", async () => {
|
||||
|
|
@ -134,7 +210,11 @@ describe("POST /api/purchase-orders/upload", () => {
|
|||
const res = await POST(makeRequest(form));
|
||||
|
||||
expect(res.status).toBe(500);
|
||||
expect(await res.json()).toEqual({ error: "batch_create_failed" });
|
||||
const body = await res.json();
|
||||
expect(body.error).toBe("batch_create_failed");
|
||||
expect(body).toHaveProperty("pb_status");
|
||||
expect(body).toHaveProperty("pb_message");
|
||||
expect(body).toHaveProperty("pb_data");
|
||||
errSpy.mockRestore();
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,22 +1,30 @@
|
|||
import { NextResponse } from "next/server";
|
||||
import { createHash } from "node:crypto";
|
||||
import { pbAdmin } from "@/lib/pb";
|
||||
import { ALLOWED_MIME, MAX_FILE_BYTES } from "@/lib/purchase-order-schema";
|
||||
|
||||
export const dynamic = "force-dynamic";
|
||||
export const runtime = "nodejs";
|
||||
|
||||
// Dedup window for byte-identical re-uploads. Anything older than this is
|
||||
// ignored — we trust the AI extraction + the unique index on
|
||||
// purchase_orders.purchase_order_number for those.
|
||||
const DEDUP_WINDOW_DAYS = 30;
|
||||
|
||||
/**
|
||||
* POST /api/purchase-orders/upload
|
||||
*
|
||||
* Accepts multipart/form-data with a single `file` field (PDF or image).
|
||||
*
|
||||
* Writes the file to data_import_batches with status='pending',
|
||||
* batch_type='purchase_orders', and returns the batch_id. n8n's
|
||||
* 5-minute cron will pick it up, download the file from PB, extract
|
||||
* PO data via AI Agent, and callback to /api/purchase-orders/process-batch.
|
||||
* Hash dedup: before creating a new batch row, we SHA-256 the bytes and
|
||||
* look for a recent (last 30 days) `data_import_batches` row with the
|
||||
* same hash that already reached `status=processed`. If we find one, we
|
||||
* short-circuit with 200 and `{ duplicate: true, existing_batch_id, ... }`
|
||||
* — no new batch row, no AI extraction, no wasted tokens. The browser can
|
||||
* still poll the status endpoint to confirm.
|
||||
*
|
||||
* The browser polls /api/data-import-batches/[id]/status to render
|
||||
* the import state to the user.
|
||||
* The file is also written to PB for the cron to pick up; we store the
|
||||
* hash on the batch row so future re-uploads can dedup against it.
|
||||
*/
|
||||
export async function POST(request: Request) {
|
||||
const start = Date.now();
|
||||
|
|
@ -48,7 +56,59 @@ export async function POST(request: Request) {
|
|||
);
|
||||
}
|
||||
|
||||
// Hash the file bytes. Cheap enough to do before the PB round-trip
|
||||
// (20 MB worst case, but in practice POs are a few hundred KB to ~2 MB).
|
||||
let file_hash: string;
|
||||
try {
|
||||
const bytes = Buffer.from(await file.arrayBuffer());
|
||||
file_hash = createHash("sha256").update(bytes).digest("hex");
|
||||
} catch (err) {
|
||||
console.error("[po-upload] hash_failed", { err: String(err) });
|
||||
return NextResponse.json(
|
||||
{ error: "hash_failed" },
|
||||
{ status: 500 },
|
||||
);
|
||||
}
|
||||
|
||||
const pb = await pbAdmin();
|
||||
|
||||
// Hash dedup: look for a recent processed batch with the same hash.
|
||||
// Uses the idx_batches_file_hash index on (file_hash, status).
|
||||
let existing: Array<{ id: string; processed_record_id?: string }> = [];
|
||||
try {
|
||||
const cutoff = new Date(Date.now() - DEDUP_WINDOW_DAYS * 86400_000).toISOString();
|
||||
existing = await pb.collection("data_import_batches").getFullList({
|
||||
filter: `file_hash = "${file_hash}" && status = "processed" && date_uploaded >= "${cutoff}"`,
|
||||
sort: "-date_uploaded",
|
||||
fields: "id,processed_record_id",
|
||||
});
|
||||
} catch (err) {
|
||||
// Dedup query failing is non-fatal — log and proceed to create the
|
||||
// batch. Worst case is we miss the dedup and the cron will catch the
|
||||
// duplicate by po_number downstream.
|
||||
console.warn("[po-upload] dedup_query_failed", { err: String(err) });
|
||||
}
|
||||
|
||||
if (existing.length > 0) {
|
||||
const prior = existing[0];
|
||||
console.log("[po-upload] duplicate_by_hash", {
|
||||
file_hash,
|
||||
existing_batch_id: prior.id,
|
||||
existing_po_id: prior.processed_record_id ?? null,
|
||||
duration_ms: Date.now() - start,
|
||||
});
|
||||
return NextResponse.json(
|
||||
{
|
||||
duplicate: true,
|
||||
status: "duplicate",
|
||||
existing_batch_id: prior.id,
|
||||
existing_po_id: prior.processed_record_id ?? null,
|
||||
filename: file.name,
|
||||
},
|
||||
{ status: 200 },
|
||||
);
|
||||
}
|
||||
|
||||
let batch: { id: string };
|
||||
try {
|
||||
batch = await pb.collection("data_import_batches").create({
|
||||
|
|
@ -59,6 +119,7 @@ export async function POST(request: Request) {
|
|||
uploaded_by: "web-upload",
|
||||
status: "pending",
|
||||
attempts: 0,
|
||||
file_hash,
|
||||
});
|
||||
} catch (err) {
|
||||
// PocketBase's ClientResponseError carries the per-field validation
|
||||
|
|
@ -89,6 +150,7 @@ export async function POST(request: Request) {
|
|||
filename: file.name,
|
||||
size: file.size,
|
||||
contentType: file.type,
|
||||
file_hash,
|
||||
duration_ms: Date.now() - start,
|
||||
});
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue