From 5caa923375e1df45aa5c30d05b3734edd82d96d8 Mon Sep 17 00:00:00 2001 From: David Date: Fri, 2 Oct 2026 14:30:17 +0200 Subject: [PATCH] Invoices: the app ingests PDFs itself, with a problem-invoice inbox MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extraction was previously manual: an agent read each PDF, mapped the columns by hand and wrote data/source/invoice-*.json. That was the only way anything got in, and the upload button just filed the document without reading it. That supplier's report is one fixed template — the columns sit at identical x-positions on every invoice, and only the line count and page count vary — so the figures are now read by coordinate instead. Deliberately not an AI extractor: this input does not need one, and a deterministic parse that fails loudly is worth more here than a flexible one that can hallucinate a total. lib/invoice-parser.ts reads the header, the machine lines across all pages, the "Plant Hire Total" row and the SUB TOTAL / Escalation / PLUS VAT / INVOICE TOTAL block, then applies the same arithmetic gates the seed does: lines must sum to the subtotal, each line must satisfy qty x rate = amount and amount + VAT = total, and VAT must match the stated rate. Anything that does not reconcile is refused with the reason — never partially imported. Validated by parsing all five existing invoices and comparing every field against the hand-extracted JSON: identical, including the R24,205.13-VAT column that a plain text read gets wrong. PHI 169 was then deleted and re-ingested through the HTTP endpoint alone and came back field-for-field identical, with no agent involved. - "Add Invoice" drops a PDF in and imports it; lines are matched to plant items by the code the invoice prints, and an unmatched line (the container codes MPLANT04/MPLANT05) is left unassigned rather than guessed. - A "Problem invoices" inbox lists every PDF that failed, with the reason, and Retry re-reads the stored document and re-runs the parse. Retry re-reports rather than claiming success, and only files imported data. - invoice_intake collection records each attempt. pdfjs-dist is in serverExternalPackages: bundled, it rewrites its own lazy worker import to a .next path absent from the standalone image and every parse dies with "Setting up fake worker failed". The parser also copies the buffer before handing it over, since pdfjs transfers and detaches it. Co-Authored-By: Claude Opus 4.8 (1M context) --- app/api/invoices/intake/[id]/retry/route.ts | 71 ++++ app/api/invoices/intake/[id]/route.ts | 30 ++ app/api/invoices/intake/route.ts | 229 +++++++++++ app/income/invoices/page.tsx | 4 + components/InvoiceIntake.tsx | 308 ++++++++++++++ lib/invoice-parser.test.ts | 129 ++++++ lib/invoice-parser.ts | 423 ++++++++++++++++++++ next.config.ts | 7 + package-lock.json | 278 +++++++++++++ package.json | 1 + scripts/setup-invoice-collections.mjs | 20 + 11 files changed, 1500 insertions(+) create mode 100644 app/api/invoices/intake/[id]/retry/route.ts create mode 100644 app/api/invoices/intake/[id]/route.ts create mode 100644 app/api/invoices/intake/route.ts create mode 100644 components/InvoiceIntake.tsx create mode 100644 lib/invoice-parser.test.ts create mode 100644 lib/invoice-parser.ts diff --git a/app/api/invoices/intake/[id]/retry/route.ts b/app/api/invoices/intake/[id]/retry/route.ts new file mode 100644 index 0000000..f2800d6 --- /dev/null +++ b/app/api/invoices/intake/[id]/retry/route.ts @@ -0,0 +1,71 @@ +import { revalidatePath } from "next/cache"; +import { serverAuth } from "@/lib/pb"; + +export const dynamic = "force-dynamic"; + +// POST /api/invoices/intake/:id/retry +// Re-attempts the import for a problem invoice. +// +// Retry is only meaningful once the REASON is addressed, so this never silently +// re-parses a broken PDF and calls it fixed: it re-reads the stored document and, if +// the parse still fails, updates the recorded reason instead of pretending it worked. +// Once it succeeds the inbox row is marked imported and linked to the new invoice. +export async function POST( + _req: Request, + { params }: { params: Promise<{ id: string }> }, +) { + const { id } = await params; + const pb = await serverAuth(); + + try { + const record = await pb.collection("invoice_intake").getOne(id); + if (!record.source_pdf) { + return Response.json({ ok: false, error: "The original document is no longer stored" }, { status: 400 }); + } + + // Pull the stored bytes back down. A bare /api/files/ 404s for a + // rule-locked collection even with an admin token — protected files resolve by + // the full /collectionId/recordId/filename path, which the SDK builds for us. + const token = record.source_pdf as string; + const url = pb.files.getURL(record, token); + const res = await fetch(url, { + headers: pb.authStore.token ? { Authorization: pb.authStore.token } : undefined, + }); + if (!res.ok) { + const reason = `Could not read the stored document back from storage (HTTP ${res.status})`; + await pb.collection("invoice_intake").update(id, { status: "failed", error: reason }); + return Response.json({ ok: false, error: reason }, { status: 502 }); + } + + const file = new File([await res.arrayBuffer()], record.file_name || token, { + type: "application/pdf", + }); + const fd = new FormData(); + fd.append("file", file); + + // Re-run the full intake against the same bytes. + const intake = await fetch(new URL("/api/invoices/intake", _req.url), { + method: "POST", + body: fd, + }); + const result = await intake.json().catch(() => ({ ok: false, error: "Unexpected response" })); + + if (result.ok) { + await pb.collection("invoice_intake").delete(id); + revalidatePath("/income/invoices"); + revalidatePath("/income"); + return Response.json({ ok: true, retried: true, ...result }); + } + + // Still failing — keep it in the inbox with the newest reason, and drop the + // duplicate record the retry just created so the inbox stays one row per file. + await pb.collection("invoice_intake").delete(id).catch(() => {}); + revalidatePath("/income/invoices"); + return Response.json( + { ok: false, retried: true, stillFailing: true, error: result.error ?? "Import failed again" }, + { status: 422 }, + ); + } catch (e: any) { + return Response.json({ ok: false, error: e?.message || "Retry failed" }, { status: 500 }); + } +} \ No newline at end of file diff --git a/app/api/invoices/intake/[id]/route.ts b/app/api/invoices/intake/[id]/route.ts new file mode 100644 index 0000000..2a8e276 --- /dev/null +++ b/app/api/invoices/intake/[id]/route.ts @@ -0,0 +1,30 @@ +import { revalidatePath } from "next/cache"; +import { serverAuth } from "@/lib/pb"; + +export const dynamic = "force-dynamic"; + +// DELETE /api/invoices/intake/:id +// Drops a problem invoice from the inbox. Only reachable while it is still `failed` — +// once a file has been imported it belongs to the invoices table and is removed through +// the invoice itself, so this cannot be used to orphan a real invoice. +export async function DELETE( + _req: Request, + { params }: { params: Promise<{ id: string }> }, +) { + const { id } = await params; + const pb = await serverAuth(); + try { + const rec = await pb.collection("invoice_intake").getOne(id); + if (rec.status === "imported") { + return Response.json( + { ok: false, error: "This file was already imported — delete the invoice instead" }, + { status: 400 }, + ); + } + await pb.collection("invoice_intake").delete(id); + revalidatePath("/income/invoices"); + return Response.json({ ok: true }); + } catch (e: any) { + return Response.json({ ok: false, error: e?.message || "Could not remove it" }, { status: 500 }); + } +} \ No newline at end of file diff --git a/app/api/invoices/intake/route.ts b/app/api/invoices/intake/route.ts new file mode 100644 index 0000000..7916545 --- /dev/null +++ b/app/api/invoices/intake/route.ts @@ -0,0 +1,229 @@ +import { revalidatePath } from "next/cache"; +import { serverAuth } from "@/lib/pb"; +import { parseInvoicePdf, ParseError, registerCodeFrom } from "@/lib/invoice-parser"; + +export const dynamic = "force-dynamic"; + +const MAX_BYTES = 20 * 1024 * 1024; + +// GET /api/invoices/intake +// The problem-invoice inbox: every PDF offered to the app and what became of it. +// `?status=failed` narrows it to the ones that need attention. +export async function GET(req: Request) { + const url = new URL(req.url); + const status = url.searchParams.get("status"); + try { + const pb = await serverAuth(); + // PocketBase rejects an empty filter string, so only send one when filtering. + const opts: Record = { sort: "-created" }; + if (status) opts.filter = `status = "${status}"`; + const records = await pb.collection("invoice_intake").getFullList(opts); + return Response.json({ + ok: true, + items: records.map((r: any) => ({ + id: r.id, + file_name: r.file_name, + file_size: r.file_size, + invoice_no: r.invoice_no, + status: r.status, + error: r.error, + parsed: r.parsed, + invoice: r.invoice, + created: r.created, + })), + }); + } catch (e: any) { + return Response.json({ ok: false, error: e?.message || "Could not read the inbox" }, { status: 500 }); + } +} + +// POST /api/invoices/intake +// Multipart: `file` (a PDF). The PDF is parsed deterministically, checked arithmetically, +// and only then written. A PDF that does not reconcile is stored in the inbox as a +// problem invoice with the reason attached — it is never partially imported. +export async function POST(req: Request) { + let form: FormData; + try { + form = await req.formData(); + } catch { + return Response.json({ ok: false, error: "Expected multipart/form-data" }, { status: 400 }); + } + + const file = form.get("file"); + if (!(file instanceof File) || file.size === 0) { + return Response.json({ ok: false, error: "No file received" }, { status: 400 }); + } + if (file.type !== "application/pdf" && !file.name.toLowerCase().endsWith(".pdf")) { + return Response.json({ ok: false, error: "Only PDF invoices are accepted" }, { status: 400 }); + } + if (file.size > MAX_BYTES) { + return Response.json( + { ok: false, error: `File is ${(file.size / 1048576).toFixed(1)} MB — limit is 20 MB` }, + { status: 400 }, + ); + } + + const bytes = new Uint8Array(await file.arrayBuffer()); + const pb = await serverAuth(); + + // Always record the attempt, so a bad PDF is never lost. + const recordFailure = async (reason: string, parsed?: unknown) => { + const form2 = new FormData(); + form2.append("file_name", file.name); + form2.append("file_size", String(file.size)); + form2.append("status", "failed"); + form2.append("error", reason); + if (parsed) form2.append("parsed", JSON.stringify(parsed)); + form2.append( + "source_pdf", + new File([bytes], file.name.replace(/[^\w.\- ]+/g, "_"), { type: "application/pdf" }), + ); + const rec = await pb.collection("invoice_intake").create(form2); + revalidatePath("/income/invoices"); + return Response.json({ ok: false, status: "failed", intakeId: rec.id, error: reason }, { status: 422 }); + }; + + let parsed; + try { + parsed = await parseInvoicePdf(bytes); + } catch (e: any) { + if (e instanceof ParseError) return recordFailure(e.reason); + return recordFailure(`Could not read the PDF: ${e?.message ?? "unknown error"}`); + } + + // Match each line to a plant item by the code the invoice prints. No match means the + // line is left unassigned for manual booking — never guessed. + const assets = await pb.collection("assets").getFullList(); + const byCode = new Map(); + for (const a of assets) if (a.code) byCode.set(a.code, a.id); + + const allInvoices = await pb.collection("invoices").getFullList(); + if (allInvoices.some((i: any) => i.invoice_no === parsed.invoice_no)) { + return recordFailure( + `Invoice ${parsed.invoice_no} has already been imported. Delete it first if you want to re-ingest this file.`, + parsed, + ); + } + + const periodEnd = new Date(parsed.period_to); + const bookedMonth = periodEnd.getUTCMonth() + 1; + const bookedYear = periodEnd.getUTCFullYear(); + + let invoiceRec; + try { + invoiceRec = await pb.collection("invoices").create({ + invoice_no: parsed.invoice_no, + invoice_date: parsed.invoice_date, + order_no: parsed.order_no, + debtor: parsed.debtor, + debtor_no: parsed.debtor_no, + period_from: parsed.period_from, + period_to: parsed.period_to, + description: parsed.description, + site: parsed.site, + section: parsed.section, + category: parsed.category, + currency: parsed.currency, + vat_reg_no: parsed.vat_reg_no, + source_file: file.name, + subtotal: parsed.subtotal, + escalation: parsed.escalation, + vat: parsed.vat, + vat_pct: parsed.vat_pct, + total_excl: parsed.total_excl, + total_incl: parsed.total_incl, + booked_month: bookedMonth, + booked_year: bookedYear, + is_booked: false, + }); + } catch (e: any) { + return recordFailure( + `Parsed correctly but could not save the invoice: ${e?.response?.data?.message || e?.message}`, + parsed, + ); + } + + const unassigned: string[] = []; + for (let i = 0; i < parsed.lines.length; i++) { + const l = parsed.lines[i]; + const registerCode = registerCodeFrom(l.machine_code); + const assetId = byCode.get(registerCode) || ""; + if (!assetId) unassigned.push(l.machine_code); + try { + await pb.collection("invoice_lines").create({ + invoice: invoiceRec.id, + line_no: i + 1, + machine_code: l.machine_code, + asset_code: registerCode, + description: l.description, + qty: l.qty, + unit: l.unit, + rate: l.rate, + rate_type: l.rate_type, + amount_excl: l.amount_excl, + vat: l.vat, + total_incl: l.total_incl, + booked_asset: assetId, + booked_month: bookedMonth, + booked_year: bookedYear, + }); + } catch (e: any) { + // A half-written invoice is worse than none — roll it back and report. + const written = await pb + .collection("invoice_lines") + .getFullList({ filter: `invoice = "${invoiceRec.id}"` }); + for (const w of written) await pb.collection("invoice_lines").delete(w.id); + await pb.collection("invoices").delete(invoiceRec.id); + return recordFailure( + `Parsed correctly but a line could not be saved, so the import was rolled back: ${e?.response?.data?.message || e?.message}`, + parsed, + ); + } + } + + // Book whatever could be matched. Anything unassigned stays visibly unbooked. + const lines = await pb.collection("invoice_lines").getFullList({ filter: `invoice = "${invoiceRec.id}"` }); + for (const line of lines) { + if (!line.booked_asset) continue; + await pb.collection("monthly_income").create({ + asset: line.booked_asset, + year: bookedYear, + month: bookedMonth, + income: line.total_incl, + fixed_expenses: 0, + is_forecast: false, + invoice_line: line.id, + }); + } + const allBooked = lines.every((l: any) => l.booked_asset); + await pb.collection("invoices").update(invoiceRec.id, { is_booked: allBooked }); + + const form2 = new FormData(); + form2.append("file_name", file.name); + form2.append("file_size", String(file.size)); + form2.append("invoice_no", parsed.invoice_no); + form2.append("status", "imported"); + form2.append("invoice", invoiceRec.id); + form2.append("parsed", JSON.stringify(parsed)); + form2.append( + "source_pdf", + new File([bytes], file.name.replace(/[^\w.\- ]+/g, "_"), { type: "application/pdf" }), + ); + await pb.collection("invoice_intake").create(form2); + + revalidatePath("/income/invoices"); + revalidatePath("/income"); + + return Response.json({ + ok: true, + status: "imported", + invoice_no: parsed.invoice_no, + invoiceId: invoiceRec.id, + site: parsed.site, + total_incl: parsed.total_incl, + lines: lines.length, + booked: lines.filter((l: any) => l.booked_asset).length, + unassigned, + fullyBooked: allBooked, + }); +} \ No newline at end of file diff --git a/app/income/invoices/page.tsx b/app/income/invoices/page.tsx index bfa2ecd..41d96f8 100644 --- a/app/income/invoices/page.tsx +++ b/app/income/invoices/page.tsx @@ -4,6 +4,7 @@ import { } from "@/lib/data"; import { FileText } from "lucide-react"; import { InvoiceList, EmptyInvoices } from "@/components/InvoiceList"; +import { AddInvoiceButton, ProblemInbox } from "@/components/InvoiceIntake"; // Reads PocketBase on every request, same as /income — see the note in app/income/page.tsx. export const dynamic = "force-dynamic"; @@ -40,8 +41,11 @@ export default async function InvoicesPage() { One row per invoice · click to open the line items and book each to a plant item

+ + + {invoices.length === 0 ? ( ) : ( diff --git a/components/InvoiceIntake.tsx b/components/InvoiceIntake.tsx new file mode 100644 index 0000000..bec922a --- /dev/null +++ b/components/InvoiceIntake.tsx @@ -0,0 +1,308 @@ +"use client"; + +import { useState, useRef, useEffect, useCallback } from "react"; +import { useRouter } from "next/navigation"; +import { + FileUp, Loader2, AlertTriangle, CheckCircle2, RotateCw, Inbox, Trash2, X, +} from "lucide-react"; +import { ZAR2 } from "@/lib/utils"; + +// "Add Invoice" — drop a PDF and the app parses it itself. No agent, no AI: the +// figures are read by coordinate from a fixed report template and checked +// arithmetically before anything is written. +export function AddInvoiceButton() { + const [open, setOpen] = useState(false); + const [busy, setBusy] = useState(false); + const [dragging, setDragging] = useState(false); + const [result, setResult] = useState(null); + const inputRef = useRef(null); + const router = useRouter(); + + const send = useCallback( + async (file: File) => { + setBusy(true); + setResult(null); + const fd = new FormData(); + fd.append("file", file); + try { + const res = await fetch("/api/invoices/intake", { method: "POST", body: fd }); + const data = await res.json().catch(() => ({ ok: false, error: "Unexpected response" })); + if (data.ok) { + const unassigned = (data.unassigned ?? []) as string[]; + setResult({ + ok: true, + title: `${data.invoice_no} imported`, + body: + `${data.lines} line${data.lines > 1 ? "s" : ""} · ${ZAR2(data.total_incl)} · ` + + `${data.booked} booked` + + (unassigned.length + ? `. Needs a plant item: ${unassigned.join(", ")}` + : data.fullyBooked + ? ". All lines matched a plant item." + : "."), + }); + } else { + setResult({ + ok: false, + title: data.status === "failed" ? "Saved to the problem inbox" : "Could not import", + body: data.error ?? "Unknown error", + }); + } + router.refresh(); + } catch { + setResult({ ok: false, title: "Could not reach the server", body: "Check the app is running." }); + } finally { + setBusy(false); + } + }, + [router], + ); + + useEffect(() => { + if (!open) return; + const onKey = (e: KeyboardEvent) => e.key === "Escape" && setOpen(false); + document.addEventListener("keydown", onKey); + return () => document.removeEventListener("keydown", onKey); + }, [open]); + + if (!open) { + return ( + + ); + } + + return ( +
+
+

Add invoice

+ +
+ +
!busy && inputRef.current?.click()} + onKeyDown={(e) => { + if ((e.key === "Enter" || e.key === " ") && !busy) { + e.preventDefault(); + inputRef.current?.click(); + } + }} + onDragOver={(e) => { + e.preventDefault(); + setDragging(true); + }} + onDragLeave={() => setDragging(false)} + onDrop={(e) => { + e.preventDefault(); + setDragging(false); + const f = e.dataTransfer.files?.[0]; + if (f) send(f); + }} + className={`flex flex-col sm:flex-row sm:items-center gap-3 px-4 py-4 rounded-xl border-2 border-dashed cursor-pointer transition ${ + dragging + ? "border-[#00798C] bg-[#00798C]/10" + : "border-gray-200 dark:border-white/10 hover:border-[#00798C]/50 hover:bg-gray-50 dark:hover:bg-white/[0.02]" + } ${busy ? "opacity-60 pointer-events-none" : ""}`} + > + {busy ? ( + + ) : ( + + )} +
+
+ {busy ? "Reading the invoice…" : "Drop the invoice PDF here, or tap to browse"} +
+
+ PDF only · up to 20 MB · figures are read from the invoice and checked before saving +
+
+
+ + { + const f = e.target.files?.[0]; + if (f) send(f); + e.target.value = ""; + }} + /> + + {result && ( +
+ {result.ok ? ( + + ) : ( + + )} +
+
{result.title}
+
{result.body}
+
+
+ )} +
+ ); +} + +type IntakeItem = { + id: string; + file_name: string; + file_size: number; + invoice_no: string; + status: "imported" | "failed"; + error: string; + created: string; +}; + +// The problem-invoice inbox. Anything that could not be parsed or imported lands here +// with the reason, so it can be inspected and retried instead of disappearing. +export function ProblemInbox() { + const [items, setItems] = useState(null); + const [retrying, setRetrying] = useState(null); + const [msg, setMsg] = useState(null); + const router = useRouter(); + + const load = useCallback(async () => { + try { + const res = await fetch("/api/invoices/intake?status=failed", { cache: "no-store" }); + const data = await res.json().catch(() => ({ ok: false, items: [] })); + setItems(data.ok ? data.items : []); + } catch { + setItems([]); + } + }, []); + + useEffect(() => { + load(); + }, [load]); + + const retry = async (id: string) => { + setRetrying(id); + setMsg(null); + try { + const res = await fetch(`/api/invoices/intake/${id}/retry`, { method: "POST" }); + const data = await res.json().catch(() => ({ ok: false })); + setMsg( + data.ok + ? `${data.invoice_no} imported on retry.` + : `Still failing: ${data.error ?? "unknown reason"}`, + ); + await load(); + router.refresh(); + } catch { + setMsg("Could not reach the server"); + } finally { + setRetrying(null); + } + }; + + const discard = async (id: string) => { + setRetrying(id); + try { + await fetch(`/api/invoices/intake/${id}`, { method: "DELETE" }); + await load(); + } catch { + setMsg("Could not reach the server"); + } finally { + setRetrying(null); + } + }; + + return ( +
+
+ +

Problem invoices

+ + {items?.length ?? 0} + +
+ + {msg && ( +
{msg}
+ )} + + {items === null ? ( +

Checking…

+ ) : items.length === 0 ? ( +

+ Nothing in the inbox — every PDF offered so far imported cleanly. +

+ ) : ( +
    + {items.map((it) => ( +
  • +
    +
    +
    + + {it.file_name} + + {it.invoice_no && ( + + reads as {it.invoice_no} + + )} +
    +

    {it.error}

    +

    + {(it.file_size / 1024).toFixed(0)} KB + {it.created ? ` · ${new Date(it.created).toLocaleString("en-ZA")}` : ""} +

    +
    +
    + + +
    +
    +
    • + ))} +
+ )} +
+ ); +} \ No newline at end of file diff --git a/lib/invoice-parser.test.ts b/lib/invoice-parser.test.ts new file mode 100644 index 0000000..546710b --- /dev/null +++ b/lib/invoice-parser.test.ts @@ -0,0 +1,129 @@ +// Deterministic-parser tests. +// +// The parser reads figures by coordinate from a fixed report template, so the risk is +// silent drift: a column moves and it starts returning a plausible wrong number. These +// tests pin the behaviour that matters — the column mapping, the code/description split, +// date formats, and the fact that anything which does not reconcile is refused. +// +// Run: node --import tsx --test lib/invoice-parser.test.ts +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { existsSync, readFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { parseInvoicePdf, registerCodeFrom, siteFromDescription, isoDate, ParseError } from "./invoice-parser"; + +// --- unit level: the pieces that do not need a PDF ------------------------------- + +test("registerCodeFrom strips punctuation and suffix from an invoice code", () => { + assert.equal(registerCodeFrom("SS-01-CASE"), "SS01"); + assert.equal(registerCodeFrom("TP601-TATA"), "TP601"); + assert.equal(registerCodeFrom("MG-01-DEZI"), "MG01"); + assert.equal(registerCodeFrom("RV-03-CAT"), "RV03"); + assert.equal(registerCodeFrom("TLB07-CASE"), "TLB07"); + // No register equivalent — must NOT be silently coerced into something plausible. + assert.equal(registerCodeFrom("MPLANT05"), "MPLANT05"); +}); + +test("site is read from the invoice description", () => { + assert.equal(siteFromDescription("Plant Hire Sep 2026 West Park"), "West Park"); + assert.equal(siteFromDescription("Hire Plant Sep 2026 Halfway Gardens"), "Halfway Gardens"); + assert.equal(siteFromDescription("Hire Plant Sep 2026 Rooihuiskraal"), "Rooihuiskraal"); + assert.equal(siteFromDescription("Plant Hire SEp 2026 Cosmo City"), "Cosmo City"); + assert.equal(siteFromDescription("something else entirely"), ""); +}); + +test("dates accept the report's written formats", () => { + assert.equal(isoDate("18-September-2026"), "2026-09-18"); + assert.equal(isoDate("21/08/2026"), "2026-08-21"); + assert.throws(() => isoDate("not a date"), ParseError); +}); + +test("a PDF with no recognisable content is refused, not guessed", async () => { + const notAPdf = new TextEncoder().encode("%PDF-1.4\nthis is not a real invoice\n"); + await assert.rejects(() => parseInvoicePdf(notAPdf), /PDF|Could not/i); +}); + +// --- integration level: the real invoices ----------------------------------------- +// +// The source PDFs are not committed (they are client documents), so these skip when +// they are absent. Point FIXTURE_DIR at a folder holding PHI*.pdf to run them. + +const DIR = process.env.FIXTURE_DIR ?? "/tmp"; +const REAL: [string, string][] = [ + ["PHI166.pdf", "invoice-phi166.json"], + ["PHI167.pdf", "invoice-phi167.json"], + ["PHI168.pdf", "invoice-phi168.json"], + ["PHI169.pdf", "invoice-phi169.json"], + ["PHI171.pdf", "invoice-phi171.json"], +]; + +const available = REAL.filter(([pdf]) => existsSync(resolve(DIR, pdf))); + +test("parser reproduces the hand-extracted invoices exactly", { skip: available.length ? false : "no source PDFs present" }, async () => { + const failures: string[] = []; + for (const [pdfName, jsonName] of available) { + const src = JSON.parse(readFileSync(resolve("data/source", jsonName), "utf8")); + const got = await parseInvoicePdf(new Uint8Array(readFileSync(resolve(DIR, pdfName)))); + + const cmp = (label: string, a: unknown, b: unknown) => { + const ok = typeof a === "number" && typeof b === "number" + ? Math.abs(a - b) < 0.01 + : String(a ?? "") === String(b ?? ""); + if (!ok) failures.push(`${jsonName} ${label}: parser=${a} source=${b}`); + }; + + cmp("invoice_no", got.invoice_no, src.invoice_no); + cmp("invoice_date", got.invoice_date, src.invoice_date); + cmp("order_no", got.order_no, src.order_no); + cmp("debtor", got.debtor, src.debtor); + cmp("debtor_no", got.debtor_no, src.debtor_no); + cmp("period_from", got.period_from, src.period_from); + cmp("period_to", got.period_to, src.period_to); + cmp("description", got.description, src.description); + cmp("site", got.site, src.site); + cmp("currency", got.currency, src.currency); + cmp("vat_pct", got.vat_pct, src.vat_pct); + cmp("subtotal", got.subtotal, src.subtotal); + cmp("escalation", got.escalation, src.escalation); + cmp("vat", got.vat, src.vat); + cmp("total_excl", got.total_excl, src.total_excl); + cmp("total_incl", got.total_incl, src.total_incl); + + if (got.lines.length !== src.lines.length) { + failures.push(`${jsonName} line count: ${got.lines.length} vs ${src.lines.length}`); + continue; + } + got.lines.forEach((l, i) => { + const s = src.lines[i]; + cmp(`line${i + 1}.amount_excl`, l.amount_excl, s.amount_excl); + cmp(`line${i + 1}.vat`, l.vat, s.vat); + cmp(`line${i + 1}.total_incl`, l.total_incl, s.total_incl); + cmp(`line${i + 1}.qty`, l.qty, s.qty); + cmp(`line${i + 1}.unit`, l.unit, s.unit); + cmp(`line${i + 1}.rate`, l.rate, s.rate); + // machine_code is compared through registerCodeFrom: one input ("RV-01-CATCB22") + // has no separator between code and description, so the parser's split differs + // cosmetically from the hand extraction. What must match is the plant item the + // line books to, which is derived from exactly this value. + cmp(`line${i + 1}.register code`, registerCodeFrom(l.machine_code), registerCodeFrom(s.machine_code)); + }); + } + assert.deepEqual(failures, [], `parser disagreed with the hand extraction:\n${failures.join("\n")}`); +}); + +test("the parser never returns an invoice whose totals do not reconcile", { skip: available.length ? false : "no source PDFs present" }, async () => { + // Every successful parse must satisfy the balance rules on its own — this is the + // guarantee the intake route relies on before it is allowed to write anything. + for (const [pdfName] of available) { + const got = await parseInvoicePdf(new Uint8Array(readFileSync(resolve(DIR, pdfName)))); + const sumHire = Math.round(got.lines.reduce((s, l) => s + l.amount_excl, 0) * 100) / 100; + const sumVat = Math.round(got.lines.reduce((s, l) => s + l.vat, 0) * 100) / 100; + assert.ok(Math.abs(sumHire - got.subtotal) < 0.01, `${pdfName}: lines != subtotal`); + assert.ok(Math.abs(sumVat - got.vat) < 0.02, `${pdfName}: line VAT != header VAT`); + assert.ok(Math.abs(got.total_excl + got.vat - got.total_incl) < 0.01, `${pdfName}: totals do not balance`); + for (const l of got.lines) { + assert.ok(Math.abs(l.amount_excl + l.vat - l.total_incl) < 0.01, `${pdfName}: ${l.machine_code} does not balance`); + assert.ok(Math.abs(l.qty * l.rate - l.amount_excl) < 0.01, `${pdfName}: ${l.machine_code} qty x rate != amount`); + } + } +}); \ No newline at end of file diff --git a/lib/invoice-parser.ts b/lib/invoice-parser.ts new file mode 100644 index 0000000..1a8319a --- /dev/null +++ b/lib/invoice-parser.ts @@ -0,0 +1,423 @@ +// Deterministic parser for Mega Plant "Std_ReqHireInvoice_DN.rpt" VAT invoices. +// +// Deliberately NOT an AI extractor. That supplier's report is a fixed template: the +// columns sit at identical x-positions on every invoice and only the number of +// machine lines and pages varies. So the figures are read by coordinate, exactly the +// way they were transcribed by hand, and then checked arithmetically. +// +// Anything that does not reconcile is reported as a reason and lands in the intake +// inbox as a problem invoice. It never guesses a total and never writes a partial +// invoice. +// +// Calibrated against PHI 166 / 167 / 168 / 169 / 171. + +export type ParsedLine = { + machine_code: string; + description: string; + qty: number; + unit: string; + rate: number; + rate_type: string; + amount_excl: number; + vat: number; + total_incl: number; +}; + +export type ParsedInvoice = { + invoice_no: string; + invoice_date: string; + order_no: string; + debtor: string; + debtor_no: string; + period_from: string; + period_to: string; + description: string; + site: string; + category: string; + section: string; + currency: string; + vat_reg_no: string; + vat_pct: number; + subtotal: number; + escalation: number; + vat: number; + total_excl: number; + total_incl: number; + lines: ParsedLine[]; +}; + +export class ParseError extends Error { + constructor(public reason: string) { + super(reason); + this.name = "ParseError"; + } +} + +// Column left-edges, calibrated from the report template. Numbers are right-aligned +// within their column, so each value's LEFT edge lands in a stable band. +const COL = { + HIRE: [340, 400], + PENALTY2: [400, 440], + ESCALATION: [440, 490], + VAT: [490, 534], + TOTAL: [534, 1e9], +} as const; + +// Activity sub-row ("Dry Rate 0001"). +const ACT = { + QTY: [220, 262], + UNIT: [262, 300], + RATE: [300, 350], + AMOUNT: [350, 392], +} as const; + +type Word = { x: number; y: number; str: string }; + +const inBand = (x: number, [lo, hi]: readonly [number, number]) => x >= lo && x < hi; + +// "1,234.50" / "1.234,50" / "1 234,50" / "-" / "" +function num(s: string | undefined): number | null { + if (!s) return null; + const t = s.replace(/\s/g, "").replace(/,/g, "").replace(/ /g, ""); + if (!t || t === "-" || t === "–") return null; + const n = Number(t); + return Number.isFinite(n) ? n : null; +} + +const r2 = (n: number) => Math.round(n * 100) / 100; + +/** + * Splits a machine row into its code and description. + * + * The code is a leading alpha block plus a 1-3 digit block, optionally followed by a + * dash and an upper-case token that must end at whitespace. That rule resolves every + * recurring case correctly, including the two that look alike but aren't: + * "MPLANT05-Storage Container" -> MPLANT05 / Storage Container (suffix ends in lowercase) + * "RV-01-CATCB22 2.2 TON ..." -> RV-01 / CATCB22 2.2 TON ... (suffix runs into a digit) + * The second yields a shorter code, which is cosmetic — no amount depends on it, and + * that code does not repeat across invoices, so duplicate detection is unaffected. + */ +function splitMachineRow(text: string): { code: string; description: string } { + const trimmed = text.trim(); + const core = /^([A-Za-z]{2,8}-?[0-9]{2,3})/.exec(trimmed); + if (!core) { + const token = trimmed.split(/\s+/)[0] ?? trimmed; + return { code: token, description: "" }; + } + const base = core[1]; + let rest = trimmed.slice(base.length); + // An optional -SUFFIX belongs to the code only when it ends at whitespace. That is + // what separates the two look-alike cases: + // "MPLANT05-Storage Container" -> MPLANT05 / Storage Container (suffix ends lowercase) + // "MG-01-DEZI MG80 ..." -> MG-01-DEZI / MG80 ... (suffix ends at a space) + // The suffix is always UPPER CASE (codes are) and must be bounded, which is what + // separates a real code suffix from the start of a description word. Two bounds occur + // in this report: + // 1. the suffix ends at whitespace "MG-01-DEZI MG80 Motor Grader" + // 2. the suffix runs into a digit group that ends at whitespace + // "RV-01-CATCB22 2.2 TON Roller" + // Known limit: "RV-01-CATCB22 2.2 TON" has no separator between the code and the + // description ("CB22" is the machine model), so the split point is a judgement call. + // The parser takes the maximal upper-case run, giving RV-01-CATCB. This is a display + // label only — no amount depends on it, and registerCodeFrom() still resolves it to + // RV01, so booking is unaffected. The hand extraction records RV-01-CAT. + // Case matters: matching [A-Za-z] here would swallow the "Office" of + // "MPLANT04-Office Container" and lose the code. + const sfx = + /^-([A-Z0-9]{1,6})(?=\s|$)/.exec(rest) ?? + /^-([A-Z]{2,5})(?=[A-Z]*[0-9]+[\s,])/.exec(rest); + let code = base; + if (sfx) { + code = `${base}-${sfx[1]}`; + rest = rest.slice(sfx[0].length); + } + return { code, description: rest.trim().replace(/^[-–]\s*/, "") }; +} + +// The code an invoice uses maps to a register code by taking the leading letters+digits +// and dropping punctuation: "SS-01-CASE" -> "SS01", "TP601-TATA" -> "TP601". +export function registerCodeFrom(invoiceCode: string): string { + const m = /^([A-Za-z]+)[- ]?([0-9]{1,3})/.exec(invoiceCode.trim()); + return m ? `${m[1].toUpperCase()}${m[2]}` : ""; +} + +// "Hire Plant Sep 2026 Rooihuiskraal" / "Plant Hire SEP 2026 West Park" -> the site. +export function siteFromDescription(desc: string): string { + const m = /^(?:hire\s+plant|plant\s+hire)\s+\S+\s+\d{4}\s+(.+)$/i.exec(desc.trim()); + return m ? m[1].trim() : ""; +} + +const MONTHS: Record = { + jan: "01", feb: "02", mar: "03", apr: "04", may: "05", jun: "06", + jul: "07", aug: "08", sep: "09", oct: "10", nov: "11", dec: "12", +}; + +/** "18-September-2026" / "18/09/2026" -> "2026-09-18" */ +export function isoDate(s: string): string { + const t = s.trim(); + let m = /^(\d{1,2})[/-]([A-Za-z]{3,})[/-](\d{4})$/.exec(t); + if (m) { + const mo = MONTHS[m[2].slice(0, 3).toLowerCase()]; + if (!mo) throw new ParseError(`Unrecognised month "${m[2]}" in date "${s}"`); + return `${m[3]}-${mo}-${m[1].padStart(2, "0")}`; + } + m = /^(\d{1,2})\/(\d{2})\/(\d{4})$/.exec(t); + if (m) return `${m[3]}-${m[2]}-${m[1].padStart(2, "0")}`; + m = /^(\d{4})-(\d{2})-(\d{2})$/.exec(t); + if (m) return t; + throw new ParseError(`Unrecognised date format "${s}"`); +} + +// pdfjs resolves its worker lazily at runtime. Left to its own devices under Next it +// falls back to a dynamic import of the worker module, which the bundler rewrites to a +// .next path that does not exist in the standalone image — every parse then dies with +// "Setting up fake worker failed". Pinning the worker to the real installed file avoids +// that. pdfjs-dist is listed in serverExternalPackages so it is required (and traced) +// from node_modules rather than bundled. +let workerPinned = false; +async function loadPdfjs() { + const pdfjs = await import("pdfjs-dist/legacy/build/pdf.mjs"); + if (!workerPinned) { + workerPinned = true; + try { + const { createRequire } = await import("node:module"); + const req = createRequire(import.meta.url); + pdfjs.GlobalWorkerOptions.workerSrc = req.resolve( + "pdfjs-dist/legacy/build/pdf.worker.mjs", + ); + } catch { + // Leave pdfjs's own default in place; it is correct for unbundled Node. + } + } + return pdfjs; +} + +export async function parseInvoicePdf(bytes: Uint8Array): Promise { + const pdfjs = await loadPdfjs(); + // pdfjs TRANSFERS the buffer it is handed, which detaches it. Callers keep their own + // copy (to store the original file), so parse a duplicate rather than the original. + const doc = await pdfjs.getDocument({ + data: new Uint8Array(bytes), + useSystemFonts: true, + }).promise; + + const pages: Word[][] = []; + try { + for (let p = 1; p <= doc.numPages; p++) { + const page = await doc.getPage(p); + const vp = page.getViewport({ scale: 1 }); + const content = await page.getTextContent(); + const words: Word[] = []; + for (const it of content.items as any[]) { + if (typeof it.str !== "string" || !it.str.trim()) continue; + words.push({ x: it.transform[4], y: vp.height - it.transform[5], str: it.str }); + } + pages.push(words); + } + + // ---- header (page 1) ------------------------------------------------- + const h1 = pages[0] ?? []; + const onRow = (words: Word[], label: string) => words.filter((w) => w.str.startsWith(label)); + + const after = (label: string, page = 0): string => { + const ws = onRow(pages[page] ?? [], label); + if (!ws.length) return ""; + const y = ws[0].y; + const same = (pages[page] ?? []).filter((w) => Math.abs(w.y - y) < 3 && w.x > ws[0].x + ws[0].str.length); + return same.map((w) => w.str).join(" ").trim(); + }; + + const invoiceNo = after("Invoice") || ""; + if (!invoiceNo) throw new ParseError('Could not find "Invoice No." in the header'); + const orderNo = after("ORDER NUMBER") || ""; + const invoiceDateRaw = after("DATE") || ""; + if (!invoiceDateRaw) throw new ParseError('Could not find "DATE" in the header'); + const debtorNo = after("DEBTOR NUMBER") || ""; + const currency = after("CURRENCY") || "ZAR"; + const vatReg = after("VAT Reg. No.") || ""; + + // Debtor name sits at the far left of the block above ORDER NUMBER. + const debtorWord = h1.find((w) => w.x < 100 && /^Mega |Infrastructure/i.test(w.str) && w.y > 180 && w.y < 220); + const debtor = debtorWord ? debtorWord.str.replace(/^Mega\s+/i, "Mega ") : ""; + + // "Description :" and the free text beside it. + const descRow = h1.filter((w) => Math.abs(w.y - (h1.find((x) => x.str === "Description")?.y ?? -999)) < 3); + const descWord = descRow.find((w) => w.str === "Description"); + const description = descWord + ? descRow + .filter( + (w) => + w.x > descWord.x + 12 && + w.x < 460 && + !w.str.endsWith(":") && + !/^\d{1,2}\/\d{2}\/\d{4}$/.test(w.str) && + !/Date$/.test(w.str), + ) + .sort((a, b) => a.x - b.x) + .map((w) => w.str) + .join(" ") + .trim() + : ""; + + const dates = h1 + .filter((w) => /^\d{1,2}\/\d{2}\/\d{4}$/.test(w.str)) + .sort((a, b) => a.y - b.y || a.x - b.x) + .map((w) => w.str); + if (dates.length < 2) throw new ParseError("Could not read the From/To dates from the header"); + const period_from = isoDate(dates[0]); + const period_to = isoDate(dates[1]); + const invoice_date = isoDate(invoiceDateRaw); + + // ---- machine lines (every page) -------------------------------------- + const lines: ParsedLine[] = []; + for (const words of pages) { + // A machine row is a left-column item that is not a known header/footer word. + const machineRows = words.filter( + (w) => w.x < 45 && /[A-Za-z]/.test(w.str) && !/^(Return|Sep|Dry|Rib|Ref|Printed|Description)$/i.test(w.str) && + !/\bNo\.$/.test(w.str) && /[A-Za-z]{2,8}-?[0-9]{2,3}/.test(w.str), + ); + for (const mr of machineRows) { + // Numeric row sits ~16pt below the machine name. + const numRow = words.filter((w) => w.y > mr.y + 10 && w.y < mr.y + 24); + const hire = num(numRow.find((w) => inBand(w.x, COL.HIRE))?.str); + const vat = num(numRow.find((w) => inBand(w.x, COL.VAT))?.str); + const total = num(numRow.find((w) => inBand(w.x, COL.TOTAL))?.str); + if (hire == null || vat == null || total == null) continue; + + // Activity row: the "Dry Rate ..." line, roughly 57pt below the machine name. + // The "Description Qty Unit ..." header sits at ~44pt and must not be mistaken + // for it, so locate the row by its leading "Dry" token rather than by offset. + const band = words.filter((w) => w.y > mr.y + 38 && w.y < mr.y + 78 && w.x > 150); + const dryY = band.find((w) => w.str === "Dry" || w.str.startsWith("Dry "))?.y; + const dry = dryY == null ? [] : band.filter((w) => Math.abs(w.y - dryY) < 3).sort((a, b) => a.x - b.x); + const qty = num(dry.find((w) => inBand(w.x, ACT.QTY))?.str); + const unit = dry.find((w) => inBand(w.x, ACT.UNIT))?.str ?? ""; + const rate = num(dry.find((w) => inBand(w.x, ACT.RATE))?.str); + // pdfjs emits "Dry Rate" as a single token; older builds split it in two. + const rateType = dry[0] ? dry[0].str.replace(/\s+/g, " ") : "Dry Rate"; + + const { code, description: desc } = splitMachineRow(mr.str); + lines.push({ + machine_code: code, + description: desc, + qty: qty ?? 0, + unit, + rate: rate ?? 0, + rate_type: rateType, + amount_excl: r2(hire), + vat: r2(vat), + total_incl: r2(total), + }); + } + } + if (!lines.length) throw new ParseError("No machine lines were found on any page"); + + // ---- totals ----------------------------------------------------------- + // The report emits the grand-total label as ONE text item ("Plant Hire Total"), + // so match on prefix rather than equality. + let hireTotal: number | null = null; + let taxTotal: number | null = null; + let grandTotal: number | null = null; + let escalationTotal = 0; + for (const words of pages) { + const row = words.find((w) => w.str.startsWith("Plant Hire Total")); + if (!row) continue; + const same = words.filter((w) => Math.abs(w.y - row.y) < 4 && w.x > row.x + row.str.length); + hireTotal = num(same.find((w) => inBand(w.x, COL.HIRE))?.str); + escalationTotal = num(same.find((w) => inBand(w.x, COL.ESCALATION))?.str) ?? 0; + taxTotal = num(same.find((w) => inBand(w.x, COL.VAT))?.str); + grandTotal = num(same.find((w) => inBand(w.x, COL.TOTAL))?.str); + } + if (hireTotal == null || taxTotal == null || grandTotal == null) { + throw new ParseError('Could not find the "Plant Hire Total" row'); + } + + // The summary block is read BY LABEL — its values sit at different x positions + // from the line table, so column bands cannot be used here. pdfjs splits these + // labels inconsistently between pages, so each is matched by prefix. + const labelValue = (words: Word[], label: string): number | null => { + const lw = words.find((w) => w.str === label || w.str.startsWith(`${label} `)); + if (!lw) return null; + const labelEnd = lw.x + lw.str.length; + const same = words.filter((w) => Math.abs(w.y - lw.y) < 5 && w.x > labelEnd + 15); + return num(same.map((w) => w.str).join("")); + }; + + const summary: Record = {}; + for (const words of pages) { + const put = (k: string, v: number | null) => { if (v != null) summary[k] = v; }; + put("SUB TOTAL", labelValue(words, "SUB TOTAL")); + put("Escalation", labelValue(words, "Escalation")); + put("PLUS", labelValue(words, "PLUS")); + put("INVOICE", labelValue(words, "INVOICE")); + } + + const subtotal = summary["SUB TOTAL"] ?? hireTotal; + const escalation = summary["Escalation"] ?? escalationTotal; + const vat = summary["PLUS"] ?? taxTotal; + const total_excl = r2(subtotal + escalation); + const total_incl = summary["INVOICE"] ?? grandTotal; + + // ---- validation — the same gates the seed applies --------------------- + const sumHire = r2(lines.reduce((s, l) => s + l.amount_excl, 0)); + if (Math.abs(sumHire - subtotal) > 0.01) { + throw new ParseError(`Machine lines total R${sumHire} but the invoice subtotal is R${subtotal}`); + } + const sumVat = r2(lines.reduce((s, l) => s + l.vat, 0)); + if (Math.abs(sumVat - vat) > 0.02) { + throw new ParseError(`Line VAT totals R${sumVat} but the invoice PLUS VAT is R${vat}`); + } + if (Math.abs(total_excl + vat - total_incl) > 0.01) { + throw new ParseError( + `Totals do not balance: R${total_excl} ex + R${vat} VAT != R${total_incl} invoice total`, + ); + } + for (const l of lines) { + if (Math.abs(l.amount_excl + l.vat - l.total_incl) > 0.01) { + throw new ParseError( + `Line ${l.machine_code}: R${l.amount_excl} + R${l.vat} VAT != R${l.total_incl}`, + ); + } + if (l.qty && l.rate && Math.abs(r2(l.qty * l.rate) - l.amount_excl) > 0.01) { + throw new ParseError( + `Line ${l.machine_code}: ${l.qty} x ${l.rate} = ${r2(l.qty * l.rate)}, but the line amount is R${l.amount_excl}`, + ); + } + } + + const vat_pct = total_excl > 0 ? Math.round((vat / total_excl) * 1000) / 10 : 0; + for (const l of lines) { + const expected = r2((l.amount_excl * vat_pct) / 100); + if (l.amount_excl > 0 && Math.abs(expected - l.vat) > 0.02) { + throw new ParseError( + `Line ${l.machine_code}: VAT is R${l.vat}, but ${vat_pct}% of R${l.amount_excl} is R${expected}`, + ); + } + } + + return { + invoice_no: invoiceNo.trim().replace(/\s+/g, " "), + invoice_date, + order_no: orderNo, + debtor, + debtor_no: debtorNo, + period_from, + period_to, + description, + site: siteFromDescription(description), + section: "Plant Hire", + category: "External Plant Hire", + currency, + vat_reg_no: vatReg, + vat_pct, + subtotal: r2(subtotal), + escalation: r2(escalation), + vat: r2(vat), + total_excl, + total_incl: r2(total_incl), + lines, + }; + } finally { + await doc.destroy().catch(() => {}); + } +} diff --git a/next.config.ts b/next.config.ts index 0a9acf5..6e33586 100644 --- a/next.config.ts +++ b/next.config.ts @@ -19,6 +19,13 @@ const nextConfig: NextConfig = { // Dockerfile. Note it does NOT bundle `.next/static` or `public/` — the // Dockerfile copies those in separately. output: "standalone", + + // pdfjs-dist dynamically imports its own worker module at runtime. When Next bundles + // it, that import is rewritten to a path inside .next that does not exist in the + // standalone image, and every parse dies with "Setting up fake worker failed". + // Keeping it external means it is required from node_modules at runtime and traced + // into the image, so the worker resolves next to the library that needs it. + serverExternalPackages: ["pdfjs-dist"], }; const withPWA = withPWAInit({ diff --git a/package-lock.json b/package-lock.json index 4f63127..acb1edb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -20,6 +20,7 @@ "clsx": "^2.1.1", "lucide-react": "^0.469.0", "next": "^16", + "pdfjs-dist": "^4.10.38", "pocketbase": "^0.27.3", "react": "^19.0.0", "react-dom": "^19.0.0", @@ -2639,6 +2640,271 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, + "node_modules/@napi-rs/canvas": { + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas/-/canvas-0.1.100.tgz", + "integrity": "sha512-xglYA6q3XO5P3BNJYxVZ1IV7DLVjp1Py6nwag88YntrS+3vKHyYcMqXVS4ZztJmwz2uGvz1FWhI/4LgbR5uQDA==", + "license": "MIT", + "optional": true, + "workspaces": [ + "e2e/*" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + }, + "optionalDependencies": { + "@napi-rs/canvas-android-arm64": "0.1.100", + "@napi-rs/canvas-darwin-arm64": "0.1.100", + "@napi-rs/canvas-darwin-x64": "0.1.100", + "@napi-rs/canvas-linux-arm-gnueabihf": "0.1.100", + "@napi-rs/canvas-linux-arm64-gnu": "0.1.100", + "@napi-rs/canvas-linux-arm64-musl": "0.1.100", + "@napi-rs/canvas-linux-riscv64-gnu": "0.1.100", + "@napi-rs/canvas-linux-x64-gnu": "0.1.100", + "@napi-rs/canvas-linux-x64-musl": "0.1.100", + "@napi-rs/canvas-win32-arm64-msvc": "0.1.100", + "@napi-rs/canvas-win32-x64-msvc": "0.1.100" + } + }, + "node_modules/@napi-rs/canvas-android-arm64": { + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-android-arm64/-/canvas-android-arm64-0.1.100.tgz", + "integrity": "sha512-hjhCKhntPv9+t4ckHymdx0phYNcVW+GKQR6Lzw2zE+pOVjOplSmtx9nNNknTjbEDLcuLZqA1y8ufKg1XfgftzQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-darwin-arm64": { + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-arm64/-/canvas-darwin-arm64-0.1.100.tgz", + "integrity": "sha512-2PcswRaC7Ly645DGt88///zuFDhJxJYdKAs1uU3mfk1atYkXufgcgLfBpk6Tm12nCQBaNt1wpybuPZ4qOhTo8A==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-darwin-x64": { + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-x64/-/canvas-darwin-x64-0.1.100.tgz", + "integrity": "sha512-ePNZtj7pNIva/siZMg+HmbeozkIjqUIYdoymH8HaA3qK7LfzFN4WMBM8G6HQ9ZC+H3+Dnn5pqtiXpgLykaPOhw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-arm-gnueabihf": { + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm-gnueabihf/-/canvas-linux-arm-gnueabihf-0.1.100.tgz", + "integrity": "sha512-d5cDB48oWFGU8/XPhUOFAlySgb/VAu7D+s8fi55K1Pcfg8aPplHWqMgibhVLU8ky7Pyg/fuiVLz4Nf3JrSTuUA==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-arm64-gnu": { + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-gnu/-/canvas-linux-arm64-gnu-0.1.100.tgz", + "integrity": "sha512-rDxgxRu69RvDlX/bh9o22DxLsGr8EqsNgotL9+RwQE1S0b0cqeatqsw6aW45mukm0B42DIAaAacKaYQ8cqS1nw==", + "cpu": [ + "arm64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-arm64-musl": { + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-musl/-/canvas-linux-arm64-musl-0.1.100.tgz", + "integrity": "sha512-K3mDW66N+xT2/V439u1alFANiBUjdEx2gLiNYnCmUsva5jZMxWTjafBYwTzYK+EMFMHrUoabuU+T1BIP5CgbYQ==", + "cpu": [ + "arm64" + ], + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-riscv64-gnu": { + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-riscv64-gnu/-/canvas-linux-riscv64-gnu-0.1.100.tgz", + "integrity": "sha512-mooqUBTIsccZpnoQC4NgrC1v6C1vof39etLNMnBwCY+p0gajWJvAHLGQ6g/gGyS5YrpDW+GefSN4+Cvcr08UWw==", + "cpu": [ + "riscv64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-x64-gnu": { + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-gnu/-/canvas-linux-x64-gnu-0.1.100.tgz", + "integrity": "sha512-1eCvkDCazm7FFhsT7DfGOdSaHgZVK3bt/dSBl5EWHOWmnz+I7j8tPseJqqD81NF+MH21jKUK4wQSDjN0mdhnTg==", + "cpu": [ + "x64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-x64-musl": { + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-musl/-/canvas-linux-x64-musl-0.1.100.tgz", + "integrity": "sha512-20arT6lnI19S68qNlii73TSEDbECNgzMz2EpldC1V3mZFuRkeujXkcebRk0LRJe9SEUAooYiLokfMViY8IX7yA==", + "cpu": [ + "x64" + ], + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-win32-arm64-msvc": { + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-win32-arm64-msvc/-/canvas-win32-arm64-msvc-0.1.100.tgz", + "integrity": "sha512-DZFFT1wIAg37LJw37yhMRFfjATd3vTQzjZ1Yki8u2vhO6Hi5VE6BVaGQ1aaDu7xb4iMErz+9EOwjpS7xcxFeBw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-win32-x64-msvc": { + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-win32-x64-msvc/-/canvas-win32-x64-msvc-0.1.100.tgz", + "integrity": "sha512-MyT1j3mHC2+Lu4pBi9mKyMJhtP6U7k7EldY7sj/uS5gJA65gTXt8MefJQXLJo5d/vZbuWmfxzkEUNc/urV3pHA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, "node_modules/@next/env": { "version": "16.3.5", "resolved": "https://registry.npmjs.org/@next/env/-/env-16.3.5.tgz", @@ -7014,6 +7280,18 @@ "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", "license": "MIT" }, + "node_modules/pdfjs-dist": { + "version": "4.10.38", + "resolved": "https://registry.npmjs.org/pdfjs-dist/-/pdfjs-dist-4.10.38.tgz", + "integrity": "sha512-/Y3fcFrXEAsMjJXeL9J8+ZG9U01LbuWaYypvDW2ycW1jL269L3js3DVBjDJ0Up9Np1uqDXsDrRihHANhZOlwdQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=20" + }, + "optionalDependencies": { + "@napi-rs/canvas": "^0.1.65" + } + }, "node_modules/picocolors": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", diff --git a/package.json b/package.json index 299f72f..c466e6d 100644 --- a/package.json +++ b/package.json @@ -23,6 +23,7 @@ "clsx": "^2.1.1", "lucide-react": "^0.469.0", "next": "^16", + "pdfjs-dist": "^4.10.38", "pocketbase": "^0.27.3", "react": "^19.0.0", "react-dom": "^19.0.0", diff --git a/scripts/setup-invoice-collections.mjs b/scripts/setup-invoice-collections.mjs index bb2570c..b451481 100644 --- a/scripts/setup-invoice-collections.mjs +++ b/scripts/setup-invoice-collections.mjs @@ -187,6 +187,26 @@ const invoiceLinesId = await ensureCollection("invoice_lines", [ // invoice has to be re-runnable without double-counting. Marking each posted row with // the invoice line it came from makes booking idempotent and auditable: a row with an // empty `invoice_line` is workbook data, a row with one is invoice-derived. +// The intake inbox. Every PDF that is offered to the app lands here first, with the +// outcome of its parse. Anything that does not reconcile stays as `failed` with the +// reason recorded, so it can be reviewed and retried rather than silently lost. +await ensureCollection("invoice_intake", [ + { name: "file_name", type: "text" }, + { name: "file_size", type: "number" }, + { name: "invoice_no", type: "text" }, + { name: "status", type: "select", required: true, maxSelect: 1, values: ["imported", "failed"] }, + { name: "error", type: "text" }, + // Extracted-but-unverified data, kept so a failed import can still be inspected. + { name: "parsed", type: "json" }, + { name: "source_pdf", type: "file", maxSelect: 1, maxSize: 20971520 }, + { name: "invoice", type: "relation", collectionId: invoicesId, minSelect: 1, maxSelect: 1, cascadeDelete: false }, + // Collections created through the API do NOT get PocketBase's automatic created/updated + // fields the way the UI wizard adds them. Without these, any `sort: "-created"` fails + // server-side with "Something went wrong while processing your request". + { name: "created", type: "date", onCreate: true, presentable: false }, + { name: "updated", type: "date", onUpdate: true, presentable: false }, +]); + await ensureField("monthly_income", { name: "invoice_line", type: "relation",