6ae4e30284
Every PDF import failed in production with "Setting up fake worker failed: Cannot find module '/app/node_modules/pdfjs-dist/legacy/build/pdf.worker.mjs'". output: "standalone" ships only the files Next's output tracer can see. It traced pdf.mjs, because lib/invoice-parser.ts imports it by literal path, but the worker's sole reference is a createRequire(...).resolve() call, which the tracer does not follow. So the image contained pdf.mjs alone, pdfjs fell back to its default "./pdf.worker.mjs" relative to pdf.mjs, and the import hit a file that was never copied. Copy the whole package in the runner stage: the worker has to sit next to pdf.mjs, and so do the standard fonts, which the parser needs because it passes useSystemFonts. Verified against the reconstructed runner layout — the old layout reproduces the error verbatim, the new one parses and extracts text. Also in lib/invoice-parser.ts: workerPinned was set before the resolve, so a failed resolve was never retried; and the empty catch swallowed the MODULE_NOT_FOUND that named the real cause. Both fixed.
432 lines
18 KiB
TypeScript
432 lines
18 KiB
TypeScript
// Deterministic parser for Mega Plant "Std_ReqHireInvoice_DN.rpt" VAT invoices.
|
||
//
|
||
// Deliberately NOT an AI extractor. That supplier's report is a fixed template: the
|
||
// columns sit at identical x-positions on every invoice and only the number of
|
||
// machine lines and pages varies. So the figures are read by coordinate, exactly the
|
||
// way they were transcribed by hand, and then checked arithmetically.
|
||
//
|
||
// Anything that does not reconcile is reported as a reason and lands in the intake
|
||
// inbox as a problem invoice. It never guesses a total and never writes a partial
|
||
// invoice.
|
||
//
|
||
// Calibrated against PHI 166 / 167 / 168 / 169 / 171.
|
||
|
||
export type ParsedLine = {
|
||
machine_code: string;
|
||
description: string;
|
||
qty: number;
|
||
unit: string;
|
||
rate: number;
|
||
rate_type: string;
|
||
amount_excl: number;
|
||
vat: number;
|
||
total_incl: number;
|
||
};
|
||
|
||
export type ParsedInvoice = {
|
||
invoice_no: string;
|
||
invoice_date: string;
|
||
order_no: string;
|
||
debtor: string;
|
||
debtor_no: string;
|
||
period_from: string;
|
||
period_to: string;
|
||
description: string;
|
||
site: string;
|
||
category: string;
|
||
section: string;
|
||
currency: string;
|
||
vat_reg_no: string;
|
||
vat_pct: number;
|
||
subtotal: number;
|
||
escalation: number;
|
||
vat: number;
|
||
total_excl: number;
|
||
total_incl: number;
|
||
lines: ParsedLine[];
|
||
};
|
||
|
||
export class ParseError extends Error {
|
||
constructor(public reason: string) {
|
||
super(reason);
|
||
this.name = "ParseError";
|
||
}
|
||
}
|
||
|
||
// Column left-edges, calibrated from the report template. Numbers are right-aligned
|
||
// within their column, so each value's LEFT edge lands in a stable band.
|
||
const COL = {
|
||
HIRE: [340, 400],
|
||
PENALTY2: [400, 440],
|
||
ESCALATION: [440, 490],
|
||
VAT: [490, 534],
|
||
TOTAL: [534, 1e9],
|
||
} as const;
|
||
|
||
// Activity sub-row ("Dry Rate <qty> <unit> <rate> <amount> 0001").
|
||
const ACT = {
|
||
QTY: [220, 262],
|
||
UNIT: [262, 300],
|
||
RATE: [300, 350],
|
||
AMOUNT: [350, 392],
|
||
} as const;
|
||
|
||
type Word = { x: number; y: number; str: string };
|
||
|
||
const inBand = (x: number, [lo, hi]: readonly [number, number]) => x >= lo && x < hi;
|
||
|
||
// "1,234.50" / "1.234,50" / "1 234,50" / "-" / ""
|
||
function num(s: string | undefined): number | null {
|
||
if (!s) return null;
|
||
const t = s.replace(/\s/g, "").replace(/,/g, "").replace(/ /g, "");
|
||
if (!t || t === "-" || t === "–") return null;
|
||
const n = Number(t);
|
||
return Number.isFinite(n) ? n : null;
|
||
}
|
||
|
||
const r2 = (n: number) => Math.round(n * 100) / 100;
|
||
|
||
/**
|
||
* Splits a machine row into its code and description.
|
||
*
|
||
* The code is a leading alpha block plus a 1-3 digit block, optionally followed by a
|
||
* dash and an upper-case token that must end at whitespace. That rule resolves every
|
||
* recurring case correctly, including the two that look alike but aren't:
|
||
* "MPLANT05-Storage Container" -> MPLANT05 / Storage Container (suffix ends in lowercase)
|
||
* "RV-01-CATCB22 2.2 TON ..." -> RV-01 / CATCB22 2.2 TON ... (suffix runs into a digit)
|
||
* The second yields a shorter code, which is cosmetic — no amount depends on it, and
|
||
* that code does not repeat across invoices, so duplicate detection is unaffected.
|
||
*/
|
||
function splitMachineRow(text: string): { code: string; description: string } {
|
||
const trimmed = text.trim();
|
||
const core = /^([A-Za-z]{2,8}-?[0-9]{2,3})/.exec(trimmed);
|
||
if (!core) {
|
||
const token = trimmed.split(/\s+/)[0] ?? trimmed;
|
||
return { code: token, description: "" };
|
||
}
|
||
const base = core[1];
|
||
let rest = trimmed.slice(base.length);
|
||
// An optional -SUFFIX belongs to the code only when it ends at whitespace. That is
|
||
// what separates the two look-alike cases:
|
||
// "MPLANT05-Storage Container" -> MPLANT05 / Storage Container (suffix ends lowercase)
|
||
// "MG-01-DEZI MG80 ..." -> MG-01-DEZI / MG80 ... (suffix ends at a space)
|
||
// The suffix is always UPPER CASE (codes are) and must be bounded, which is what
|
||
// separates a real code suffix from the start of a description word. Two bounds occur
|
||
// in this report:
|
||
// 1. the suffix ends at whitespace "MG-01-DEZI MG80 Motor Grader"
|
||
// 2. the suffix runs into a digit group that ends at whitespace
|
||
// "RV-01-CATCB22 2.2 TON Roller"
|
||
// Known limit: "RV-01-CATCB22 2.2 TON" has no separator between the code and the
|
||
// description ("CB22" is the machine model), so the split point is a judgement call.
|
||
// The parser takes the maximal upper-case run, giving RV-01-CATCB. This is a display
|
||
// label only — no amount depends on it, and registerCodeFrom() still resolves it to
|
||
// RV01, so booking is unaffected. The hand extraction records RV-01-CAT.
|
||
// Case matters: matching [A-Za-z] here would swallow the "Office" of
|
||
// "MPLANT04-Office Container" and lose the code.
|
||
const sfx =
|
||
/^-([A-Z0-9]{1,6})(?=\s|$)/.exec(rest) ??
|
||
/^-([A-Z]{2,5})(?=[A-Z]*[0-9]+[\s,])/.exec(rest);
|
||
let code = base;
|
||
if (sfx) {
|
||
code = `${base}-${sfx[1]}`;
|
||
rest = rest.slice(sfx[0].length);
|
||
}
|
||
return { code, description: rest.trim().replace(/^[-–]\s*/, "") };
|
||
}
|
||
|
||
// The code an invoice uses maps to a register code by taking the leading letters+digits
|
||
// and dropping punctuation: "SS-01-CASE" -> "SS01", "TP601-TATA" -> "TP601".
|
||
export function registerCodeFrom(invoiceCode: string): string {
|
||
const m = /^([A-Za-z]+)[- ]?([0-9]{1,3})/.exec(invoiceCode.trim());
|
||
return m ? `${m[1].toUpperCase()}${m[2]}` : "";
|
||
}
|
||
|
||
// "Hire Plant Sep 2026 Rooihuiskraal" / "Plant Hire SEP 2026 West Park" -> the site.
|
||
export function siteFromDescription(desc: string): string {
|
||
const m = /^(?:hire\s+plant|plant\s+hire)\s+\S+\s+\d{4}\s+(.+)$/i.exec(desc.trim());
|
||
return m ? m[1].trim() : "";
|
||
}
|
||
|
||
const MONTHS: Record<string, string> = {
|
||
jan: "01", feb: "02", mar: "03", apr: "04", may: "05", jun: "06",
|
||
jul: "07", aug: "08", sep: "09", oct: "10", nov: "11", dec: "12",
|
||
};
|
||
|
||
/** "18-September-2026" / "18/09/2026" -> "2026-09-18" */
|
||
export function isoDate(s: string): string {
|
||
const t = s.trim();
|
||
let m = /^(\d{1,2})[/-]([A-Za-z]{3,})[/-](\d{4})$/.exec(t);
|
||
if (m) {
|
||
const mo = MONTHS[m[2].slice(0, 3).toLowerCase()];
|
||
if (!mo) throw new ParseError(`Unrecognised month "${m[2]}" in date "${s}"`);
|
||
return `${m[3]}-${mo}-${m[1].padStart(2, "0")}`;
|
||
}
|
||
m = /^(\d{1,2})\/(\d{2})\/(\d{4})$/.exec(t);
|
||
if (m) return `${m[3]}-${m[2]}-${m[1].padStart(2, "0")}`;
|
||
m = /^(\d{4})-(\d{2})-(\d{2})$/.exec(t);
|
||
if (m) return t;
|
||
throw new ParseError(`Unrecognised date format "${s}"`);
|
||
}
|
||
|
||
// pdfjs resolves its worker lazily at runtime. Left to its own devices under Next it
|
||
// falls back to a dynamic import of the worker module, which the bundler rewrites to a
|
||
// .next path that does not exist in the standalone image — every parse then dies with
|
||
// "Setting up fake worker failed". Pinning the worker to the real installed file avoids
|
||
// that, but only if the file is actually there: pdfjs-dist is in
|
||
// serverExternalPackages so it is required from node_modules rather than bundled, and
|
||
// Next's tracer does NOT copy pdf.worker.mjs — the standalone bundle contains pdf.mjs
|
||
// alone. The Dockerfile copies the whole package in for that reason. Do not assume this
|
||
// pin alone is sufficient.
|
||
let workerPinned = false;
|
||
async function loadPdfjs() {
|
||
const pdfjs = await import("pdfjs-dist/legacy/build/pdf.mjs");
|
||
if (!workerPinned) {
|
||
try {
|
||
const { createRequire } = await import("node:module");
|
||
const req = createRequire(import.meta.url);
|
||
pdfjs.GlobalWorkerOptions.workerSrc = req.resolve(
|
||
"pdfjs-dist/legacy/build/pdf.worker.mjs",
|
||
);
|
||
workerPinned = true;
|
||
} catch (err) {
|
||
// Not fatal, but it must not be silent: pdfjs then falls back to a worker
|
||
// path relative to pdf.mjs, so the parse fails much later with an opaque
|
||
// "Setting up fake worker failed". Staying UNpinned lets the next parse
|
||
// retry, so the worker is picked up as soon as it is on disk.
|
||
console.error(
|
||
`[invoice-parser] could not resolve pdf.worker.mjs: ${(err as Error).message}`,
|
||
);
|
||
}
|
||
}
|
||
return pdfjs;
|
||
}
|
||
|
||
export async function parseInvoicePdf(bytes: Uint8Array): Promise<ParsedInvoice> {
|
||
const pdfjs = await loadPdfjs();
|
||
// pdfjs TRANSFERS the buffer it is handed, which detaches it. Callers keep their own
|
||
// copy (to store the original file), so parse a duplicate rather than the original.
|
||
const doc = await pdfjs.getDocument({
|
||
data: new Uint8Array(bytes),
|
||
useSystemFonts: true,
|
||
}).promise;
|
||
|
||
const pages: Word[][] = [];
|
||
try {
|
||
for (let p = 1; p <= doc.numPages; p++) {
|
||
const page = await doc.getPage(p);
|
||
const vp = page.getViewport({ scale: 1 });
|
||
const content = await page.getTextContent();
|
||
const words: Word[] = [];
|
||
for (const it of content.items as any[]) {
|
||
if (typeof it.str !== "string" || !it.str.trim()) continue;
|
||
words.push({ x: it.transform[4], y: vp.height - it.transform[5], str: it.str });
|
||
}
|
||
pages.push(words);
|
||
}
|
||
|
||
// ---- header (page 1) -------------------------------------------------
|
||
const h1 = pages[0] ?? [];
|
||
const onRow = (words: Word[], label: string) => words.filter((w) => w.str.startsWith(label));
|
||
|
||
const after = (label: string, page = 0): string => {
|
||
const ws = onRow(pages[page] ?? [], label);
|
||
if (!ws.length) return "";
|
||
const y = ws[0].y;
|
||
const same = (pages[page] ?? []).filter((w) => Math.abs(w.y - y) < 3 && w.x > ws[0].x + ws[0].str.length);
|
||
return same.map((w) => w.str).join(" ").trim();
|
||
};
|
||
|
||
const invoiceNo = after("Invoice") || "";
|
||
if (!invoiceNo) throw new ParseError('Could not find "Invoice No." in the header');
|
||
const orderNo = after("ORDER NUMBER") || "";
|
||
const invoiceDateRaw = after("DATE") || "";
|
||
if (!invoiceDateRaw) throw new ParseError('Could not find "DATE" in the header');
|
||
const debtorNo = after("DEBTOR NUMBER") || "";
|
||
const currency = after("CURRENCY") || "ZAR";
|
||
const vatReg = after("VAT Reg. No.") || "";
|
||
|
||
// Debtor name sits at the far left of the block above ORDER NUMBER.
|
||
const debtorWord = h1.find((w) => w.x < 100 && /^Mega |Infrastructure/i.test(w.str) && w.y > 180 && w.y < 220);
|
||
const debtor = debtorWord ? debtorWord.str.replace(/^Mega\s+/i, "Mega ") : "";
|
||
|
||
// "Description :" and the free text beside it.
|
||
const descRow = h1.filter((w) => Math.abs(w.y - (h1.find((x) => x.str === "Description")?.y ?? -999)) < 3);
|
||
const descWord = descRow.find((w) => w.str === "Description");
|
||
const description = descWord
|
||
? descRow
|
||
.filter(
|
||
(w) =>
|
||
w.x > descWord.x + 12 &&
|
||
w.x < 460 &&
|
||
!w.str.endsWith(":") &&
|
||
!/^\d{1,2}\/\d{2}\/\d{4}$/.test(w.str) &&
|
||
!/Date$/.test(w.str),
|
||
)
|
||
.sort((a, b) => a.x - b.x)
|
||
.map((w) => w.str)
|
||
.join(" ")
|
||
.trim()
|
||
: "";
|
||
|
||
const dates = h1
|
||
.filter((w) => /^\d{1,2}\/\d{2}\/\d{4}$/.test(w.str))
|
||
.sort((a, b) => a.y - b.y || a.x - b.x)
|
||
.map((w) => w.str);
|
||
if (dates.length < 2) throw new ParseError("Could not read the From/To dates from the header");
|
||
const period_from = isoDate(dates[0]);
|
||
const period_to = isoDate(dates[1]);
|
||
const invoice_date = isoDate(invoiceDateRaw);
|
||
|
||
// ---- machine lines (every page) --------------------------------------
|
||
const lines: ParsedLine[] = [];
|
||
for (const words of pages) {
|
||
// A machine row is a left-column item that is not a known header/footer word.
|
||
const machineRows = words.filter(
|
||
(w) => w.x < 45 && /[A-Za-z]/.test(w.str) && !/^(Return|Sep|Dry|Rib|Ref|Printed|Description)$/i.test(w.str) &&
|
||
!/\bNo\.$/.test(w.str) && /[A-Za-z]{2,8}-?[0-9]{2,3}/.test(w.str),
|
||
);
|
||
for (const mr of machineRows) {
|
||
// Numeric row sits ~16pt below the machine name.
|
||
const numRow = words.filter((w) => w.y > mr.y + 10 && w.y < mr.y + 24);
|
||
const hire = num(numRow.find((w) => inBand(w.x, COL.HIRE))?.str);
|
||
const vat = num(numRow.find((w) => inBand(w.x, COL.VAT))?.str);
|
||
const total = num(numRow.find((w) => inBand(w.x, COL.TOTAL))?.str);
|
||
if (hire == null || vat == null || total == null) continue;
|
||
|
||
// Activity row: the "Dry Rate ..." line, roughly 57pt below the machine name.
|
||
// The "Description Qty Unit ..." header sits at ~44pt and must not be mistaken
|
||
// for it, so locate the row by its leading "Dry" token rather than by offset.
|
||
const band = words.filter((w) => w.y > mr.y + 38 && w.y < mr.y + 78 && w.x > 150);
|
||
const dryY = band.find((w) => w.str === "Dry" || w.str.startsWith("Dry "))?.y;
|
||
const dry = dryY == null ? [] : band.filter((w) => Math.abs(w.y - dryY) < 3).sort((a, b) => a.x - b.x);
|
||
const qty = num(dry.find((w) => inBand(w.x, ACT.QTY))?.str);
|
||
const unit = dry.find((w) => inBand(w.x, ACT.UNIT))?.str ?? "";
|
||
const rate = num(dry.find((w) => inBand(w.x, ACT.RATE))?.str);
|
||
// pdfjs emits "Dry Rate" as a single token; older builds split it in two.
|
||
const rateType = dry[0] ? dry[0].str.replace(/\s+/g, " ") : "Dry Rate";
|
||
|
||
const { code, description: desc } = splitMachineRow(mr.str);
|
||
lines.push({
|
||
machine_code: code,
|
||
description: desc,
|
||
qty: qty ?? 0,
|
||
unit,
|
||
rate: rate ?? 0,
|
||
rate_type: rateType,
|
||
amount_excl: r2(hire),
|
||
vat: r2(vat),
|
||
total_incl: r2(total),
|
||
});
|
||
}
|
||
}
|
||
if (!lines.length) throw new ParseError("No machine lines were found on any page");
|
||
|
||
// ---- totals -----------------------------------------------------------
|
||
// The report emits the grand-total label as ONE text item ("Plant Hire Total"),
|
||
// so match on prefix rather than equality.
|
||
let hireTotal: number | null = null;
|
||
let taxTotal: number | null = null;
|
||
let grandTotal: number | null = null;
|
||
let escalationTotal = 0;
|
||
for (const words of pages) {
|
||
const row = words.find((w) => w.str.startsWith("Plant Hire Total"));
|
||
if (!row) continue;
|
||
const same = words.filter((w) => Math.abs(w.y - row.y) < 4 && w.x > row.x + row.str.length);
|
||
hireTotal = num(same.find((w) => inBand(w.x, COL.HIRE))?.str);
|
||
escalationTotal = num(same.find((w) => inBand(w.x, COL.ESCALATION))?.str) ?? 0;
|
||
taxTotal = num(same.find((w) => inBand(w.x, COL.VAT))?.str);
|
||
grandTotal = num(same.find((w) => inBand(w.x, COL.TOTAL))?.str);
|
||
}
|
||
if (hireTotal == null || taxTotal == null || grandTotal == null) {
|
||
throw new ParseError('Could not find the "Plant Hire Total" row');
|
||
}
|
||
|
||
// The summary block is read BY LABEL — its values sit at different x positions
|
||
// from the line table, so column bands cannot be used here. pdfjs splits these
|
||
// labels inconsistently between pages, so each is matched by prefix.
|
||
const labelValue = (words: Word[], label: string): number | null => {
|
||
const lw = words.find((w) => w.str === label || w.str.startsWith(`${label} `));
|
||
if (!lw) return null;
|
||
const labelEnd = lw.x + lw.str.length;
|
||
const same = words.filter((w) => Math.abs(w.y - lw.y) < 5 && w.x > labelEnd + 15);
|
||
return num(same.map((w) => w.str).join(""));
|
||
};
|
||
|
||
const summary: Record<string, number> = {};
|
||
for (const words of pages) {
|
||
const put = (k: string, v: number | null) => { if (v != null) summary[k] = v; };
|
||
put("SUB TOTAL", labelValue(words, "SUB TOTAL"));
|
||
put("Escalation", labelValue(words, "Escalation"));
|
||
put("PLUS", labelValue(words, "PLUS"));
|
||
put("INVOICE", labelValue(words, "INVOICE"));
|
||
}
|
||
|
||
const subtotal = summary["SUB TOTAL"] ?? hireTotal;
|
||
const escalation = summary["Escalation"] ?? escalationTotal;
|
||
const vat = summary["PLUS"] ?? taxTotal;
|
||
const total_excl = r2(subtotal + escalation);
|
||
const total_incl = summary["INVOICE"] ?? grandTotal;
|
||
|
||
// ---- validation — the same gates the seed applies ---------------------
|
||
const sumHire = r2(lines.reduce((s, l) => s + l.amount_excl, 0));
|
||
if (Math.abs(sumHire - subtotal) > 0.01) {
|
||
throw new ParseError(`Machine lines total R${sumHire} but the invoice subtotal is R${subtotal}`);
|
||
}
|
||
const sumVat = r2(lines.reduce((s, l) => s + l.vat, 0));
|
||
if (Math.abs(sumVat - vat) > 0.02) {
|
||
throw new ParseError(`Line VAT totals R${sumVat} but the invoice PLUS VAT is R${vat}`);
|
||
}
|
||
if (Math.abs(total_excl + vat - total_incl) > 0.01) {
|
||
throw new ParseError(
|
||
`Totals do not balance: R${total_excl} ex + R${vat} VAT != R${total_incl} invoice total`,
|
||
);
|
||
}
|
||
for (const l of lines) {
|
||
if (Math.abs(l.amount_excl + l.vat - l.total_incl) > 0.01) {
|
||
throw new ParseError(
|
||
`Line ${l.machine_code}: R${l.amount_excl} + R${l.vat} VAT != R${l.total_incl}`,
|
||
);
|
||
}
|
||
if (l.qty && l.rate && Math.abs(r2(l.qty * l.rate) - l.amount_excl) > 0.01) {
|
||
throw new ParseError(
|
||
`Line ${l.machine_code}: ${l.qty} x ${l.rate} = ${r2(l.qty * l.rate)}, but the line amount is R${l.amount_excl}`,
|
||
);
|
||
}
|
||
}
|
||
|
||
const vat_pct = total_excl > 0 ? Math.round((vat / total_excl) * 1000) / 10 : 0;
|
||
for (const l of lines) {
|
||
const expected = r2((l.amount_excl * vat_pct) / 100);
|
||
if (l.amount_excl > 0 && Math.abs(expected - l.vat) > 0.02) {
|
||
throw new ParseError(
|
||
`Line ${l.machine_code}: VAT is R${l.vat}, but ${vat_pct}% of R${l.amount_excl} is R${expected}`,
|
||
);
|
||
}
|
||
}
|
||
|
||
return {
|
||
invoice_no: invoiceNo.trim().replace(/\s+/g, " "),
|
||
invoice_date,
|
||
order_no: orderNo,
|
||
debtor,
|
||
debtor_no: debtorNo,
|
||
period_from,
|
||
period_to,
|
||
description,
|
||
site: siteFromDescription(description),
|
||
section: "Plant Hire",
|
||
category: "External Plant Hire",
|
||
currency,
|
||
vat_reg_no: vatReg,
|
||
vat_pct,
|
||
subtotal: r2(subtotal),
|
||
escalation: r2(escalation),
|
||
vat: r2(vat),
|
||
total_excl,
|
||
total_incl: r2(total_incl),
|
||
lines,
|
||
};
|
||
} finally {
|
||
await doc.destroy().catch(() => {});
|
||
}
|
||
}
|