ab89113489
PHI 180 prints "Westpark" where the register says "West Park", so a spacing-insensitive fold looks like an obvious typo fix. Tried it, and it is wrong: the register ALSO carries both "Autumns Gate" and "Autumnsgate", and no rule can tell a missing space from two genuinely different places — it merged that pair. Reverted lib/sites.ts to its previous conservative behaviour and pinned the decision with a test. PHI 180's stored name was corrected to "West Park" by hand instead. Merging the two Autumn sites is the client's call, not a side effect of a re-import.
296 lines
16 KiB
TypeScript
296 lines
16 KiB
TypeScript
// Deterministic-parser tests.
|
|
//
|
|
// The parser reads figures by coordinate from a fixed report template, so the risk is
|
|
// silent drift: a column moves and it starts returning a plausible wrong number. These
|
|
// tests pin the behaviour that matters — the column mapping, the code/description split,
|
|
// date formats, and the fact that anything which does not reconcile is refused.
|
|
//
|
|
// Run: node --import tsx --test lib/invoice-parser.test.ts
|
|
import { test } from "node:test";
|
|
import assert from "node:assert/strict";
|
|
import { existsSync, readFileSync } from "node:fs";
|
|
import { resolve } from "node:path";
|
|
import { parseInvoicePdf, registerCodeFrom, siteFromDescription, isoDate, extractLines, ParseError } from "./invoice-parser";
|
|
|
|
// --- unit level: the pieces that do not need a PDF -------------------------------
|
|
|
|
test("registerCodeFrom strips punctuation and suffix from an invoice code", () => {
|
|
assert.equal(registerCodeFrom("SS-01-CASE"), "SS01");
|
|
assert.equal(registerCodeFrom("TP601-TATA"), "TP601");
|
|
assert.equal(registerCodeFrom("MG-01-DEZI"), "MG01");
|
|
assert.equal(registerCodeFrom("RV-03-CAT"), "RV03");
|
|
assert.equal(registerCodeFrom("TLB07-CASE"), "TLB07");
|
|
// No register equivalent — must NOT be silently coerced into something plausible.
|
|
assert.equal(registerCodeFrom("MPLANT05"), "MPLANT05");
|
|
});
|
|
|
|
test("site is read from the invoice description", () => {
|
|
assert.equal(siteFromDescription("Plant Hire Sep 2026 West Park"), "West Park");
|
|
assert.equal(siteFromDescription("Hire Plant Sep 2026 Halfway Gardens"), "Halfway Gardens");
|
|
assert.equal(siteFromDescription("Hire Plant Sep 2026 Rooihuiskraal"), "Rooihuiskraal");
|
|
assert.equal(siteFromDescription("Plant Hire SEp 2026 Cosmo City"), "Cosmo City");
|
|
assert.equal(siteFromDescription("something else entirely"), "");
|
|
// The wording BEFORE the date is not fixed: PHI 179 leads with the machine type.
|
|
// Anchoring on "Hire Plant"/"Plant Hire" returned "" and left the invoice unlinked.
|
|
assert.equal(siteFromDescription("Crane Truck Hire Sep 2026 Cosmo"), "Cosmo");
|
|
assert.equal(siteFromDescription("Boom Lift Hire October 2026 The Hills"), "The Hills");
|
|
});
|
|
|
|
test("dates accept the report's written formats", () => {
|
|
assert.equal(isoDate("18-September-2026"), "2026-09-18");
|
|
assert.equal(isoDate("21/08/2026"), "2026-08-21");
|
|
assert.throws(() => isoDate("not a date"), ParseError);
|
|
});
|
|
|
|
// --- the line table: one line per CHARGE -------------------------------------
|
|
//
|
|
// PHI 179 is a single CT01 crane truck hired on 4 Sep, 16 Sep and 18 Sep. The report
|
|
// prints one machine heading followed by a repeating block per charge. The parser used
|
|
// to read exactly one block per machine, so it returned R5 265 against a R13 730
|
|
// subtotal and the invoice was refused — two thirds of a real invoice went missing
|
|
// because it had been hired over more than one period.
|
|
//
|
|
// These are the REAL word positions from that PDF, including the detail that pdfjs
|
|
// splits one printed row across two y positions (the TOTAL sits ~2pt above the rest).
|
|
|
|
const w = (x: number, y: number, str: string) => ({ x, y, str });
|
|
|
|
const PHI179_LINES = [
|
|
w(29.6, 441, "CT01-Crane Truck ISUZU FREIGHTER 1800"),
|
|
|
|
// charge 1 — 4 Sep, dry rate
|
|
w(29.6, 457, "SEP"), w(68.7, 457, "04-Sep-2026"), w(119.4, 457, "-"), w(546.8, 457, "6,054.75"),
|
|
w(142.2, 458, "04-Sep-2026"),
|
|
w(211.5, 459, "1.00"), w(255.6, 459, "-"), w(278.5, 459, "8.00"), w(306.7, 459, "100.00"),
|
|
w(357.8, 459, "5,265.00"), w(426.6, 459, "-"), w(459.0, 459, "0.00"), w(500.2, 459, "789.75"),
|
|
w(313.7, 485, "Rate"), w(359.3, 485, "Amount"),
|
|
w(159.8, 486, "Description"), w(234.4, 486, "Qty"), w(270.4, 486, "Unit"), w(395.5, 486, "Activity"),
|
|
w(159.8, 498, "Dry Rate"), w(233.9, 498, "1.00"), w(273.1, 498, "Day"),
|
|
w(300.8, 498, "5,265.00"), w(357.8, 498, "5,265.00"), w(395.5, 498, "0001"),
|
|
|
|
// charge 2 — 16 Sep, idle time
|
|
w(29.6, 511, "SEP"), w(68.6, 511, "16-Sep-2026"), w(119.3, 511, "-"), w(546.8, 511, "3,680.00"),
|
|
w(142.1, 512, "16-Sep-2026"),
|
|
w(211.5, 513, "1.00"), w(255.6, 513, "-"), w(278.5, 513, "8.00"), w(306.7, 513, "100.00"),
|
|
w(357.8, 513, "3,200.00"), w(426.6, 513, "-"), w(459.0, 513, "0.00"), w(500.2, 513, "480.00"),
|
|
w(159.8, 540, "Description"), w(234.4, 540, "Qty"), w(270.4, 540, "Unit"),
|
|
w(313.7, 540, "Rate"), w(359.3, 540, "Amount"), w(395.5, 540, "Activity"),
|
|
w(159.8, 553, "Idle Time"), w(233.9, 553, "1.00"), w(273.1, 553, "Day"),
|
|
w(300.8, 553, "3,200.00"), w(357.8, 553, "3,200.00"), w(395.5, 553, "0001"),
|
|
|
|
// charge 3 — 18 Sep, dry rate
|
|
w(29.6, 566, "SEP"), w(68.6, 566, "18-Sep-2026"), w(119.3, 566, "-"), w(546.8, 566, "6,054.75"),
|
|
w(142.1, 567, "18-Sep-2026"),
|
|
w(211.5, 567, "1.00"), w(255.6, 567, "-"), w(278.5, 567, "8.00"), w(306.7, 567, "100.00"),
|
|
w(357.8, 567, "5,265.00"), w(426.6, 567, "-"), w(459.0, 567, "0.00"), w(500.2, 567, "789.75"),
|
|
w(313.7, 594, "Rate"), w(359.3, 594, "Amount"),
|
|
w(159.8, 595, "Description"), w(234.4, 595, "Qty"), w(270.4, 595, "Unit"), w(395.5, 595, "Activity"),
|
|
w(159.8, 607, "Dry Rate"), w(233.9, 607, "1.00"), w(273.1, 607, "Day"),
|
|
w(300.8, 607, "5,265.00"), w(357.8, 607, "5,265.00"), w(395.5, 607, "0001"),
|
|
|
|
// the grand total — it is a money row too, and must not become a fourth line
|
|
w(245.5, 628, "Plant Hire Total"),
|
|
w(354.9, 628, "13,730.00"), w(420.7, 628, "0.00"), w(462.0, 628, "0.00"),
|
|
w(494.9, 628, "2,059.50"), w(543.9, 628, "15,789.50"),
|
|
];
|
|
|
|
test("a machine hired over several periods yields one line per charge", () => {
|
|
const lines = extractLines(PHI179_LINES);
|
|
assert.equal(lines.length, 3, "three charges, so three lines — not one per machine");
|
|
assert.deepEqual(lines.map((l) => l.amount_excl), [5265, 3200, 5265]);
|
|
assert.deepEqual(lines.map((l) => l.vat), [789.75, 480, 789.75]);
|
|
assert.deepEqual(lines.map((l) => l.total_incl), [6054.75, 3680, 6054.75]);
|
|
// All three belong to the same plant item, and each must reconcile on its own.
|
|
assert.deepEqual(lines.map((l) => l.machine_code), ["CT01", "CT01", "CT01"]);
|
|
for (const l of lines) {
|
|
assert.equal(l.qty * l.rate, l.amount_excl, `${l.rate_type}: qty x rate != amount`);
|
|
assert.ok(Math.abs(l.amount_excl + l.vat - l.total_incl) < 0.01, "line does not balance");
|
|
}
|
|
});
|
|
|
|
test("the grand total row is never emitted as a line", () => {
|
|
// It carries a figure in HIRE, VAT and TOTAL exactly like a charge does, so only its
|
|
// position after the machine block keeps it out of the invoice.
|
|
const withTotal = PHI179_LINES.filter(
|
|
(x) => !["Plant Hire Total", "13,730.00", "2,059.50", "15,789.50"].includes(x.str),
|
|
);
|
|
assert.equal(extractLines(withTotal).length, 3);
|
|
assert.equal(extractLines(PHI179_LINES).length, 3);
|
|
});
|
|
|
|
test("a machine billed once still yields exactly one line", () => {
|
|
// The single-period invoices (PHI 166-171) all have one charge per machine, and that
|
|
// behaviour must not change.
|
|
const single = PHI179_LINES.filter((x) => x.y >= 441 && x.y <= 498);
|
|
const lines = extractLines(single);
|
|
assert.equal(lines.length, 1);
|
|
assert.equal(lines[0].amount_excl, 5265);
|
|
assert.equal(lines[0].rate_type, "Dry Rate");
|
|
});
|
|
|
|
test("the activity header is not mistaken for the activity row", () => {
|
|
// "Description Qty Unit Rate Amount" has no NUMBER in the Qty column; the rate type
|
|
// comes from the row below it, so a wrong pick yields an empty label, not a wrong one.
|
|
const [first] = extractLines(PHI179_LINES);
|
|
assert.equal(first.rate_type, "Dry Rate");
|
|
assert.equal(first.unit, "Day");
|
|
});
|
|
|
|
test("a PDF with no recognisable content is refused, not guessed", async () => {
|
|
const notAPdf = new TextEncoder().encode("%PDF-1.4\nthis is not a real invoice\n");
|
|
await assert.rejects(() => parseInvoicePdf(notAPdf), /PDF|Could not/i);
|
|
});
|
|
|
|
// --- integration level: the real invoices -----------------------------------------
|
|
//
|
|
// The source PDFs are not committed (they are client documents), so these skip when
|
|
// they are absent. Point FIXTURE_DIR at a folder holding PHI*.pdf to run them.
|
|
|
|
const DIR = process.env.FIXTURE_DIR ?? "/tmp";
|
|
const REAL: [string, string][] = [
|
|
["PHI166.pdf", "invoice-phi166.json"],
|
|
["PHI167.pdf", "invoice-phi167.json"],
|
|
["PHI168.pdf", "invoice-phi168.json"],
|
|
["PHI169.pdf", "invoice-phi169.json"],
|
|
["PHI171.pdf", "invoice-phi171.json"],
|
|
];
|
|
|
|
const available = REAL.filter(([pdf]) => existsSync(resolve(DIR, pdf)));
|
|
|
|
test("parser reproduces the hand-extracted invoices exactly", { skip: available.length ? false : "no source PDFs present" }, async () => {
|
|
const failures: string[] = [];
|
|
for (const [pdfName, jsonName] of available) {
|
|
const src = JSON.parse(readFileSync(resolve("data/source", jsonName), "utf8"));
|
|
const got = await parseInvoicePdf(new Uint8Array(readFileSync(resolve(DIR, pdfName))));
|
|
|
|
const cmp = (label: string, a: unknown, b: unknown) => {
|
|
const ok = typeof a === "number" && typeof b === "number"
|
|
? Math.abs(a - b) < 0.01
|
|
: String(a ?? "") === String(b ?? "");
|
|
if (!ok) failures.push(`${jsonName} ${label}: parser=${a} source=${b}`);
|
|
};
|
|
|
|
cmp("invoice_no", got.invoice_no, src.invoice_no);
|
|
cmp("invoice_date", got.invoice_date, src.invoice_date);
|
|
cmp("order_no", got.order_no, src.order_no);
|
|
cmp("debtor", got.debtor, src.debtor);
|
|
cmp("debtor_no", got.debtor_no, src.debtor_no);
|
|
cmp("period_from", got.period_from, src.period_from);
|
|
cmp("period_to", got.period_to, src.period_to);
|
|
cmp("description", got.description, src.description);
|
|
cmp("site", got.site, src.site);
|
|
cmp("currency", got.currency, src.currency);
|
|
cmp("vat_pct", got.vat_pct, src.vat_pct);
|
|
cmp("subtotal", got.subtotal, src.subtotal);
|
|
cmp("escalation", got.escalation, src.escalation);
|
|
cmp("vat", got.vat, src.vat);
|
|
cmp("total_excl", got.total_excl, src.total_excl);
|
|
cmp("total_incl", got.total_incl, src.total_incl);
|
|
|
|
if (got.lines.length !== src.lines.length) {
|
|
failures.push(`${jsonName} line count: ${got.lines.length} vs ${src.lines.length}`);
|
|
continue;
|
|
}
|
|
got.lines.forEach((l, i) => {
|
|
const s = src.lines[i];
|
|
cmp(`line${i + 1}.amount_excl`, l.amount_excl, s.amount_excl);
|
|
cmp(`line${i + 1}.vat`, l.vat, s.vat);
|
|
cmp(`line${i + 1}.total_incl`, l.total_incl, s.total_incl);
|
|
cmp(`line${i + 1}.qty`, l.qty, s.qty);
|
|
cmp(`line${i + 1}.unit`, l.unit, s.unit);
|
|
cmp(`line${i + 1}.rate`, l.rate, s.rate);
|
|
// machine_code is compared through registerCodeFrom: one input ("RV-01-CATCB22")
|
|
// has no separator between code and description, so the parser's split differs
|
|
// cosmetically from the hand extraction. What must match is the plant item the
|
|
// line books to, which is derived from exactly this value.
|
|
cmp(`line${i + 1}.register code`, registerCodeFrom(l.machine_code), registerCodeFrom(s.machine_code));
|
|
});
|
|
}
|
|
assert.deepEqual(failures, [], `parser disagreed with the hand extraction:\n${failures.join("\n")}`);
|
|
});
|
|
|
|
test("the parser never returns an invoice whose totals do not reconcile", { skip: available.length ? false : "no source PDFs present" }, async () => {
|
|
// Every successful parse must satisfy the balance rules on its own — this is the
|
|
// guarantee the intake route relies on before it is allowed to write anything.
|
|
for (const [pdfName] of available) {
|
|
const got = await parseInvoicePdf(new Uint8Array(readFileSync(resolve(DIR, pdfName))));
|
|
const sumHire = Math.round(got.lines.reduce((s, l) => s + l.amount_excl, 0) * 100) / 100;
|
|
const sumVat = Math.round(got.lines.reduce((s, l) => s + l.vat, 0) * 100) / 100;
|
|
assert.ok(Math.abs(sumHire - got.subtotal) < 0.01, `${pdfName}: lines != subtotal`);
|
|
assert.ok(Math.abs(sumVat - got.vat) < 0.02, `${pdfName}: line VAT != header VAT`);
|
|
assert.ok(Math.abs(got.total_excl + got.vat - got.total_incl) < 0.01, `${pdfName}: totals do not balance`);
|
|
for (const l of got.lines) {
|
|
assert.ok(Math.abs(l.amount_excl + l.vat - l.total_incl) < 0.01, `${pdfName}: ${l.machine_code} does not balance`);
|
|
assert.ok(Math.abs(l.qty * l.rate - l.amount_excl) < 0.01, `${pdfName}: ${l.machine_code} qty x rate != amount`);
|
|
}
|
|
}
|
|
});
|
|
// --- container auto-allocation --------------------------------------------------
|
|
// A container line prints a generic code ("MPLANT05") that the plant register does not
|
|
// carry, so it has no natural asset. Each site records the container it hires, and the
|
|
// code the invoice uses for it. These tests pin the lookup — done here rather than by
|
|
// deleting a real invoice and re-importing it.
|
|
|
|
import { findSiteContainer, resolveSiteName } from "./sites";
|
|
|
|
const site = (name: string, code: string, asset: string) =>
|
|
({ id: name, name, container_code: code, container_asset: asset }) as any;
|
|
|
|
test("a container line is allocated to the container the site hires", () => {
|
|
const sites = [site("Autumns Gate", "MPLANT05", "a-sc02")];
|
|
assert.equal(findSiteContainer(sites, "Autumns Gate", "MPLANT05")?.container_asset, "a-sc02");
|
|
});
|
|
|
|
test("allocation is scoped to the invoice's own site", () => {
|
|
// Two sites can each hire an "MPLANT05"; a line must go to its own site's unit.
|
|
const sites = [site("Autumns Gate", "MPLANT05", "a-sc02"), site("The Hills", "MPLANT05", "h-sc03")];
|
|
assert.equal(findSiteContainer(sites, "The Hills", "MPLANT05")?.container_asset, "h-sc03");
|
|
assert.equal(findSiteContainer(sites, "Autumns Gate", "MPLANT05")?.container_asset, "a-sc02");
|
|
});
|
|
|
|
test("no allocation when the site has no matching container", () => {
|
|
const sites = [site("Pitsana", "WT02", "")];
|
|
assert.equal(findSiteContainer(sites, "Pitsana", "MPLANT05"), null, "code does not match");
|
|
assert.equal(findSiteContainer(sites, "Pitsana", "WT02"), null, "no asset assigned");
|
|
assert.equal(findSiteContainer(sites, "Elsewhere", "WT02"), null, "different site");
|
|
assert.equal(findSiteContainer([], "Pitsana", "WT02"), null, "no sites at all");
|
|
});
|
|
|
|
test("an abbreviated site folds onto the name the register already uses", () => {
|
|
// PHI 179 prints "Crane Truck Hire Sep 2026 Cosmo" for the site every other invoice
|
|
// calls "Cosmo City". Sites are derived from the invoice wording, so taking it
|
|
// literally would create a SECOND "Cosmo" and split the customer across two rows.
|
|
const known = ["West Park", "Halfway Gardens", "Cosmo City"];
|
|
assert.equal(resolveSiteName("Cosmo", known), "Cosmo City");
|
|
assert.equal(resolveSiteName("Cosmo City", known), "Cosmo City");
|
|
assert.equal(resolveSiteName("cosmo city", known), "Cosmo City");
|
|
assert.equal(resolveSiteName("West Park", known), "West Park");
|
|
});
|
|
|
|
test("folding never invents a site or splits a different one", () => {
|
|
const known = ["Cosmo City", "West Park"];
|
|
// No match at all — a genuinely new site keeps the invoice's own wording.
|
|
assert.equal(resolveSiteName("Rooihuiskraal", known), "Rooihuiskraal");
|
|
assert.equal(resolveSiteName("", known), "");
|
|
assert.equal(resolveSiteName("Cosmo", []), "Cosmo");
|
|
// A partial word must not match — "Cos" is not a shorter name for "Cosmo City".
|
|
assert.equal(resolveSiteName("Cos", known), "Cos");
|
|
// Folding runs one way only. An invoice that names a LONGER place is not an
|
|
// abbreviation of the shorter one, and merging the two would lose a real site.
|
|
assert.equal(resolveSiteName("Cosmo City Extension", known), "Cosmo City Extension");
|
|
assert.equal(resolveSiteName("Cosmo City North", known), "Cosmo City North");
|
|
});
|
|
|
|
test("a site typed without its space is NOT folded — that would merge two real sites", () => {
|
|
// PHI 180 prints "Westpark" where the register says "West Park", so this looks like a
|
|
// spacing typo worth folding. It is deliberately left alone: the register ALSO carries
|
|
// both "Autumns Gate" and "Autumnsgate", and a spacing-insensitive rule cannot tell a
|
|
// typo from two genuinely different places — it would silently merge that pair.
|
|
//
|
|
// PHI 180's stored name was corrected to "West Park" by hand instead. Merging the two
|
|
// Autumn sites is the client's call, not a side effect of a re-import.
|
|
const known = ["West Park", "Autumns Gate", "Autumnsgate"];
|
|
assert.equal(resolveSiteName("Westpark", known), "Westpark");
|
|
assert.equal(resolveSiteName("Autumnsgate", known), "Autumnsgate");
|
|
assert.equal(resolveSiteName("Autumns Gate", known), "Autumns Gate");
|
|
});
|