mega-plant/lib/invoice-parser.ts

// Deterministic parser for Mega Plant "Std_ReqHireInvoice_DN.rpt" VAT invoices.
//
// Deliberately NOT an AI extractor. That supplier's report is a fixed template: the
// columns sit at identical x-positions on every invoice and only the number of
// machine lines and pages varies. So the figures are read by coordinate, exactly the
// way they were transcribed by hand, and then checked arithmetically.
//
// Anything that does not reconcile is reported as a reason and lands in the intake
// inbox as a problem invoice. It never guesses a total and never writes a partial
// invoice.
//
// Calibrated against PHI 166 / 167 / 168 / 169 / 171.

export type ParsedLine = {
  machine_code: string;
  description: string;
  qty: number;
  unit: string;
  rate: number;
  rate_type: string;
  amount_excl: number;
  vat: number;
  total_incl: number;
};

export type ParsedInvoice = {
  invoice_no: string;
  invoice_date: string;
  order_no: string;
  debtor: string;
  debtor_no: string;
  period_from: string;
  period_to: string;
  description: string;
  site: string;
  category: string;
  section: string;
  currency: string;
  vat_reg_no: string;
  vat_pct: number;
  subtotal: number;
  escalation: number;
  vat: number;
  total_excl: number;
  total_incl: number;
  lines: ParsedLine[];
};

export class ParseError extends Error {
  constructor(public reason: string) {
    super(reason);
    this.name = "ParseError";
  }
}

// Column left-edges, calibrated from the report template. Numbers are right-aligned
// within their column, so each value's LEFT edge lands in a stable band.
const COL = {
  HIRE: [340, 400],
  PENALTY2: [400, 440],
  ESCALATION: [440, 490],
  VAT: [490, 534],
  TOTAL: [534, 1e9],
} as const;

// Activity sub-row ("Dry Rate <qty> <unit> <rate> <amount> 0001").
const ACT = {
  QTY: [220, 262],
  UNIT: [262, 300],
  RATE: [300, 350],
  AMOUNT: [350, 392],
} as const;

type Word = { x: number; y: number; str: string };

const inBand = (x: number, [lo, hi]: readonly [number, number]) => x >= lo && x < hi;

// "1,234.50" / "1.234,50" / "1 234,50" / "-" / ""
function num(s: string | undefined): number | null {
  if (!s) return null;
  const t = s.replace(/\s/g, "").replace(/,/g, "").replace(/ /g, "");
  if (!t || t === "-" || t === "–") return null;
  const n = Number(t);
  return Number.isFinite(n) ? n : null;
}

const r2 = (n: number) => Math.round(n * 100) / 100;

/**
 * Splits a machine row into its code and description.
 *
 * The code is a leading alpha block plus a 1-3 digit block, optionally followed by a
 * dash and an upper-case token that must end at whitespace. That rule resolves every
 * recurring case correctly, including the two that look alike but aren't:
 *   "MPLANT05-Storage Container"  -> MPLANT05 / Storage Container   (suffix ends in lowercase)
 *   "RV-01-CATCB22 2.2 TON ..."   -> RV-01     / CATCB22 2.2 TON ... (suffix runs into a digit)
 * The second yields a shorter code, which is cosmetic — no amount depends on it, and
 * that code does not repeat across invoices, so duplicate detection is unaffected.
 */
function splitMachineRow(text: string): { code: string; description: string } {
  const trimmed = text.trim();
  const core = /^([A-Za-z]{2,8}-?[0-9]{2,3})/.exec(trimmed);
  if (!core) {
    const token = trimmed.split(/\s+/)[0] ?? trimmed;
    return { code: token, description: "" };
  }
  const base = core[1];
  let rest = trimmed.slice(base.length);
  // An optional -SUFFIX belongs to the code only when it ends at whitespace. That is
  // what separates the two look-alike cases:
  //   "MPLANT05-Storage Container" -> MPLANT05 / Storage Container (suffix ends lowercase)
  //   "MG-01-DEZI MG80 ..."        -> MG-01-DEZI / MG80 ...       (suffix ends at a space)
  // The suffix is always UPPER CASE (codes are) and must be bounded, which is what
  // separates a real code suffix from the start of a description word. Two bounds occur
  // in this report:
  //   1. the suffix ends at whitespace           "MG-01-DEZI MG80 Motor Grader"
  //   2. the suffix runs into a digit group that ends at whitespace
  //                                               "RV-01-CATCB22 2.2 TON Roller"
  // Known limit: "RV-01-CATCB22 2.2 TON" has no separator between the code and the
  // description ("CB22" is the machine model), so the split point is a judgement call.
  // The parser takes the maximal upper-case run, giving RV-01-CATCB. This is a display
  // label only — no amount depends on it, and registerCodeFrom() still resolves it to
  // RV01, so booking is unaffected. The hand extraction records RV-01-CAT.
  // Case matters: matching [A-Za-z] here would swallow the "Office" of
  // "MPLANT04-Office Container" and lose the code.
  const sfx =
    /^-([A-Z0-9]{1,6})(?=\s|$)/.exec(rest) ??
    /^-([A-Z]{2,5})(?=[A-Z]*[0-9]+[\s,])/.exec(rest);
  let code = base;
  if (sfx) {
    code = `${base}-${sfx[1]}`;
    rest = rest.slice(sfx[0].length);
  }
  return { code, description: rest.trim().replace(/^[-–]\s*/, "") };
}

// The code an invoice uses maps to a register code by taking the leading letters+digits
// and dropping punctuation: "SS-01-CASE" -> "SS01", "TP601-TATA" -> "TP601".
export function registerCodeFrom(invoiceCode: string): string {
  const m = /^([A-Za-z]+)[- ]?([0-9]{1,3})/.exec(invoiceCode.trim());
  return m ? `${m[1].toUpperCase()}${m[2]}` : "";
}

// The site is whatever follows the month and year in the description:
//
//   "Hire Plant Sep 2026 Cosmo City"      -> Cosmo City
//   "Plant Hire Sep 2026 West Park"       -> West Park
//   "Crane Truck Hire Sep 2026 Cosmo"      -> Cosmo
//
// The wording BEFORE the date is not fixed — PHI 179 leads with the machine type, not
// with "Hire Plant" — so anchoring on those two phrases silently returned "" and left the
// invoice unlinked to any site. The month/year pair is the one part every description
// shares. A description with no month and year is not an invoice description, so it
// yields "" rather than a guess.
export function siteFromDescription(desc: string): string {
  const m =
    /(?:jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)[a-z]*\s+\d{4}\s+(.+)$/i.exec(desc.trim());
  return m ? m[1].trim() : "";
}

const MONTHS: Record<string, string> = {
  jan: "01", feb: "02", mar: "03", apr: "04", may: "05", jun: "06",
  jul: "07", aug: "08", sep: "09", oct: "10", nov: "11", dec: "12",
};

/** "18-September-2026" / "18/09/2026" -> "2026-09-18" */
export function isoDate(s: string): string {
  const t = s.trim();
  let m = /^(\d{1,2})[/-]([A-Za-z]{3,})[/-](\d{4})$/.exec(t);
  if (m) {
    const mo = MONTHS[m[2].slice(0, 3).toLowerCase()];
    if (!mo) throw new ParseError(`Unrecognised month "${m[2]}" in date "${s}"`);
    return `${m[3]}-${mo}-${m[1].padStart(2, "0")}`;
  }
  m = /^(\d{1,2})\/(\d{2})\/(\d{4})$/.exec(t);
  if (m) return `${m[3]}-${m[2]}-${m[1].padStart(2, "0")}`;
  m = /^(\d{4})-(\d{2})-(\d{2})$/.exec(t);
  if (m) return t;
  throw new ParseError(`Unrecognised date format "${s}"`);
}

// pdfjs resolves its worker lazily at runtime. Left to its own devices under Next it
// falls back to a dynamic import of the worker module, which the bundler rewrites to a
// .next path that does not exist in the standalone image — every parse then dies with
// "Setting up fake worker failed". Pinning the worker to the real installed file avoids
// that, but only if the file is actually there: pdfjs-dist is in
// serverExternalPackages so it is required from node_modules rather than bundled, and
// Next's tracer does NOT copy pdf.worker.mjs — the standalone bundle contains pdf.mjs
// alone. The Dockerfile copies the whole package in for that reason. Do not assume this
// pin alone is sufficient.
let workerPinned = false;
async function loadPdfjs() {
  const pdfjs = await import("pdfjs-dist/legacy/build/pdf.mjs");
  if (!workerPinned) {
    try {
      const { createRequire } = await import("node:module");
      const req = createRequire(import.meta.url);
      pdfjs.GlobalWorkerOptions.workerSrc = req.resolve(
        "pdfjs-dist/legacy/build/pdf.worker.mjs",
      );
      workerPinned = true;
    } catch (err) {
      // Not fatal, but it must not be silent: pdfjs then falls back to a worker
      // path relative to pdf.mjs, so the parse fails much later with an opaque
      // "Setting up fake worker failed". Staying UNpinned lets the next parse
      // retry, so the worker is picked up as soon as it is on disk.
      console.error(
        `[invoice-parser] could not resolve pdf.worker.mjs: ${(err as Error).message}`,
      );
    }
  }
  return pdfjs;
}

type Row = { y: number; words: Word[] };

// pdfjs splits ONE printed table row across two or three y positions — on PHI 179 the
// TOTAL sits ~2pt above the rest of its own row — so a "row" has to be a y-BAND, not a
// y. 4pt is the tightest grouping that still never merges two printed rows: the report
// repeats its charge block every ~54pt and its lines are ~13pt apart at the closest.
function clusterRows(words: Word[]): Row[] {
  const rows: Row[] = [];
  for (const w of [...words].sort((a, b) => a.y - b.y || a.x - b.x)) {
    const last = rows[rows.length - 1];
    if (last && w.y - last.y <= 4) last.words.push(w);
    else rows.push({ y: w.y, words: [w] });
  }
  for (const r of rows) r.words.sort((a, b) => a.x - b.x);
  return rows;
}

// Left-column words that are NOT machine rows: the template's own labels.
const NOT_A_MACHINE = /^(Return|Sep|Dry|Idle|Rib|Ref|Printed|Description|External)$/i;

const machineWordOf = (row: Row): Word | undefined =>
  row.words.find(
    (w) =>
      w.x < 45 &&
      /[A-Za-z]/.test(w.str) &&
      !NOT_A_MACHINE.test(w.str) &&
      !/\bNo\.$/.test(w.str) &&
      /[A-Za-z]{2,8}-?[0-9]{2,3}/.test(w.str),
  );

const isGrandTotalRow = (row: Row) => row.words.some((w) => w.str.startsWith("Plant Hire Total"));

// A money row is one carrying a figure in HIRE, VAT and TOTAL.
function moneyOf(row: Row): { hire: number; vat: number; total: number } | null {
  const hire = num(row.words.find((w) => inBand(w.x, COL.HIRE))?.str);
  const vat = num(row.words.find((w) => inBand(w.x, COL.VAT))?.str);
  const total = num(row.words.find((w) => inBand(w.x, COL.TOTAL))?.str);
  return hire == null || vat == null || total == null ? null : { hire, vat, total };
}

/**
 * The activity sub-row belonging to a money row: the next row carrying a NUMBER in the
 * Qty column. The "Description Qty Unit Rate" header printed between the two has no
 * number there, so it is skipped by shape rather than by a fixed pixel offset.
 */
function activityAfter(rows: Row[], from: number): Word[] {
  for (let i = from + 1; i < rows.length && i <= from + 4; i++) {
    // The next charge begins — this row had no activity sub-row of its own.
    if (moneyOf(rows[i])) return [];
    if (num(rows[i].words.find((w) => inBand(w.x, ACT.QTY))?.str) != null) return rows[i].words;
  }
  return [];
}

/**
 * One invoice line per CHARGE, not per machine.
 *
 * A machine hired over more than one period is printed as a single machine heading
 * followed by a repeating block per charge. PHI 179 is one CT01 crane truck billed on
 * 4 Sep, 16 Sep and 18 Sep, at R5 265 + R3 200 + R5 265 = R13 730. Reading only the
 * first block per machine produced R5 265 against a R13 730 subtotal and the invoice
 * was refused as a problem invoice. Every money row therefore yields its own line,
 * carrying the same code as its machine, so the line count follows the CHARGES rather
 * than the machines — and a machine billed once still yields exactly one line, which is
 * what the single-period invoices all do.
 */
export function extractLines(words: Word[]): ParsedLine[] {
  const rows = clusterRows(words);
  const out: ParsedLine[] = [];
  const machineIdxs = rows.map((r, i) => (machineWordOf(r) ? i : -1)).filter((i) => i >= 0);
  const grandTotal = rows.findIndex(isGrandTotalRow);

  machineIdxs.forEach((m, n) => {
    const mw = machineWordOf(rows[m])!;
    const next = machineIdxs[n + 1] ?? rows.length;
    // The grand total is itself a money row, so each machine block is cut short at it.
    const stop = grandTotal > m && grandTotal < next ? grandTotal : next;
    const { code, description } = splitMachineRow(mw.str);

    for (let i = m + 1; i < stop; i++) {
      const money = moneyOf(rows[i]);
      if (!money) continue;

      const act = activityAfter(rows, i);
      const qty = num(act.find((w) => inBand(w.x, ACT.QTY))?.str) ?? 0;
      const unit = act.find((w) => inBand(w.x, ACT.UNIT))?.str ?? "";
      const rate = num(act.find((w) => inBand(w.x, ACT.RATE))?.str) ?? 0;
      const label = act
        .filter((w) => w.x < ACT.QTY[0])
        .map((w) => w.str)
        .join(" ")
        .replace(/\s+/g, " ")
        .trim();

      out.push({
        machine_code: code,
        description,
        qty,
        unit,
        rate,
        rate_type: label,
        amount_excl: r2(money.hire),
        vat: r2(money.vat),
        total_incl: r2(money.total),
      });
    }
  });

  return out;
}

export async function parseInvoicePdf(bytes: Uint8Array): Promise<ParsedInvoice> {
  const pdfjs = await loadPdfjs();
  // pdfjs TRANSFERS the buffer it is handed, which detaches it. Callers keep their own
  // copy (to store the original file), so parse a duplicate rather than the original.
  const doc = await pdfjs.getDocument({
    data: new Uint8Array(bytes),
    useSystemFonts: true,
  }).promise;

  const pages: Word[][] = [];
  try {
    for (let p = 1; p <= doc.numPages; p++) {
      const page = await doc.getPage(p);
      const vp = page.getViewport({ scale: 1 });
      const content = await page.getTextContent();
      const words: Word[] = [];
      for (const it of content.items as any[]) {
        if (typeof it.str !== "string" || !it.str.trim()) continue;
        words.push({ x: it.transform[4], y: vp.height - it.transform[5], str: it.str });
      }
      pages.push(words);
    }

    // ---- header (page 1) -------------------------------------------------
    const h1 = pages[0] ?? [];
    const onRow = (words: Word[], label: string) => words.filter((w) => w.str.startsWith(label));

    const after = (label: string, page = 0): string => {
      const ws = onRow(pages[page] ?? [], label);
      if (!ws.length) return "";
      const y = ws[0].y;
      const same = (pages[page] ?? []).filter((w) => Math.abs(w.y - y) < 3 && w.x > ws[0].x + ws[0].str.length);
      return same.map((w) => w.str).join(" ").trim();
    };

    const invoiceNo = after("Invoice") || "";
    if (!invoiceNo) throw new ParseError('Could not find "Invoice No." in the header');
    const orderNo = after("ORDER NUMBER") || "";
    const invoiceDateRaw = after("DATE") || "";
    if (!invoiceDateRaw) throw new ParseError('Could not find "DATE" in the header');
    const debtorNo = after("DEBTOR NUMBER") || "";
    const currency = after("CURRENCY") || "ZAR";
    const vatReg = after("VAT Reg. No.") || "";

    // Debtor name sits at the far left of the block above ORDER NUMBER.
    const debtorWord = h1.find((w) => w.x < 100 && /^Mega |Infrastructure/i.test(w.str) && w.y > 180 && w.y < 220);
    const debtor = debtorWord ? debtorWord.str.replace(/^Mega\s+/i, "Mega ") : "";

    // "Description :" and the free text beside it.
    const descRow = h1.filter((w) => Math.abs(w.y - (h1.find((x) => x.str === "Description")?.y ?? -999)) < 3);
    const descWord = descRow.find((w) => w.str === "Description");
    const description = descWord
      ? descRow
          .filter(
            (w) =>
              w.x > descWord.x + 12 &&
              w.x < 460 &&
              !w.str.endsWith(":") &&
              !/^\d{1,2}\/\d{2}\/\d{4}$/.test(w.str) &&
              !/Date$/.test(w.str),
          )
          .sort((a, b) => a.x - b.x)
          .map((w) => w.str)
          .join(" ")
          .trim()
      : "";

    const dates = h1
      .filter((w) => /^\d{1,2}\/\d{2}\/\d{4}$/.test(w.str))
      .sort((a, b) => a.y - b.y || a.x - b.x)
      .map((w) => w.str);
    if (dates.length < 2) throw new ParseError("Could not read the From/To dates from the header");
    const period_from = isoDate(dates[0]);
    const period_to = isoDate(dates[1]);
    const invoice_date = isoDate(invoiceDateRaw);

    // ---- machine lines (every page) --------------------------------------
    const lines: ParsedLine[] = [];
    for (const words of pages) lines.push(...extractLines(words));
    if (!lines.length) throw new ParseError("No machine lines were found on any page");

    // ---- totals -----------------------------------------------------------
    // The report emits the grand-total label as ONE text item ("Plant Hire Total"),
    // so match on prefix rather than equality.
    let hireTotal: number | null = null;
    let taxTotal: number | null = null;
    let grandTotal: number | null = null;
    let escalationTotal = 0;
    for (const words of pages) {
      const row = words.find((w) => w.str.startsWith("Plant Hire Total"));
      if (!row) continue;
      const same = words.filter((w) => Math.abs(w.y - row.y) < 4 && w.x > row.x + row.str.length);
      hireTotal = num(same.find((w) => inBand(w.x, COL.HIRE))?.str);
      escalationTotal = num(same.find((w) => inBand(w.x, COL.ESCALATION))?.str) ?? 0;
      taxTotal = num(same.find((w) => inBand(w.x, COL.VAT))?.str);
      grandTotal = num(same.find((w) => inBand(w.x, COL.TOTAL))?.str);
    }
    if (hireTotal == null || taxTotal == null || grandTotal == null) {
      throw new ParseError('Could not find the "Plant Hire Total" row');
    }

    // The summary block is read BY LABEL — its values sit at different x positions
    // from the line table, so column bands cannot be used here. pdfjs splits these
    // labels inconsistently between pages, so each is matched by prefix.
    const labelValue = (words: Word[], label: string): number | null => {
      const lw = words.find((w) => w.str === label || w.str.startsWith(`${label} `));
      if (!lw) return null;
      const labelEnd = lw.x + lw.str.length;
      const same = words.filter((w) => Math.abs(w.y - lw.y) < 5 && w.x > labelEnd + 15);
      return num(same.map((w) => w.str).join(""));
    };

    const summary: Record<string, number> = {};
    for (const words of pages) {
      const put = (k: string, v: number | null) => { if (v != null) summary[k] = v; };
      put("SUB TOTAL", labelValue(words, "SUB TOTAL"));
      put("Escalation", labelValue(words, "Escalation"));
      put("PLUS", labelValue(words, "PLUS"));
      put("INVOICE", labelValue(words, "INVOICE"));
    }

    const subtotal = summary["SUB TOTAL"] ?? hireTotal;
    const escalation = summary["Escalation"] ?? escalationTotal;
    const vat = summary["PLUS"] ?? taxTotal;
    const total_excl = r2(subtotal + escalation);
    const total_incl = summary["INVOICE"] ?? grandTotal;

    // ---- validation — the same gates the seed applies ---------------------
    const sumHire = r2(lines.reduce((s, l) => s + l.amount_excl, 0));
    if (Math.abs(sumHire - subtotal) > 0.01) {
      throw new ParseError(`Machine lines total R${sumHire} but the invoice subtotal is R${subtotal}`);
    }
    const sumVat = r2(lines.reduce((s, l) => s + l.vat, 0));
    if (Math.abs(sumVat - vat) > 0.02) {
      throw new ParseError(`Line VAT totals R${sumVat} but the invoice PLUS VAT is R${vat}`);
    }
    if (Math.abs(total_excl + vat - total_incl) > 0.01) {
      throw new ParseError(
        `Totals do not balance: R${total_excl} ex + R${vat} VAT != R${total_incl} invoice total`,
      );
    }
    for (const l of lines) {
      if (Math.abs(l.amount_excl + l.vat - l.total_incl) > 0.01) {
        throw new ParseError(
          `Line ${l.machine_code}: R${l.amount_excl} + R${l.vat} VAT != R${l.total_incl}`,
        );
      }
      if (l.qty && l.rate && Math.abs(r2(l.qty * l.rate) - l.amount_excl) > 0.01) {
        throw new ParseError(
          `Line ${l.machine_code}: ${l.qty} x ${l.rate} = ${r2(l.qty * l.rate)}, but the line amount is R${l.amount_excl}`,
        );
      }
    }

    const vat_pct = total_excl > 0 ? Math.round((vat / total_excl) * 1000) / 10 : 0;
    for (const l of lines) {
      const expected = r2((l.amount_excl * vat_pct) / 100);
      if (l.amount_excl > 0 && Math.abs(expected - l.vat) > 0.02) {
        throw new ParseError(
          `Line ${l.machine_code}: VAT is R${l.vat}, but ${vat_pct}% of R${l.amount_excl} is R${expected}`,
        );
      }
    }

    return {
      invoice_no: invoiceNo.trim().replace(/\s+/g, " "),
      invoice_date,
      order_no: orderNo,
      debtor,
      debtor_no: debtorNo,
      period_from,
      period_to,
      description,
      site: siteFromDescription(description),
      section: "Plant Hire",
      category: "External Plant Hire",
      currency,
      vat_reg_no: vatReg,
      vat_pct,
      subtotal: r2(subtotal),
      escalation: r2(escalation),
      vat: r2(vat),
      total_excl,
      total_incl: r2(total_incl),
      lines,
    };
  } finally {
    await doc.destroy().catch(() => {});
  }
}