mega-plant/lib/invoice-parser.ts

// Deterministic parser for Mega Plant "Std_ReqHireInvoice_DN.rpt" VAT invoices.
//
// Deliberately NOT an AI extractor. That supplier's report is a fixed template: the
// columns sit at identical x-positions on every invoice and only the number of
// machine lines and pages varies. So the figures are read by coordinate, exactly the
// way they were transcribed by hand, and then checked arithmetically.
//
// Anything that does not reconcile is reported as a reason and lands in the intake
// inbox as a problem invoice. It never guesses a total and never writes a partial
// invoice.
//
// Calibrated against PHI 166 / 167 / 168 / 169 / 171.

export type ParsedLine = {
  machine_code: string;
  description: string;
  qty: number;
  unit: string;
  rate: number;
  rate_type: string;
  amount_excl: number;
  vat: number;
  total_incl: number;
};

export type ParsedInvoice = {
  invoice_no: string;
  invoice_date: string;
  order_no: string;
  debtor: string;
  debtor_no: string;
  period_from: string;
  period_to: string;
  description: string;
  site: string;
  category: string;
  section: string;
  currency: string;
  vat_reg_no: string;
  vat_pct: number;
  subtotal: number;
  escalation: number;
  vat: number;
  total_excl: number;
  total_incl: number;
  lines: ParsedLine[];
};

export class ParseError extends Error {
  constructor(public reason: string) {
    super(reason);
    this.name = "ParseError";
  }
}

// Column left-edges, calibrated from the report template. Numbers are right-aligned
// within their column, so each value's LEFT edge lands in a stable band.
const COL = {
  HIRE: [340, 400],
  PENALTY2: [400, 440],
  ESCALATION: [440, 490],
  VAT: [490, 534],
  TOTAL: [534, 1e9],
} as const;

// Activity sub-row ("Dry Rate <qty> <unit> <rate> <amount> 0001").
const ACT = {
  QTY: [220, 262],
  UNIT: [262, 300],
  RATE: [300, 350],
  AMOUNT: [350, 392],
} as const;

type Word = { x: number; y: number; str: string };

const inBand = (x: number, [lo, hi]: readonly [number, number]) => x >= lo && x < hi;

// "1,234.50" / "1.234,50" / "1 234,50" / "-" / ""
function num(s: string | undefined): number | null {
  if (!s) return null;
  const t = s.replace(/\s/g, "").replace(/,/g, "").replace(/ /g, "");
  if (!t || t === "-" || t === "–") return null;
  const n = Number(t);
  return Number.isFinite(n) ? n : null;
}

const r2 = (n: number) => Math.round(n * 100) / 100;

/**
 * Splits a machine row into its code and description.
 *
 * The code is a leading alpha block plus a 1-3 digit block, optionally followed by a
 * dash and an upper-case token that must end at whitespace. That rule resolves every
 * recurring case correctly, including the two that look alike but aren't:
 *   "MPLANT05-Storage Container"  -> MPLANT05 / Storage Container   (suffix ends in lowercase)
 *   "RV-01-CATCB22 2.2 TON ..."   -> RV-01     / CATCB22 2.2 TON ... (suffix runs into a digit)
 * The second yields a shorter code, which is cosmetic — no amount depends on it, and
 * that code does not repeat across invoices, so duplicate detection is unaffected.
 */
function splitMachineRow(text: string): { code: string; description: string } {
  const trimmed = text.trim();
  const core = /^([A-Za-z]{2,8}-?[0-9]{2,3})/.exec(trimmed);
  if (!core) {
    const token = trimmed.split(/\s+/)[0] ?? trimmed;
    return { code: token, description: "" };
  }
  const base = core[1];
  let rest = trimmed.slice(base.length);
  // An optional -SUFFIX belongs to the code only when it ends at whitespace. That is
  // what separates the two look-alike cases:
  //   "MPLANT05-Storage Container" -> MPLANT05 / Storage Container (suffix ends lowercase)
  //   "MG-01-DEZI MG80 ..."        -> MG-01-DEZI / MG80 ...       (suffix ends at a space)
  // The suffix is always UPPER CASE (codes are) and must be bounded, which is what
  // separates a real code suffix from the start of a description word. Two bounds occur
  // in this report:
  //   1. the suffix ends at whitespace           "MG-01-DEZI MG80 Motor Grader"
  //   2. the suffix runs into a digit group that ends at whitespace
  //                                               "RV-01-CATCB22 2.2 TON Roller"
  // Known limit: "RV-01-CATCB22 2.2 TON" has no separator between the code and the
  // description ("CB22" is the machine model), so the split point is a judgement call.
  // The parser takes the maximal upper-case run, giving RV-01-CATCB. This is a display
  // label only — no amount depends on it, and registerCodeFrom() still resolves it to
  // RV01, so booking is unaffected. The hand extraction records RV-01-CAT.
  // Case matters: matching [A-Za-z] here would swallow the "Office" of
  // "MPLANT04-Office Container" and lose the code.
  const sfx =
    /^-([A-Z0-9]{1,6})(?=\s|$)/.exec(rest) ??
    /^-([A-Z]{2,5})(?=[A-Z]*[0-9]+[\s,])/.exec(rest);
  let code = base;
  if (sfx) {
    code = `${base}-${sfx[1]}`;
    rest = rest.slice(sfx[0].length);
  }
  return { code, description: rest.trim().replace(/^[-–]\s*/, "") };
}

// The code an invoice uses maps to a register code by taking the leading letters+digits
// and dropping punctuation: "SS-01-CASE" -> "SS01", "TP601-TATA" -> "TP601".
export function registerCodeFrom(invoiceCode: string): string {
  const m = /^([A-Za-z]+)[- ]?([0-9]{1,3})/.exec(invoiceCode.trim());
  return m ? `${m[1].toUpperCase()}${m[2]}` : "";
}

// "Hire Plant Sep 2026 Rooihuiskraal" / "Plant Hire SEP 2026 West Park" -> the site.
export function siteFromDescription(desc: string): string {
  const m = /^(?:hire\s+plant|plant\s+hire)\s+\S+\s+\d{4}\s+(.+)$/i.exec(desc.trim());
  return m ? m[1].trim() : "";
}

const MONTHS: Record<string, string> = {
  jan: "01", feb: "02", mar: "03", apr: "04", may: "05", jun: "06",
  jul: "07", aug: "08", sep: "09", oct: "10", nov: "11", dec: "12",
};

/** "18-September-2026" / "18/09/2026" -> "2026-09-18" */
export function isoDate(s: string): string {
  const t = s.trim();
  let m = /^(\d{1,2})[/-]([A-Za-z]{3,})[/-](\d{4})$/.exec(t);
  if (m) {
    const mo = MONTHS[m[2].slice(0, 3).toLowerCase()];
    if (!mo) throw new ParseError(`Unrecognised month "${m[2]}" in date "${s}"`);
    return `${m[3]}-${mo}-${m[1].padStart(2, "0")}`;
  }
  m = /^(\d{1,2})\/(\d{2})\/(\d{4})$/.exec(t);
  if (m) return `${m[3]}-${m[2]}-${m[1].padStart(2, "0")}`;
  m = /^(\d{4})-(\d{2})-(\d{2})$/.exec(t);
  if (m) return t;
  throw new ParseError(`Unrecognised date format "${s}"`);
}

// pdfjs resolves its worker lazily at runtime. Left to its own devices under Next it
// falls back to a dynamic import of the worker module, which the bundler rewrites to a
// .next path that does not exist in the standalone image — every parse then dies with
// "Setting up fake worker failed". Pinning the worker to the real installed file avoids
// that. pdfjs-dist is listed in serverExternalPackages so it is required (and traced)
// from node_modules rather than bundled.
let workerPinned = false;
async function loadPdfjs() {
  const pdfjs = await import("pdfjs-dist/legacy/build/pdf.mjs");
  if (!workerPinned) {
    workerPinned = true;
    try {
      const { createRequire } = await import("node:module");
      const req = createRequire(import.meta.url);
      pdfjs.GlobalWorkerOptions.workerSrc = req.resolve(
        "pdfjs-dist/legacy/build/pdf.worker.mjs",
      );
    } catch {
      // Leave pdfjs's own default in place; it is correct for unbundled Node.
    }
  }
  return pdfjs;
}

export async function parseInvoicePdf(bytes: Uint8Array): Promise<ParsedInvoice> {
  const pdfjs = await loadPdfjs();
  // pdfjs TRANSFERS the buffer it is handed, which detaches it. Callers keep their own
  // copy (to store the original file), so parse a duplicate rather than the original.
  const doc = await pdfjs.getDocument({
    data: new Uint8Array(bytes),
    useSystemFonts: true,
  }).promise;

  const pages: Word[][] = [];
  try {
    for (let p = 1; p <= doc.numPages; p++) {
      const page = await doc.getPage(p);
      const vp = page.getViewport({ scale: 1 });
      const content = await page.getTextContent();
      const words: Word[] = [];
      for (const it of content.items as any[]) {
        if (typeof it.str !== "string" || !it.str.trim()) continue;
        words.push({ x: it.transform[4], y: vp.height - it.transform[5], str: it.str });
      }
      pages.push(words);
    }

    // ---- header (page 1) -------------------------------------------------
    const h1 = pages[0] ?? [];
    const onRow = (words: Word[], label: string) => words.filter((w) => w.str.startsWith(label));

    const after = (label: string, page = 0): string => {
      const ws = onRow(pages[page] ?? [], label);
      if (!ws.length) return "";
      const y = ws[0].y;
      const same = (pages[page] ?? []).filter((w) => Math.abs(w.y - y) < 3 && w.x > ws[0].x + ws[0].str.length);
      return same.map((w) => w.str).join(" ").trim();
    };

    const invoiceNo = after("Invoice") || "";
    if (!invoiceNo) throw new ParseError('Could not find "Invoice No." in the header');
    const orderNo = after("ORDER NUMBER") || "";
    const invoiceDateRaw = after("DATE") || "";
    if (!invoiceDateRaw) throw new ParseError('Could not find "DATE" in the header');
    const debtorNo = after("DEBTOR NUMBER") || "";
    const currency = after("CURRENCY") || "ZAR";
    const vatReg = after("VAT Reg. No.") || "";

    // Debtor name sits at the far left of the block above ORDER NUMBER.
    const debtorWord = h1.find((w) => w.x < 100 && /^Mega |Infrastructure/i.test(w.str) && w.y > 180 && w.y < 220);
    const debtor = debtorWord ? debtorWord.str.replace(/^Mega\s+/i, "Mega ") : "";

    // "Description :" and the free text beside it.
    const descRow = h1.filter((w) => Math.abs(w.y - (h1.find((x) => x.str === "Description")?.y ?? -999)) < 3);
    const descWord = descRow.find((w) => w.str === "Description");
    const description = descWord
      ? descRow
          .filter(
            (w) =>
              w.x > descWord.x + 12 &&
              w.x < 460 &&
              !w.str.endsWith(":") &&
              !/^\d{1,2}\/\d{2}\/\d{4}$/.test(w.str) &&
              !/Date$/.test(w.str),
          )
          .sort((a, b) => a.x - b.x)
          .map((w) => w.str)
          .join(" ")
          .trim()
      : "";

    const dates = h1
      .filter((w) => /^\d{1,2}\/\d{2}\/\d{4}$/.test(w.str))
      .sort((a, b) => a.y - b.y || a.x - b.x)
      .map((w) => w.str);
    if (dates.length < 2) throw new ParseError("Could not read the From/To dates from the header");
    const period_from = isoDate(dates[0]);
    const period_to = isoDate(dates[1]);
    const invoice_date = isoDate(invoiceDateRaw);

    // ---- machine lines (every page) --------------------------------------
    const lines: ParsedLine[] = [];
    for (const words of pages) {
      // A machine row is a left-column item that is not a known header/footer word.
      const machineRows = words.filter(
        (w) => w.x < 45 && /[A-Za-z]/.test(w.str) && !/^(Return|Sep|Dry|Rib|Ref|Printed|Description)$/i.test(w.str) &&
               !/\bNo\.$/.test(w.str) && /[A-Za-z]{2,8}-?[0-9]{2,3}/.test(w.str),
      );
      for (const mr of machineRows) {
        // Numeric row sits ~16pt below the machine name.
        const numRow = words.filter((w) => w.y > mr.y + 10 && w.y < mr.y + 24);
        const hire = num(numRow.find((w) => inBand(w.x, COL.HIRE))?.str);
        const vat = num(numRow.find((w) => inBand(w.x, COL.VAT))?.str);
        const total = num(numRow.find((w) => inBand(w.x, COL.TOTAL))?.str);
        if (hire == null || vat == null || total == null) continue;

        // Activity row: the "Dry Rate ..." line, roughly 57pt below the machine name.
        // The "Description Qty Unit ..." header sits at ~44pt and must not be mistaken
        // for it, so locate the row by its leading "Dry" token rather than by offset.
        const band = words.filter((w) => w.y > mr.y + 38 && w.y < mr.y + 78 && w.x > 150);
        const dryY = band.find((w) => w.str === "Dry" || w.str.startsWith("Dry "))?.y;
        const dry = dryY == null ? [] : band.filter((w) => Math.abs(w.y - dryY) < 3).sort((a, b) => a.x - b.x);
        const qty = num(dry.find((w) => inBand(w.x, ACT.QTY))?.str);
        const unit = dry.find((w) => inBand(w.x, ACT.UNIT))?.str ?? "";
        const rate = num(dry.find((w) => inBand(w.x, ACT.RATE))?.str);
        // pdfjs emits "Dry Rate" as a single token; older builds split it in two.
        const rateType = dry[0] ? dry[0].str.replace(/\s+/g, " ") : "Dry Rate";

        const { code, description: desc } = splitMachineRow(mr.str);
        lines.push({
          machine_code: code,
          description: desc,
          qty: qty ?? 0,
          unit,
          rate: rate ?? 0,
          rate_type: rateType,
          amount_excl: r2(hire),
          vat: r2(vat),
          total_incl: r2(total),
        });
      }
    }
    if (!lines.length) throw new ParseError("No machine lines were found on any page");

    // ---- totals -----------------------------------------------------------
    // The report emits the grand-total label as ONE text item ("Plant Hire Total"),
    // so match on prefix rather than equality.
    let hireTotal: number | null = null;
    let taxTotal: number | null = null;
    let grandTotal: number | null = null;
    let escalationTotal = 0;
    for (const words of pages) {
      const row = words.find((w) => w.str.startsWith("Plant Hire Total"));
      if (!row) continue;
      const same = words.filter((w) => Math.abs(w.y - row.y) < 4 && w.x > row.x + row.str.length);
      hireTotal = num(same.find((w) => inBand(w.x, COL.HIRE))?.str);
      escalationTotal = num(same.find((w) => inBand(w.x, COL.ESCALATION))?.str) ?? 0;
      taxTotal = num(same.find((w) => inBand(w.x, COL.VAT))?.str);
      grandTotal = num(same.find((w) => inBand(w.x, COL.TOTAL))?.str);
    }
    if (hireTotal == null || taxTotal == null || grandTotal == null) {
      throw new ParseError('Could not find the "Plant Hire Total" row');
    }

    // The summary block is read BY LABEL — its values sit at different x positions
    // from the line table, so column bands cannot be used here. pdfjs splits these
    // labels inconsistently between pages, so each is matched by prefix.
    const labelValue = (words: Word[], label: string): number | null => {
      const lw = words.find((w) => w.str === label || w.str.startsWith(`${label} `));
      if (!lw) return null;
      const labelEnd = lw.x + lw.str.length;
      const same = words.filter((w) => Math.abs(w.y - lw.y) < 5 && w.x > labelEnd + 15);
      return num(same.map((w) => w.str).join(""));
    };

    const summary: Record<string, number> = {};
    for (const words of pages) {
      const put = (k: string, v: number | null) => { if (v != null) summary[k] = v; };
      put("SUB TOTAL", labelValue(words, "SUB TOTAL"));
      put("Escalation", labelValue(words, "Escalation"));
      put("PLUS", labelValue(words, "PLUS"));
      put("INVOICE", labelValue(words, "INVOICE"));
    }

    const subtotal = summary["SUB TOTAL"] ?? hireTotal;
    const escalation = summary["Escalation"] ?? escalationTotal;
    const vat = summary["PLUS"] ?? taxTotal;
    const total_excl = r2(subtotal + escalation);
    const total_incl = summary["INVOICE"] ?? grandTotal;

    // ---- validation — the same gates the seed applies ---------------------
    const sumHire = r2(lines.reduce((s, l) => s + l.amount_excl, 0));
    if (Math.abs(sumHire - subtotal) > 0.01) {
      throw new ParseError(`Machine lines total R${sumHire} but the invoice subtotal is R${subtotal}`);
    }
    const sumVat = r2(lines.reduce((s, l) => s + l.vat, 0));
    if (Math.abs(sumVat - vat) > 0.02) {
      throw new ParseError(`Line VAT totals R${sumVat} but the invoice PLUS VAT is R${vat}`);
    }
    if (Math.abs(total_excl + vat - total_incl) > 0.01) {
      throw new ParseError(
        `Totals do not balance: R${total_excl} ex + R${vat} VAT != R${total_incl} invoice total`,
      );
    }
    for (const l of lines) {
      if (Math.abs(l.amount_excl + l.vat - l.total_incl) > 0.01) {
        throw new ParseError(
          `Line ${l.machine_code}: R${l.amount_excl} + R${l.vat} VAT != R${l.total_incl}`,
        );
      }
      if (l.qty && l.rate && Math.abs(r2(l.qty * l.rate) - l.amount_excl) > 0.01) {
        throw new ParseError(
          `Line ${l.machine_code}: ${l.qty} x ${l.rate} = ${r2(l.qty * l.rate)}, but the line amount is R${l.amount_excl}`,
        );
      }
    }

    const vat_pct = total_excl > 0 ? Math.round((vat / total_excl) * 1000) / 10 : 0;
    for (const l of lines) {
      const expected = r2((l.amount_excl * vat_pct) / 100);
      if (l.amount_excl > 0 && Math.abs(expected - l.vat) > 0.02) {
        throw new ParseError(
          `Line ${l.machine_code}: VAT is R${l.vat}, but ${vat_pct}% of R${l.amount_excl} is R${expected}`,
        );
      }
    }

    return {
      invoice_no: invoiceNo.trim().replace(/\s+/g, " "),
      invoice_date,
      order_no: orderNo,
      debtor,
      debtor_no: debtorNo,
      period_from,
      period_to,
      description,
      site: siteFromDescription(description),
      section: "Plant Hire",
      category: "External Plant Hire",
      currency,
      vat_reg_no: vatReg,
      vat_pct,
      subtotal: r2(subtotal),
      escalation: r2(escalation),
      vat: r2(vat),
      total_excl,
      total_incl: r2(total_incl),
      lines,
    };
  } finally {
    await doc.destroy().catch(() => {});
  }
}