mega-plant/scripts/seed.mjs

// Seed PocketBase with data extracted from data/source/*.json
// Regenerate those files with `node scripts/extract.mjs`
//
// Usage: PB_ADMIN=... PB_PASS=... node scripts/seed.mjs
import PocketBase from "pocketbase";
import { readFileSync, existsSync, readdirSync } from "node:fs";
import { fileURLToPath } from "node:url";
import { dirname, resolve } from "node:path";

const __dirname = dirname(fileURLToPath(import.meta.url));
const DATA_DIR = resolve(__dirname, "..", "data", "source");

// Load .env.local so `npm run seed` works without exporting vars by hand.
// Explicit process.env values still win — CI/deploys can override.
const envPath = resolve(__dirname, "..", ".env.local");
if (existsSync(envPath)) {
  for (const line of readFileSync(envPath, "utf8").split("\n")) {
    const t = line.trim();
    if (!t || t.startsWith("#") || !t.includes("=")) continue;
    const i = t.indexOf("=");
    const k = t.slice(0, i).trim();
    if (!process.env[k]) process.env[k] = t.slice(i + 1).trim();
  }
}

// Read required env — fail loudly if missing (per Part 5.2)
const PB = process.env.NEXT_PUBLIC_PB_URL;
const EMAIL = process.env.PB_ADMIN;
const PASS = process.env.PB_PASS;
const errors = [];
if (!PB) errors.push("NEXT_PUBLIC_PB_URL");
if (!EMAIL) errors.push("PB_ADMIN");
if (!PASS) errors.push("PB_PASS");
if (errors.length) {
  console.error(`✗ Missing required env: ${errors.join(", ")}`);
  console.error("  Copy .env.example to .env.local and fill in.");
  process.exit(1);
}

// Placeholder pin for every machine that has no coordinate of its own, so the
// Plant Map is never blank. Every asset therefore shares one point until the
// on-site scan sheet supplies real per-machine coordinates — any explicit
// `latitude`/`longitude` in assets.json wins over this.
// When the sheet lands, delete these two lines and the `?? DEFAULT_GEO` fallbacks
// below; the JSON values will carry every pin from then on.
const DEFAULT_GEO = {
  latitude: -26.10161543177211,
  longitude: 28.34872485354214,
};

const ASSETS_COLL = "assets";
const INCOME_COLL = "monthly_income";
const RUN_COLL = "monthly_running_costs";
const OVERHEADS_COLL = "overheads";
const COST_LINES_COLL = "cost_lines";
const INVOICES_COLL = "invoices";
const INVOICE_LINES_COLL = "invoice_lines";

const assets = JSON.parse(readFileSync(resolve(DATA_DIR, "assets.json"), "utf8"));
const machines = JSON.parse(readFileSync(resolve(DATA_DIR, "machines.json"), "utf8"));
const mrc = JSON.parse(readFileSync(resolve(DATA_DIR, "mrc.json"), "utf8"));
const overheads = JSON.parse(readFileSync(resolve(DATA_DIR, "overheads.json"), "utf8"));
const costLines = JSON.parse(readFileSync(resolve(DATA_DIR, "cost_lines.json"), "utf8"));

// Invoices: one JSON per source PDF, all loaded automatically. Dropping a new
// invoice-*.json into data/source is the whole procedure for adding another invoice.
const invoiceFiles = readdirSync(DATA_DIR).filter((f) => /^invoice-.*\.json$/i.test(f)).sort();
const invoices = invoiceFiles.map((f) => ({
  ...JSON.parse(readFileSync(resolve(DATA_DIR, f), "utf8")),
  _file: f,
}));

const pb = new PocketBase(PB);
pb.autoCancellation(false);

console.log(`→ Connecting to ${PB} as ${EMAIL}`);
await pb.collection("_superusers").authWithPassword(EMAIL, PASS); // Part 5.3: _superusers, not admins
console.log("  ✓ authenticated");

// Wipe existing rows so we can re-seed cleanly
for (const c of [ASSETS_COLL, INCOME_COLL, RUN_COLL, OVERHEADS_COLL, COST_LINES_COLL, INVOICE_LINES_COLL, INVOICES_COLL]) {
  const items = await pb.collection(c).getFullList();
  for (const r of items) await pb.collection(c).delete(r.id);
}
console.log("  ✓ wiped existing rows");

// === Build alias map for code mismatches between graph sheet and asset register ===
// Per Part 1.4 / Part B.3: derived from the workbook, not invented. Unmatched rows
// must produce a loud failure, not a silent ghost.
const codeAliases = {
  "428E": "TLB02",   // CAT 425F (428E) → TLB02
  "430":  "SS01",    // CASE420 (430) → first SS01 (handled below)
};
function resolveCode(graphCode, graphName) {
  if (!graphCode) return null;
  if (codeAliases[graphCode]) {
    // Special-case 430 — first match wins unless name indicates second
    if (graphCode === "430" && /SS02/i.test(graphName)) return "SS02";
    return codeAliases[graphCode];
  }
  return graphCode;
}

// Name normalisation for fallback matching
function normalizeName(n) {
  return String(n).toLowerCase().replace(/[^a-z0-9]+/g, "").slice(0, 12);
}

// === Insert assets ===
// Site assignments are preserved across a re-seed. Before the assets are wiped
// we snapshot the current code→site mapping, because the new rows get fresh
// ids and a name/code lookup is the only stable key across the wipe.
const SITES_COLL = "sites";
let existingSiteByCode = new Map();
let existingSiteByName = new Map();
try {
  const siteIdToName = new Map();
  for (const s of await pb.collection(SITES_COLL).getFullList()) {
    siteIdToName.set(s.id, s.name);
  }
  for (const a of await pb.collection(ASSETS_COLL).getFullList()) {
    const siteName = a.site ? siteIdToName.get(a.site) : null;
    if (!siteName) continue;
    if (a.code) existingSiteByCode.set(a.code, siteName);
    existingSiteByName.set(normalizeName(a.name), siteName);
  }
  if (existingSiteByCode.size || existingSiteByName.size) {
    console.log(`  · preserving site assignments for ${existingSiteByCode.size + existingSiteByName.size} asset lookups`);
  }
} catch {
  // `sites` is optional — a fresh install may not have run the setup script.
}

let siteIdByName = new Map();
try {
  for (const s of await pb.collection(SITES_COLL).getFullList()) {
    siteIdByName.set(normalizeName(s.name), s.id);
  }
} catch {
  console.log("  · no `sites` collection — every machine will seed unassigned");
}

console.log(`\n→ Seeding ${assets.length} assets`);
const codeToId = {};
const nameToId = {};
let ok = 0;
const seedErrors = [];
for (const a of assets) {
  try {
    // `site` in assets.json wins; otherwise fall back to whatever was assigned
    // before the wipe. An unrecognised name is a loud failure, not a silent
    // unassignment — a dropped site link is exactly the kind of quiet data
    // loss this seed is meant to surface.
    const wantedSite = a.site || existingSiteByCode.get(a.code) || existingSiteByName.get(normalizeName(a.name)) || null;
    let siteId = "";
    if (wantedSite) {
      siteId = siteIdByName.get(normalizeName(wantedSite)) || "";
      if (!siteId) seedErrors.push(`asset "${a.name}": unknown site "${wantedSite}" — add it to the sites collection or clear the assignment`);
    }
    const rec = await pb.collection(ASSETS_COLL).create({
      code: a.code,
      name: a.name,
      category: a.category || "Plant",
      type: a.type || "",         // Part C.2
      gl_account: a.gl_account || "",
      supplier: a.supplier || "",
      monthly_excl: a.monthly_excl || 0,
      monthly_incl: a.monthly_incl || 0,
      purchase_value: a.purchase_value || 0,
      is_provisional: a.is_provisional || false,  // Part C.3
      is_active: a.is_active !== false, // default true; only "AEB Plant Versekering" is false
      site: siteId,
      // Plant Map — falls back to the shared placeholder pin until the
      // on-site scan sheet supplies a real coordinate for this machine.
      latitude: a.latitude ?? DEFAULT_GEO.latitude,
      longitude: a.longitude ?? DEFAULT_GEO.longitude,
    });
    if (a.code) codeToId[a.code] = rec.id;
    nameToId[normalizeName(a.name)] = rec.id;
    ok++;
  } catch (e) {
    seedErrors.push(`asset "${a.name}": ${e.response?.data?.message || e.message}`);
  }
}
console.log(`  ✓ ${ok}/${assets.length} assets inserted`);

// === Insert cost_lines (Part C.1 — finance/insurance/tracking per machine) ===
console.log(`\n→ Seeding ${costLines.length} cost_lines`);
let clRows = 0;
const clErrors = [];
for (const cl of costLines) {
  const assetId = codeToId[cl.asset_code];
  if (!assetId) { clErrors.push(`cost_line asset_code=${cl.asset_code}: asset not found`); continue; }
  try {
    await pb.collection(COST_LINES_COLL).create({
      asset: assetId,
      year: cl.year,
      month: cl.month,
      kind: cl.kind,
      amount: cl.amount,
    });
    clRows++;
  } catch (e) {
    clErrors.push(`cost_line ${cl.asset_code} ${cl.kind} m=${cl.month}: ${e.response?.data?.message || e.message}`);
  }
}
console.log(`  ✓ ${clRows} cost_lines inserted`);

// === Insert monthly_income from Machine Graph Data ===
// Part 1.5: monthly_income no longer carries running_cost — we derive it from monthly_running_costs on read.
// Part 1.4: alias map + synthetic codes for uncoded items, fail loudly on unmatched rows.
console.log("\n→ Seeding monthly_income from Machine Graph Data");
let incomeRows = 0;
const incomeErrors = [];
const lastActualMonth = (() => {
  // Last month with any recorded income across the whole workbook
  let m = 0;
  for (const x of machines) for (const d of x.data) if (d.income > 0 && d.month > m) m = d.month;
  return m;
})();
for (const m of machines) {
  const resolvedCode = resolveCode(m.code, m.machine);
  let assetId = resolvedCode ? codeToId[resolvedCode] : null;
  if (!assetId && m.machine) assetId = nameToId[normalizeName(m.machine)];
  // Part B.3: NO synthetic code, NO auto-create. Unmatched → loud failure.
  if (!assetId) {
    incomeErrors.push(`machine "${m.machine}" (code=${m.code || "—"}) — no asset matched. Add a code or sequential alias to assets.json and re-seed.`);
    continue;
  }
  for (const d of m.data) {
    const isForecast = d.month > lastActualMonth;
    try {
      await pb.collection(INCOME_COLL).create({
        asset: assetId,
        year: 2026,
        month: d.month,
        income: d.income,
        fixed_expenses: d.fixed_expenses,
        is_forecast: isForecast, // Part 1.2
        // running_cost removed (Part 1.5) — derived from monthly_running_costs at read time
      });
      incomeRows++;
    } catch (e) {
      incomeErrors.push(`income ${m.code || m.machine} m=${d.month}: ${e.response?.data?.message || e.message}`);
    }
  }
}
console.log(`  ✓ ${incomeRows} income rows inserted`);

// === Insert monthly_running_costs ===
console.log("\n→ Seeding monthly_running_costs");
let runRows = 0;
const runErrors = [];
for (const r of mrc) {
  let assetId = codeToId[r.code];
  if (!assetId) assetId = nameToId[normalizeName(r.name)];
  if (!assetId) { runErrors.push(`running ${r.code || r.name} m=${r.month}: no asset matched`); continue; }
  if (!r.amount) continue;
  try {
    await pb.collection(RUN_COLL).create({
      asset: assetId,
      year: 2026,
      month: r.month,
      amount: r.amount,
    });
    runRows++;
  } catch (e) {
    runErrors.push(`running ${r.code} m=${r.month}: ${e.response?.data?.message || e.message}`);
  }
}
console.log(`  ✓ ${runRows} running-cost rows inserted`);

// === Insert overheads ===
// Part 1.3: skip "Plant Overheads" — those rows are machines already in assets.
// Part B.4: skip "Santam versekering" — double-counted in the workbook's own subtotal.
console.log("\n→ Seeding overheads (excluding Plant Overheads + Santam)");
let ohRows = 0;
const ohErrors = [];
const VAT_WARN = []; // Part B.3: collect VAT anomalies — DO NOT silent-correct, report loudly
for (const o of overheads) {
  if (o.section === "Plant Overheads") continue; // Part 1.3
  // Note: Santam is kept (Round 4 Part A.2 — round 3 was wrong to drop it).

  // Part B.3 / Round 4 Part A.3: VAT validation — collect warnings, then FAIL the seed
  // if any row is arithmetically broken (incl < excl or VAT math doesn't add up).
  // Per A.3: do not silently correct. Trip loudly so the client confirms the right number.
  const vatErrors = [];
  if (o.amount_incl > 0 && o.amount_excl > 0) {
    const computedIncl = o.amount_excl + o.vat;
    if (Math.abs(computedIncl - o.amount_incl) > 0.5) {
      vatErrors.push(`VAT math broken: "${o.description}" (${o.section}) — excl ${o.amount_excl} + vat ${o.vat} = ${computedIncl}, but incl is ${o.amount_incl}`);
    }
    if (o.amount_incl < o.amount_excl) {
      vatErrors.push(`incl < excl: "${o.description}" (${o.section}) — excl ${o.amount_excl}, incl ${o.amount_incl}`);
    }
    if (o.amount_excl > 0 && (o.vat / o.amount_excl) > 0.20) {
      vatErrors.push(`VAT > 20%: "${o.description}" (${o.section}) — ${((o.vat / o.amount_excl) * 100).toFixed(1)}%`);
    }
  }
  // A.3: hard fail on VAT errors. Move them to FATAL so seed exits non-zero.
  if (vatErrors.length > 0) {
    ohErrors.push(...vatErrors.map((v) => `FATAL-VAT: ${v}`));
    VAT_WARN.push(...vatErrors);
  }

  try {
    await pb.collection(OVERHEADS_COLL).create({
      section: o.section,
      description: o.description,
      gl_account: o.gl_account || "",
      supplier: o.supplier || "",
      qty: o.qty || 0,
      unit_price: o.unit_price || 0,
      amount_excl: o.amount_excl || 0,
      vat: o.vat || 0,
      amount_incl: o.amount_incl || 0,
    });
    ohRows++;
  } catch (e) {
    ohErrors.push(`overhead "${o.description}": ${e.response?.data?.message || e.message}`);
  }
}
console.log(`  ✓ ${ohRows} overhead rows inserted`);

if (VAT_WARN.length > 0) {
  console.log(`\n  ⚠ ${VAT_WARN.length} VAT anomalies (Part B.3 — flagged, NOT corrected):`);
  for (const w of VAT_WARN) console.log(`    - ${w}`);
}

// === Insert invoices + invoice lines ===
// Every data/source/invoice-*.json is picked up automatically, so dropping a new
// invoice PDF's transcription in that folder is all it takes to add another one.
//
// The line items are transcribed from the source PDF, so the arithmetic is re-checked
// here rather than trusted. Per the project's standing rule: never silently correct a
// figure to make it balance — trip loudly and let the client confirm.
console.log(`\n→ Seeding ${invoices.length} invoice(s)`);
const invErrors = [];
const round2 = (n) => Math.round(n * 100) / 100;
let invoicesSeeded = 0;
let lineRows = 0;

for (const inv of invoices) {
  const where = inv.source_file || inv.invoice_no;
  // Scoped per invoice: a warning on one invoice must never block the next one.
  // (Using the cumulative array here silently skipped every invoice after the first
  // one that had an unbooked line.)
  const errs = [];

  // Header arithmetic. `subtotal` is the ex-VAT hire, `vat` is the "PLUS VAT" column,
  // and `total_incl` is the VAT-inclusive figure the debtor actually pays.
  const sumHire = round2(inv.lines.reduce((s, l) => s + l.amount_excl, 0));
  const sumVat = round2(inv.lines.reduce((s, l) => s + l.vat, 0));
  if (sumHire !== round2(inv.subtotal)) {
    errs.push(`${where}: hire lines total ${sumHire} but invoice subtotal is ${inv.subtotal}`);
  }
  if (sumVat !== round2(inv.vat)) {
    errs.push(`${where}: VAT lines total ${sumVat} but invoice PLUS VAT is ${inv.vat}`);
  }
  if (round2(inv.subtotal + inv.escalation) !== round2(inv.total_excl)) {
    errs.push(`${where}: subtotal ${inv.subtotal} + escalation ${inv.escalation} != total_excl ${inv.total_excl}`);
  }
  if (round2(inv.total_excl + inv.vat) !== round2(inv.total_incl)) {
    errs.push(`${where}: total_excl ${inv.total_excl} + vat ${inv.vat} != total_incl ${inv.total_incl}`);
  }
  if (inv.vat > 0 && inv.vat_pct && inv.vat / inv.total_excl > inv.vat_pct / 100 + 0.001) {
    errs.push(`${where}: VAT ${inv.vat} on excl ${inv.total_excl} exceeds the stated ${inv.vat_pct}% rate`);
  }

  // Per-line arithmetic
  for (const l of inv.lines) {
    if (round2(l.amount_excl + l.vat) !== round2(l.total_incl)) {
      errs.push(`${where} line ${l.line_no} (${l.machine_code}) excl ${l.amount_excl} + vat ${l.vat} != incl ${l.total_incl}`);
    }
    if (inv.vat_pct) {
      const expected = round2(l.amount_excl * inv.vat_pct / 100);
      if (Math.abs(expected - l.vat) > 0.02) {
        errs.push(`${where} line ${l.line_no} (${l.machine_code}) VAT is ${l.vat}, but ${inv.vat_pct}% of ${l.amount_excl} is ${expected}`);
      }
    }
    if (l.qty && l.rate && round2(l.qty * l.rate) !== round2(l.amount_excl)) {
      errs.push(`${where} line ${l.line_no} (${l.machine_code}) qty ${l.qty} x rate ${l.rate} = ${round2(l.qty * l.rate)}, but amount is ${l.amount_excl}`);
    }
  }

  if (errs.length) {
    // One broken invoice must not block the others — report it and move on.
    invErrors.push(...errs);
    console.error(`  ✗ ${where}: NOT seeded —`);
    for (const e of errs) console.error(`      - ${e}`);
    continue;
  }

  const periodEnd = new Date(inv.period_to);
  let invoiceRec = null;
  try {
    invoiceRec = await pb.collection(INVOICES_COLL).create({
      invoice_no: inv.invoice_no,
      invoice_date: inv.invoice_date,
      order_no: inv.order_no,
      debtor: inv.debtor,
      debtor_no: inv.debtor_no,
      period_from: inv.period_from,
      period_to: inv.period_to,
      description: inv.description,
      site: inv.site || "",
      section: inv.section,
      category: inv.category,
      currency: inv.currency,
      vat_reg_no: inv.vat_reg_no,
      source_file: inv.source_file,
      subtotal: inv.subtotal,
      escalation: inv.escalation,
      vat: inv.vat,
      vat_pct: inv.vat_pct,
      total_excl: inv.total_excl,
      total_incl: inv.total_incl,
      booked_month: periodEnd.getUTCMonth() + 1,
      booked_year: periodEnd.getUTCFullYear(),
      is_booked: false,
    });
  } catch (e) {
    invErrors.push(`${where}: header failed — ${e.response?.data?.message || e.message}`);
    continue;
  }

  invoicesSeeded++;
  let okLines = 0;
  for (const l of inv.lines) {
    // An unmatched line is seeded with booked_asset EMPTY rather than guessed, so it
    // shows on the page as "unbooked" instead of silently charging the wrong machine.
    let assetId = l.asset_code ? codeToId[l.asset_code] : null;
    if (!assetId) {
      const why = l.asset_code
        ? `no asset matched code "${l.asset_code}"`
        : "no target asset assigned";
      invErrors.push(`UNBOOKED: ${where} line ${l.line_no} "${l.machine_code}" (${l.description}) — ${why}. ${l._unmatched || ""}`);
    }
    try {
      await pb.collection(INVOICE_LINES_COLL).create({
        invoice: invoiceRec.id,
        line_no: l.line_no,
        machine_code: l.machine_code,
        asset_code: l.asset_code || "",
        description: l.description,
        qty: l.qty,
        unit: l.unit,
        rate: l.rate,
        rate_type: l.rate_type,
        amount_excl: l.amount_excl,
        vat: l.vat,
        total_incl: l.total_incl,
        booked_asset: assetId || "",
        booked_month: invoiceRec.booked_month,
        booked_year: invoiceRec.booked_year,
      });
      okLines++;
      lineRows++;
    } catch (e) {
      invErrors.push(`${where} line ${l.line_no} ${l.machine_code}: ${e.response?.data?.message || e.message}`);
    }
  }
  console.log(`  ✓ ${inv.invoice_no}${inv.site ? ` (${inv.site})` : ""} — ${okLines}/${inv.lines.length} lines, total R${inv.total_incl.toFixed(2)} incl VAT`);
}

console.log("\n=== Summary ===");
console.log(`Assets: ${ok}/${assets.length}`);
console.log(`Cost lines: ${clRows}`);
console.log(`Income: ${incomeRows}`);
console.log(`Running: ${runRows}`);
console.log(`Overheads: ${ohRows}`);
console.log(`Invoices: ${invoicesSeeded}/${invoices.length} (+ ${lineRows} lines)`);

const totalErrors = seedErrors.length + clErrors.length + incomeErrors.length + runErrors.length + ohErrors.length + invErrors.length;
if (totalErrors > 0) {
  console.error(`\n✗ ${totalErrors} errors — seed FAILED loudly:`);
  for (const e of [...seedErrors, ...clErrors, ...incomeErrors, ...runErrors, ...ohErrors, ...invErrors]) console.error(`  - ${e}`);
  if (invErrors.length) {
    console.error("\n  Fix the target asset in the matching data/source/invoice-*.json, or assign it");
    console.error("  in /income/invoices (Booked to), then re-run: node scripts/book-invoice.mjs");
  }
  console.error("\n  ⚠ monthly_income was wiped and re-imported from the workbook only.");
  console.error("    Invoice-derived income is NOT in it — re-run `node scripts/book-invoice.mjs` after fixing the above.");
  process.exit(2);
}
console.log("\n✅ Seed complete");
console.log("   Invoice lines are seeded but NOT yet in monthly_income.");
console.log("   Run `node scripts/book-invoice.mjs` to post them.");