mega-plant/scripts/setup-invoice-collections.mjs
David 5caa923375 Invoices: the app ingests PDFs itself, with a problem-invoice inbox
Extraction was previously manual: an agent read each PDF, mapped the columns
by hand and wrote data/source/invoice-*.json. That was the only way anything
got in, and the upload button just filed the document without reading it.

That supplier's report is one fixed template — the columns sit at identical
x-positions on every invoice, and only the line count and page count vary —
so the figures are now read by coordinate instead. Deliberately not an AI
extractor: this input does not need one, and a deterministic parse that fails
loudly is worth more here than a flexible one that can hallucinate a total.

lib/invoice-parser.ts reads the header, the machine lines across all pages,
the "Plant Hire Total" row and the SUB TOTAL / Escalation / PLUS VAT /
INVOICE TOTAL block, then applies the same arithmetic gates the seed does:
lines must sum to the subtotal, each line must satisfy qty x rate = amount
and amount + VAT = total, and VAT must match the stated rate. Anything that
does not reconcile is refused with the reason — never partially imported.

Validated by parsing all five existing invoices and comparing every field
against the hand-extracted JSON: identical, including the R24,205.13-VAT
column that a plain text read gets wrong. PHI 169 was then deleted and
re-ingested through the HTTP endpoint alone and came back field-for-field
identical, with no agent involved.

- "Add Invoice" drops a PDF in and imports it; lines are matched to plant
  items by the code the invoice prints, and an unmatched line (the
  container codes MPLANT04/MPLANT05) is left unassigned rather than guessed.
- A "Problem invoices" inbox lists every PDF that failed, with the reason,
  and Retry re-reads the stored document and re-runs the parse. Retry
  re-reports rather than claiming success, and only files imported data.
- invoice_intake collection records each attempt.

pdfjs-dist is in serverExternalPackages: bundled, it rewrites its own lazy
worker import to a .next path absent from the standalone image and every
parse dies with "Setting up fake worker failed". The parser also copies the
buffer before handing it over, since pdfjs transfers and detaches it.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-10-02 14:30:17 +02:00

233 lines
No EOL
9.8 KiB
JavaScript

// Create the `invoices` + `invoice_lines` PocketBase collections.
//
// Idempotent: safe to re-run. Creates what is missing and reports drift instead of
// silently leaving a half-shaped schema behind (the project fails loudly rather than
// inventing data — see AGENTS.md).
//
// Usage: node scripts/setup-invoice-collections.mjs
import PocketBase from "pocketbase";
import { readFileSync, existsSync } from "node:fs";
import { fileURLToPath } from "node:url";
import { dirname, resolve } from "node:path";
const __dirname = dirname(fileURLToPath(import.meta.url));
const ROOT = resolve(__dirname, "..");
// Load .env.local the same way seed.mjs does — a script whose job is to shape the
// schema cannot fail on a missing URL and report success.
const envPath = resolve(ROOT, ".env.local");
if (!existsSync(envPath)) {
console.error("✗ .env.local not found. Copy .env.example to .env.local and fill it in.");
process.exit(1);
}
for (const line of readFileSync(envPath, "utf8").split("\n")) {
const t = line.trim();
if (!t || t.startsWith("#")) continue;
const i = t.indexOf("=");
if (i > 0) process.env[t.slice(0, i).trim()] = t.slice(i + 1).trim();
}
const PB_URL = process.env.NEXT_PUBLIC_PB_URL;
const EMAIL = process.env.PB_ADMIN;
const PASS = process.env.PB_PASS;
const missing = [];
if (!PB_URL) missing.push("NEXT_PUBLIC_PB_URL");
if (!EMAIL) missing.push("PB_ADMIN");
if (!PASS) missing.push("PB_PASS");
if (missing.length) {
console.error(`✗ Missing required env: ${missing.join(", ")}`);
process.exit(1);
}
const pb = new PocketBase(PB_URL);
pb.autoCancellation(false);
console.log(`→ Connecting to ${PB_URL} as ${EMAIL}`);
await pb.collection("_superusers").authWithPassword(EMAIL, PASS);
console.log(" ✓ authenticated");
// Mirror the rules the existing collections use so behaviour stays uniform.
const reference = await pb.collections.getOne("monthly_income");
const RULES = {
listRule: reference.listRule,
viewRule: reference.viewRule,
createRule: reference.createRule ?? null,
updateRule: reference.updateRule ?? null,
deleteRule: reference.deleteRule ?? null,
};
console.log(` · rules mirrored from monthly_income (list="${RULES.listRule}")`);
const assetsCollection = await pb.collections.getOne("assets");
const assetsId = assetsCollection.id;
async function ensureCollection(name, fields) {
const existing = await pb.collections.getFullList({ filter: `name = "${name}"`, perPage: 1 });
if (existing.length) {
const have = new Set(existing[0].fields.map((f) => f.name));
const missing = fields.filter((f) => !have.has(f.name));
if (!missing.length) {
console.log(` · ${name} already present (${existing[0].fields.length} fields)`);
} else {
// Grow the existing schema rather than refusing: the collection is additive and
// re-seeding rewrites its rows anyway. Report exactly what was added.
await updateSchema(name, (cur) => [...cur, ...missing], `added to ${name}: ${missing.map((f) => f.name).join(", ")}`);
}
return existing[0].id;
}
// NOTE: pb.collections.create() serialises an array schema into repeated `fields=`
// params, which PocketBase rejects with "invalid formatting". The collections API
// wants `fields` as one real JSON array inside a JSON body — so post it directly
// with a pre-stringified body, which the SDK passes through untouched.
const created = await pb.send("/api/collections", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
name,
type: "base",
fields,
listRule: RULES.listRule ?? "",
viewRule: RULES.viewRule ?? "",
createRule: RULES.createRule ?? "",
updateRule: RULES.updateRule ?? "",
deleteRule: RULES.deleteRule ?? "",
}),
});
console.log(` ✓ created ${name} (${created.id}) with ${created.fields.length} fields`);
return created.id;
}
// Adds one field to an existing collection if it is not there yet. PocketBase has no
// partial field API — the schema is replaced wholesale — so the current field list is
// read back and re-sent with the new field appended.
async function updateSchema(collectionName, transform, label) {
const coll = await pb.collections.getOne(collectionName);
const next = transform([...coll.fields]);
if (JSON.stringify(next) === JSON.stringify(coll.fields)) {
console.log(` · ${collectionName} schema already current`);
return;
}
await pb.send(`/api/collections/${coll.name}`, {
method: "PATCH",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ fields: next }),
});
console.log(` ✓ ${label}`);
}
async function ensureField(collectionName, field) {
await updateSchema(
collectionName,
(fields) => (fields.some((f) => f.name === field.name) ? fields : [...fields, field]),
`added ${collectionName}.${field.name}`
);
}
// Drops fields that a previous version of the script created and that are no longer
// part of the model. Explicit list only — never prune anything else.
async function dropField(collectionName, ...names) {
await updateSchema(
collectionName,
(fields) => fields.filter((f) => !names.includes(f.name)),
`dropped ${collectionName}.${names.join(", ")}`
);
}
console.log("\n→ Ensuring collections");
const invoicesId = await ensureCollection("invoices", [
{ name: "invoice_no", type: "text", required: true },
{ name: "invoice_date", type: "date", required: true },
{ name: "order_no", type: "text" },
{ name: "debtor", type: "text" },
{ name: "debtor_no", type: "text" },
{ name: "period_from", type: "date" },
{ name: "period_to", type: "date" },
{ name: "description", type: "text" },
{ name: "site", type: "text" },
{ name: "section", type: "text" },
{ name: "category", type: "text" },
{ name: "currency", type: "text" },
{ name: "vat_reg_no", type: "text" },
{ name: "source_file", type: "text" },
{ name: "subtotal", type: "number" },
{ name: "escalation", type: "number" },
{ name: "vat", type: "number" },
{ name: "vat_pct", type: "number" },
{ name: "total_excl", type: "number" },
{ name: "total_incl", type: "number" },
// Original document, uploaded from the Invoices page (drag-and-drop).
{ name: "source_pdf", type: "file", maxSelect: 1, maxSize: 20971520 },
{ name: "booked_month", type: "number" },
{ name: "booked_year", type: "number" },
{ name: "is_booked", type: "bool" },
]);
const invoiceLinesId = await ensureCollection("invoice_lines", [
{ name: "invoice", type: "relation", required: true, collectionId: invoicesId, minSelect: 1, maxSelect: 1, cascadeDelete: true },
{ name: "line_no", type: "number", required: true },
{ name: "machine_code", type: "text" },
{ name: "asset_code", type: "text" },
{ name: "description", type: "text" },
{ name: "qty", type: "number" },
{ name: "unit", type: "text" },
{ name: "rate", type: "number" },
{ name: "rate_type", type: "text" },
{ name: "amount_excl", type: "number" },
{ name: "vat", type: "number" },
{ name: "total_incl", type: "number" },
// The machine this line is actually booked against. Deliberately nullable: an
// invoice line whose target asset is unresolved must stay visibly unbooked
// rather than be silently charged to a guess.
{ name: "booked_asset", type: "relation", collectionId: assetsId, minSelect: 1, maxSelect: 1, cascadeDelete: false },
{ name: "booked_month", type: "number" },
{ name: "booked_year", type: "number" },
]);
// Provenance back-reference. `npm run seed` wipes monthly_income, so posting an
// invoice has to be re-runnable without double-counting. Marking each posted row with
// the invoice line it came from makes booking idempotent and auditable: a row with an
// empty `invoice_line` is workbook data, a row with one is invoice-derived.
// The intake inbox. Every PDF that is offered to the app lands here first, with the
// outcome of its parse. Anything that does not reconcile stays as `failed` with the
// reason recorded, so it can be reviewed and retried rather than silently lost.
await ensureCollection("invoice_intake", [
{ name: "file_name", type: "text" },
{ name: "file_size", type: "number" },
{ name: "invoice_no", type: "text" },
{ name: "status", type: "select", required: true, maxSelect: 1, values: ["imported", "failed"] },
{ name: "error", type: "text" },
// Extracted-but-unverified data, kept so a failed import can still be inspected.
{ name: "parsed", type: "json" },
{ name: "source_pdf", type: "file", maxSelect: 1, maxSize: 20971520 },
{ name: "invoice", type: "relation", collectionId: invoicesId, minSelect: 1, maxSelect: 1, cascadeDelete: false },
// Collections created through the API do NOT get PocketBase's automatic created/updated
// fields the way the UI wizard adds them. Without these, any `sort: "-created"` fails
// server-side with "Something went wrong while processing your request".
{ name: "created", type: "date", onCreate: true, presentable: false },
{ name: "updated", type: "date", onUpdate: true, presentable: false },
]);
await ensureField("monthly_income", {
name: "invoice_line",
type: "relation",
collectionId: invoiceLinesId,
minSelect: 1,
maxSelect: 1,
cascadeDelete: false,
});
// `escalation_pct` and `escalation_amount` came from a misreading of the source PDF:
// the R24,205.13 column is literally headed "PLUS VAT", and the Escalation column is
// R0.00 on every line. Superseded by vat_pct / vat.
await dropField("invoices", "escalation_pct");
await dropField("invoice_lines", "escalation_amount");
console.log("\n=== Summary ===");
console.log(`invoices ${invoicesId}`);
console.log(`invoice_lines -> invoices (cascade delete)`);
console.log(`invoice_lines -> assets (booked_asset)`);
if (process.exitCode) {
console.error("\n✗ Schema drift detected — see above.");
} else {
console.log("\n✅ Invoice collections ready");
}