diff --git a/scripts/pb-add-dn-batches.py b/scripts/pb-add-dn-batches.py new file mode 100755 index 0000000..3bc9eea --- /dev/null +++ b/scripts/pb-add-dn-batches.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python3 +""" +Creates the DN_BATCHES and DN_BATCH_NOTES collections, and adds the +batch back-references to DELIVERY_NOTES. + +A batch is the uploaded file. Notes found inside it live in +DN_BATCH_NOTES as candidates until they pass every check — only then is +a DELIVERY_NOTES row created. That separation is the point: before it, +delivery_notes was both the inbox and the ledger, so a half-read +document sat inside the figures used to pay suppliers. + +Idempotent — existing collections and fields are left alone. +""" +import sys + +from pb_env import PB, http, superuser_token + + +def get_collection(token, name): + code, resp = http("GET", f"/api/collections/{name}", token=token) + return (resp if code == 200 else None) + + +def collection_id(token, name): + coll = get_collection(token, name) + if not coll: + sys.exit(f"collection {name} not found — cannot resolve its id") + return coll["id"] + + +def create_collection(token, payload): + existing = get_collection(token, payload["name"]) + if existing: + print(f" • {payload['name']} already exists — skipping create") + return existing["id"] + code, resp = http("POST", "/api/collections", payload, token) + if code != 200: + sys.exit(f"CREATE {payload['name']} failed: {code} {resp}") + print(f" ✓ created {payload['name']}") + return resp["id"] + + +def add_fields(token, name, wanted): + coll = get_collection(token, name) + if not coll: + sys.exit(f"collection {name} not found") + fields = list(coll.get("fields", [])) + existing = {f.get("name") for f in fields} + additions = [w for w in wanted if w["name"] not in existing] + if not additions: + print(f" • {name}: no new fields needed") + return + code, resp = http("PATCH", f"/api/collections/{coll['id']}", {"fields": fields + additions}, token) + if code != 200: + sys.exit(f"PATCH {name} failed: {code} {resp}") + for a in additions: + print(f" ✓ {name}: added {a['name']}") + + +def main(): + token = superuser_token() + print(f"\n→ {PB}") + + sites_id = collection_id(token, "sites") + pos_id = collection_id(token, "purchase_orders") + dns_id = collection_id(token, "delivery_notes") + + batches_id = create_collection(token, { + "name": "DN_BATCHES", + "type": "base", + "fields": [ + {"name": "file", "type": "file", "required": False, "maxSelect": 1, "maxSize": 20971520}, + {"name": "file_hash", "type": "text", "required": False}, + {"name": "site", "type": "relation", "required": True, + "collectionId": sites_id, "maxSelect": 1, "cascadeDelete": False}, + {"name": "source", "type": "select", "required": False, "maxSelect": 1, + "values": ["web", "mobile", "email"]}, + {"name": "uploaded_by", "type": "text", "required": False}, + {"name": "status", "type": "select", "required": False, "maxSelect": 1, + "values": ["inbox", "analysing", "needs_attention", "processed"]}, + {"name": "page_count", "type": "number", "required": False}, + {"name": "notes_detected", "type": "number", "required": False}, + {"name": "notes_promoted", "type": "number", "required": False}, + {"name": "segmentation", "type": "json", "required": False, "maxSize": 2000000}, + {"name": "checks", "type": "json", "required": False, "maxSize": 2000000}, + {"name": "attempts", "type": "number", "required": False}, + {"name": "claimed_at", "type": "date", "required": False}, + {"name": "last_error", "type": "text", "required": False}, + ], + }) + + create_collection(token, { + "name": "DN_BATCH_NOTES", + "type": "base", + "fields": [ + {"name": "batch", "type": "relation", "required": True, + "collectionId": batches_id, "maxSelect": 1, "cascadeDelete": True}, + {"name": "note_index", "type": "number", "required": False}, + {"name": "page_from", "type": "number", "required": False}, + {"name": "page_to", "type": "number", "required": False}, + {"name": "page_file", "type": "file", "required": False, "maxSelect": 1, "maxSize": 20971520}, + {"name": "delivery_note_number", "type": "text", "required": False}, + {"name": "po_number_raw", "type": "text", "required": False}, + {"name": "po_number_normalised", "type": "text", "required": False}, + {"name": "purchase_order", "type": "relation", "required": False, + "collectionId": pos_id, "maxSelect": 1, "cascadeDelete": False}, + {"name": "gross_mass", "type": "number", "required": False}, + {"name": "tare_mass", "type": "number", "required": False}, + {"name": "nett_mass", "type": "number", "required": False}, + {"name": "quantity", "type": "number", "required": False}, + {"name": "unit", "type": "text", "required": False}, + {"name": "items", "type": "json", "required": False, "maxSize": 2000000}, + {"name": "checks", "type": "json", "required": False, "maxSize": 2000000}, + {"name": "status", "type": "select", "required": False, "maxSelect": 1, + "values": ["candidate", "blocked", "promoted"]}, + {"name": "delivery_note", "type": "relation", "required": False, + "collectionId": dns_id, "maxSelect": 1, "cascadeDelete": False}, + ], + }) + + add_fields(token, "delivery_notes", [ + {"name": "batch", "type": "relation", "required": False, + "collectionId": batches_id, "maxSelect": 1, "cascadeDelete": False}, + {"name": "page_from", "type": "number", "required": False}, + {"name": "page_to", "type": "number", "required": False}, + ]) + + +if __name__ == "__main__": + main() + print("\nDone.") diff --git a/src/lib/pb-types.ts b/src/lib/pb-types.ts index 10619f8..642b44d 100644 --- a/src/lib/pb-types.ts +++ b/src/lib/pb-types.ts @@ -336,6 +336,104 @@ export interface DeliveryNoteItemRecord extends RecordModel { }; } +/** + * The uploaded delivery-note document. One row per file (PDF or + * image) before any page-level work has happened — the inbox item, not + * the ledger row. A `DN_BATCH_NOTES` child row exists for every + * candidate note the splitter found inside it; only after those pass + * every check does a `DELIVERY_NOTES` row get promoted. + * + * Added by `scripts/pb-add-dn-batches.py`. + */ +export type DnBatchStatus = + | "inbox" + | "analysing" + | "needs_attention" + | "processed"; + +export interface DnBatchRecord extends RecordModel { + collectionName: "dn_batches"; + /** The uploaded PDF/image. Empty when only metadata was recorded. */ + file: string; + /** SHA-256 of `file`. Powers dedup against re-uploads of the same bytes. */ + file_hash: string; + /** Site the delivery belongs to. Required so n8n never has to guess. */ + site: string; + /** Upload channel. Empty when the script that wrote the row didn't set it. */ + source: "web" | "mobile" | "email" | ""; + /** Free-form operator id. Empty when uploaded by an unauthenticated flow. */ + uploaded_by: string; + /** Pipeline position. "inbox" until n8n claims it; "processed" once every child note is promoted or blocked. */ + status: DnBatchStatus; + /** Total pages in the source file. Set during ingestion; unchanged after. */ + page_count: number; + /** How many candidate notes the page splitter pulled out. */ + notes_detected: number; + /** Of those, how many became real DELIVERY_NOTES rows. */ + notes_promoted: number; + /** Raw segmentation output (per-page notes, bounding boxes, OCR text). Free-form. */ + segmentation: unknown; + /** Aggregated check results across the whole batch. Free-form. */ + checks: unknown; + /** n8n retry counter. */ + attempts: number; + /** ISO timestamp of the most recent n8n claim. Empty until first claim. */ + claimed_at: string; + /** Last error message from n8n. Empty when the batch is healthy. */ + last_error: string; + expand?: { site?: SiteRecord }; +} + +/** + * One candidate note found inside a `DN_BATCHES` row. Stays a candidate + * (status="candidate") until every check passes — only then is a real + * DELIVERY_NOTES row created and `delivery_note` is set. "blocked" notes + * are the ones that failed a check; "promoted" is the terminal state. + * + * Added by `scripts/pb-add-dn-batches.py`. + */ +export type DnBatchNoteStatus = "candidate" | "blocked" | "promoted"; + +export interface DnBatchNoteRecord extends RecordModel { + collectionName: "dn_batch_notes"; + /** Owning DN_BATCHES row. Required — cascade-deletes with the batch. */ + batch: string; + /** Position of this note within the batch (0-based). Stable across retries. */ + note_index: number; + /** First page of the source PDF/image this note came from. */ + page_from: number; + /** Last page of the source PDF/image this note came from. Inclusive. */ + page_to: number; + /** Cropped page image (PDF single-page or image) for re-OCR or operator review. */ + page_file: string; + /** As printed on the slip. Empty when OCR could not read it. */ + delivery_note_number: string; + /** PO number as OCR saw it (any case, any spacing). */ + po_number_raw: string; + /** PO number after normalisation (trim, collapse, uppercase). Used for the match lookup. */ + po_number_normalised: string; + /** Resolved purchase_orders row. Empty until the normalised PO matches a real PO. */ + purchase_order: string; + /** Gross mass (kg) from the slip, when present. */ + gross_mass: number; + /** Tare mass (kg) from the slip, when present. */ + tare_mass: number; + /** Nett mass (kg) from the slip, when present. */ + nett_mass: number; + /** Quantity from the slip in `unit`. Zero when the slip showed none. */ + quantity: number; + /** Verbatim unit from the slip, e.g. "kg" or "m³". Empty when absent. */ + unit: string; + /** Parsed line items for this note. Free-form until the items extractor is finalised. */ + items: unknown; + /** Per-check results (PO match, totals, duplicates, …). Free-form. */ + checks: unknown; + /** Promotion state. See DnBatchNoteStatus. */ + status: DnBatchNoteStatus; + /** Back-reference to the DELIVERY_NOTES row this note was promoted into. Empty while a candidate or blocked. */ + delivery_note: string; +} + export type DocumentStatus = | "pending" | "processing"