MEGAINFRA-WEBAPP/scripts/dn-backfill-dates.py

#!/usr/bin/env python3
"""
Fill in the dates that were read but never kept.

  1. dn_batch_notes: date, received_by_name, site_name_raw, customer_vat_no,
     supplier_registration_raw, taken from the stored page readings exactly
     as the pipeline aggregates them (first non-empty value over the
     document's pages; a person's manual override wins).
  2. delivery_notes: posted_at = the old date_received (when it was booked),
     then date_received = the slip's delivery date.
  3. purchase_orders: order_date = "PO Creation Date" printed on the PO PDF.

A slip date is only used when it is plausible for that upload: not after the
upload, and at most 180 days before it. Anything else (a misread year,
"2024-04" on a 2026 slip) is left empty and listed, never guessed.

Dry run by default: prints what it would change. --apply writes, after
saving every value it replaces to --backup (JSON).

    python3 scripts/dn-backfill-dates.py
    python3 scripts/dn-backfill-dates.py --apply --backup /path/backup.json
"""
import argparse
import json
import re
import subprocess
import sys
import tempfile
import urllib.parse
import urllib.request
from datetime import date, datetime, timedelta

from pb_env import PB, http, superuser_token

HEADER = ["date", "received_by_name", "site_name_raw", "customer_vat_no", "supplier_registration_raw"]
PO_DATE = re.compile(r"(?:PO\s*Creation\s*Date|Order\s*Date|PO\s*Date)\s*:?\s*(\d{1,2})[/.-](\d{1,2})[/.-](\d{4})", re.I)


def fetch_all(token, coll, fields, flt=""):
    out, page = [], 1
    while True:
        q = {"fields": fields, "perPage": 500, "page": page}
        if flt:
            q["filter"] = flt
        code, body = http("GET", f"/api/collections/{coll}/records?" + urllib.parse.urlencode(q), token=token)
        if code != 200:
            sys.exit(f"GET {coll} failed: {code} {body}")
        out += body["items"]
        if page >= body["totalPages"]:
            return out
        page += 1


def iso_day(value):
    if not isinstance(value, str):
        return None
    m = re.match(r"^(\d{4})-(\d{2})-(\d{2})", value.strip())
    if not m:
        return None
    try:
        return date(int(m[1]), int(m[2]), int(m[3]))
    except ValueError:
        return None


def plausible(slip_day, uploaded):
    if not slip_day or not uploaded:
        return False
    return uploaded - timedelta(days=180) <= slip_day <= uploaded + timedelta(days=1)


def po_order_date(token, files_token, po):
    if not po.get("purchase_order_file") or not po["purchase_order_file"].lower().endswith(".pdf"):
        return None
    url = f"{PB}/api/files/{po['collectionId']}/{po['id']}/{urllib.parse.quote(po['purchase_order_file'])}?token={files_token}"
    with tempfile.NamedTemporaryFile(suffix=".pdf") as f:
        f.write(urllib.request.urlopen(url).read())
        f.flush()
        text = subprocess.run(["pdftotext", "-l", "2", f.name, "-"], capture_output=True, text=True).stdout
    m = PO_DATE.search(text)
    if not m:
        return None
    try:
        return date(int(m[3]), int(m[2]), int(m[1]))
    except ValueError:
        return None


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--apply", action="store_true")
    ap.add_argument("--backup", default="")
    args = ap.parse_args()
    if args.apply and not args.backup:
        sys.exit("--apply needs --backup <file> for the values it replaces")
    token = superuser_token()

    batches = {b["id"]: b for b in fetch_all(token, "dn_batches", "id,uploaded_at,created")}
    pages = fetch_all(token, "dn_batch_pages", "id,batch,global_page,extraction")
    reading = {(p["batch"], p.get("global_page")): (p.get("extraction") or {}) for p in pages}
    notes = fetch_all(token, "dn_batch_notes", "id,batch,pages,doc_number,manual_overrides," + ",".join(HEADER))

    note_writes, note_day, rejected = [], {}, []
    for n in notes:
        b = batches.get(n["batch"]) or {}
        uploaded = iso_day(b.get("uploaded_at") or b.get("created"))
        reads = [reading.get((n["batch"], g), {}) for g in sorted(n.get("pages") or [])]
        overrides = n.get("manual_overrides") or {}
        values = {}
        for key in HEADER:
            o = overrides.get(key) if isinstance(overrides, dict) else None
            v = o.strip() if isinstance(o, str) and o.strip() else next((r.get(key) for r in reads if isinstance(r.get(key), str) and r.get(key).strip()), "")
            values[key] = (v or "").strip()
        day = iso_day(values["date"])
        if values["date"] and not plausible(day, uploaded):
            rejected.append((n.get("doc_number"), values["date"], str(uploaded)))
            values["date"] = ""
            day = None
        if day:
            note_day[n["id"]] = day
        change = {k: v for k, v in values.items() if v and v != (n.get(k) or "")}
        if change:
            note_writes.append((n, change))

    receipts = fetch_all(token, "delivery_notes", "id,delivery_note_number,batch_note,date_received,posted_at")
    receipt_writes, no_slip_date = [], []
    for d in receipts:
        change = {}
        if not d.get("posted_at") and d.get("date_received"):
            change["posted_at"] = d["date_received"]
        day = note_day.get(d.get("batch_note") or "")
        if day:
            target = f"{day.isoformat()} 00:00:00.000Z"
            if (d.get("date_received") or "")[:10] != day.isoformat():
                change["date_received"] = target
        else:
            no_slip_date.append(d.get("delivery_note_number"))
        if change:
            receipt_writes.append((d, change))

    pos = fetch_all(token, "purchase_orders", "id,collectionId,purchase_order_number,purchase_order_file,order_date")
    code, ft = http("POST", "/api/files/token", token=token)
    files_token = ft["token"]
    po_writes, po_unread = [], []
    for p in pos:
        if p.get("order_date"):
            continue
        day = po_order_date(token, files_token, p)
        if day:
            po_writes.append((p, {"order_date": f"{day.isoformat()} 00:00:00.000Z"}))
        else:
            po_unread.append(p["purchase_order_number"])

    print(f"delivery documents: {len(notes)}, to fill: {len(note_writes)}, with a slip date: {len(note_day)}")
    print(f"  slip dates rejected as implausible: {len(rejected)}")
    for r in rejected:
        print(f"    {r[0]}: read {r[1]}, uploaded {r[2]}")
    moved = sum(1 for _, c in receipt_writes if "date_received" in c)
    print(f"booked receipts: {len(receipts)}, date_received moved to the slip date: {moved}, posted_at kept: {sum(1 for _, c in receipt_writes if 'posted_at' in c)}")
    print(f"  receipts with no usable slip date (left as they are): {len(no_slip_date)} {no_slip_date[:20]}")
    print(f"purchase orders: {len(pos)}, order date read: {len(po_writes)}, not readable: {len(po_unread)} {po_unread}")

    if not args.apply:
        print("\nDry run. Nothing written.")
        return

    backup = {
        "notes": [{"id": n["id"], **{k: n.get(k) or "" for k in HEADER}} for n, _ in note_writes],
        "receipts": [{"id": d["id"], "date_received": d.get("date_received"), "posted_at": d.get("posted_at")} for d, _ in receipt_writes],
        "purchase_orders": [{"id": p["id"], "order_date": p.get("order_date") or ""} for p, _ in po_writes],
        "at": datetime.utcnow().isoformat() + "Z",
    }
    with open(args.backup, "w") as f:
        json.dump(backup, f, indent=1)
    print(f"backup of replaced values: {args.backup}")
    for coll, writes in (("dn_batch_notes", note_writes), ("delivery_notes", receipt_writes), ("purchase_orders", po_writes)):
        failed = 0
        for rec, change in writes:
            code, body = http("PATCH", f"/api/collections/{coll}/records/{rec['id']}", change, token)
            if code != 200:
                failed += 1
                print(f"  ! {coll} {rec['id']}: {code} {body}")
        print(f"  ✓ {coll}: {len(writes) - failed} updated, {failed} failed")


if __name__ == "__main__":
    main()