MEGAINFRA-WEBAPP/scripts/migrations/2026-09-15-drop-legacy-columns.py

#!/usr/bin/env python3
"""
Drop the columns and collections the restructure made redundant.

  PB_URL=... PB_ADMIN_EMAIL=... PB_ADMIN_PASSWORD=... \\
  python3 scripts/migrations/2026-09-15-drop-legacy-columns.py [--apply]

DRY RUN BY DEFAULT. THIS STEP IS NOT REVERSIBLE. Take a backup first:
  python3 scripts/pb-backup-collection.py <collection>

Drops:
  project_code from 8 child collections - a text copy of the parent
      site's code, never updated, documented as a drift risk in
      docs/data-model.md. On purchase_orders it never held a project code
      at all: production had requisition numbers, site names and the PO's
      own number in it.
  purchase_order_items.qty_outstanding - written once at ingest and never
      decremented, so it permanently reported nothing delivered.
      Replaced by src/lib/po-fulfilment.ts.
  PLANT collection - 0 rows, 0 call sites, deprecated alias for vehicles.
  DATA_IMPORT_BATCHES collection - replaced by the five document
      collections.

Every drop is preceded by a guard that aborts the whole run. We would
rather leave a dead column in place than delete the last copy of
something.
"""
import json
import os
import sys
import urllib.error
import urllib.parse
import urllib.request

APPLY = "--apply" in sys.argv

DROP_PROJECT_CODE_FROM = [
    "employees", "vehicles", "fixed_costs", "income_claims",
    "purchase_orders", "delivery_notes", "fuel_logs", "plant_hours_log",
]
DROP_FIELDS = {
    "purchase_order_items": ["qty_outstanding"],
    # PocketBase refuses to delete DATA_IMPORT_BATCHES while any relation
    # still points at it. Task 7 migrated every row these fields could
    # ever reference onto fuel_sheet/plant_hours_sheet/timesheet; nothing
    # in src/ reads or writes batch_reference any more (verified live,
    # 2026-09-11 — grep shows only three stale type declarations in
    # pb-types.ts, no actual field access). Guard #4 below already proves
    # no row was left depending on the old link before this can run.
    "fuel_logs": ["batch_reference"],
    "plant_hours_log": ["batch_reference"],
    "timesheet_entries": ["batch_reference"],
}
DROP_COLLECTIONS = ["PLANT", "DATA_IMPORT_BATCHES"]

# DATA_IMPORT_BATCHES may only go once its files live in the documents.
EXPECTED_DOC_COUNTS = {
    "timesheets": 5,
    "fuel_sheets": 3,
    "plant_hours_sheets": 19,
}


def admin_auth(base_url):
    body = json.dumps(
        {
            "identity": os.environ["PB_ADMIN_EMAIL"],
            "password": os.environ["PB_ADMIN_PASSWORD"],
        }
    ).encode()
    req = urllib.request.Request(
        f"{base_url}/api/collections/_superusers/auth-with-password",
        data=body, headers={"Content-Type": "application/json"}, method="POST",
    )
    with urllib.request.urlopen(req) as resp:
        return json.loads(resp.read())["token"]


def api(base_url, token, method, path, body=None):
    """
    A 404 is only soft for GET — "this collection/record is absent" is a
    legitimate answer to a read. On a write it is not: a PATCH/DELETE
    against a since-deleted collection returning None would be counted
    as a successful mutation when nothing happened. Mutations raise.
    """
    data = json.dumps(body).encode() if body is not None else None
    req = urllib.request.Request(
        f"{base_url}{path}", data=data, method=method,
        headers={"Authorization": token, "Content-Type": "application/json"},
    )
    try:
        with urllib.request.urlopen(req) as resp:
            txt = resp.read().decode()
            return json.loads(txt) if txt else None
    except urllib.error.HTTPError as e:
        if e.code == 404 and method == "GET":
            return None
        raise RuntimeError(
            f"{method} {path}: {e.code} {e.read().decode() if e.fp else ''}"
        ) from e


def count(base_url, token, coll, flt=""):
    path = f"/api/collections/{coll}/records?perPage=1&fields=id"
    if flt:
        path += "&filter=" + urllib.parse.quote(flt)
    got = api(base_url, token, "GET", path)
    return got["totalItems"] if got else 0


def guard(base_url, token):
    """Every reason to refuse. Returns a list of problems; empty means go."""
    problems = []

    # 1. project_code may only go once site is populated everywhere.
    for coll in DROP_PROJECT_CODE_FROM:
        n = count(base_url, token, coll, 'site=""')
        if n:
            problems.append(
                f"{coll}: {n} rows have no site — dropping project_code "
                f"would leave them with no site attribution at all"
            )

    # 2. PLANT must be empty. It should be; check anyway.
    n = count(base_url, token, "PLANT")
    if n:
        problems.append(f"PLANT: {n} rows — expected 0, refusing to drop")

    # 3. The batch files must already live in the documents.
    for coll, expected in EXPECTED_DOC_COUNTS.items():
        n = count(base_url, token, coll)
        if n < expected:
            problems.append(
                f"{coll}: {n} rows, expected at least {expected} — "
                f"Task 7 migration has not run, batch files would be lost"
            )

    # 4. Nothing that USED TO point at a batch may be left unrelinked.
    #    Checking "field is empty" alone is the wrong condition: fuel_logs
    #    has 73 rows that never had a batch_reference at all (they predate
    #    batch tracking entirely — verified live on 2026-09-11). Those rows
    #    have nothing to lose when data_import_batches goes; the risk is
    #    only real for a row that HAD a batch_reference and Task 7's
    #    migration failed to relink. Check that condition, not the proxy.
    for coll, field in [
        ("fuel_logs", "fuel_sheet"),
        ("plant_hours_log", "plant_hours_sheet"),
        ("timesheet_entries", "timesheet"),
    ]:
        n = count(
            base_url, token, coll,
            f'{field}="" && batch_reference != ""',
        )
        if n:
            problems.append(
                f"{coll}: {n} rows had a batch_reference but were never "
                f"relinked to {field} — they would lose their source "
                f"document"
            )

    return problems


def main():
    base_url = os.environ.get("PB_URL", "http://127.0.0.1:8090").rstrip("/")
    token = admin_auth(base_url)
    # Say where we are writing. This script mutates production by default
    # target, and a run whose output names no host is a run you cannot
    # audit afterwards.
    print(f"target: {base_url}   mode: {'APPLY' if APPLY else 'DRY RUN'}")

    print("\n=== GUARDS ===")
    problems = guard(base_url, token)
    if problems:
        print("ABORT. Nothing was dropped:")
        for p in problems:
            print(f"   - {p}")
        return 1
    print("   all clear")

    print("\n=== PLAN ===")
    for coll in DROP_PROJECT_CODE_FROM:
        print(f"   drop {coll}.project_code")
    for coll, fields in DROP_FIELDS.items():
        for f in fields:
            print(f"   drop {coll}.{f}")
    for coll in DROP_COLLECTIONS:
        existing = api(base_url, token, "GET", f"/api/collections/{coll}")
        state = f"{count(base_url, token, coll)} rows" if existing else "absent"
        print(f"   DELETE COLLECTION {coll}  ({state})")

    if not APPLY:
        print("\nDRY RUN. Re-run with --apply. THIS IS NOT REVERSIBLE.")
        return 0

    # Fields first, collections last: if a field drop fails we still have
    # the collections, and the run can be repeated.
    to_drop = {c: ["project_code"] for c in DROP_PROJECT_CODE_FROM}
    for c, fields in DROP_FIELDS.items():
        to_drop.setdefault(c, []).extend(fields)

    for coll, names in to_drop.items():
        cur = api(base_url, token, "GET", f"/api/collections/{coll}")
        if cur is None:
            print(f"   skip {coll}: not found")
            continue
        kept = [f for f in cur["fields"] if f["name"] not in names]
        if len(kept) == len(cur["fields"]):
            print(f"   skip {coll}: {names} already absent")
            continue
        api(base_url, token, "PATCH",
            f"/api/collections/{cur['id']}", {"fields": kept})
        print(f"   dropped {names} from {coll}")

    for coll in DROP_COLLECTIONS:
        cur = api(base_url, token, "GET", f"/api/collections/{coll}")
        if cur is None:
            print(f"   skip {coll}: already gone")
            continue
        api(base_url, token, "DELETE", f"/api/collections/{cur['id']}")
        print(f"   DELETED COLLECTION {coll}")

    print("\nDone.")
    return 0


if __name__ == "__main__":
    try:
        sys.exit(main())
    except KeyError as e:
        print(f"ERROR: missing environment variable {e}", file=sys.stderr)
        sys.exit(2)