MEGAINFRA-WEBAPP/scripts/migrations/2026-09-15-add-indexes.py

#!/usr/bin/env python3
"""
Add the missing indexes.

  PB_URL=... PB_ADMIN_EMAIL=... PB_ADMIN_PASSWORD=... \\
  python3 scripts/migrations/2026-09-15-add-indexes.py [--apply]

DRY RUN BY DEFAULT.

`timesheet_entries`, `fuel_logs` and `delivery_notes` have no indexes at
all. Fine at today's volumes; not once every per-site query does a full
table scan at 18-20 sites.

`purchase_orders` and `plant_hours_log` already carry a load-bearing
UNIQUE index each (`idx_po_number`, `idx_plant_hours_log_qr_start`).
Every collection's `indexes` array is PATCHed by unioning the new index
statements in, never by replacing the array — the existing UNIQUE
indexes are read live and carried through byte-for-byte unchanged.
"""
import json
import os
import sys
import urllib.error
import urllib.request

APPLY = "--apply" in sys.argv

# collection -> list of new CREATE INDEX statements to union in.
NEW_INDEXES = {
    "FUEL_LOGS": [
        "CREATE INDEX `idx_fuel_logs_site_date` ON `FUEL_LOGS` (`site`, `date_logged`)",
        "CREATE INDEX `idx_fuel_logs_vehicle_date` ON `FUEL_LOGS` (`vehicle`, `date_logged`)",
    ],
    "TIMESHEET_ENTRIES": [
        "CREATE INDEX `idx_ts_entries_site_date` ON `TIMESHEET_ENTRIES` (`site`, `date_logged`)",
        "CREATE INDEX `idx_ts_entries_emp_date` ON `TIMESHEET_ENTRIES` (`employee`, `date_logged`)",
    ],
    "DELIVERY_NOTES": [
        "CREATE INDEX `idx_dn_site_date` ON `DELIVERY_NOTES` (`site`, `date_received`)",
        "CREATE INDEX `idx_dn_status` ON `DELIVERY_NOTES` (`processing_status`)",
    ],
    "PURCHASE_ORDERS": [
        "CREATE INDEX `idx_po_site_status` ON `PURCHASE_ORDERS` (`site`, `processing_status`)",
    ],
    "PLANT_HOURS_LOG": [
        "CREATE INDEX `idx_ph_log_vehicle_start` ON `PLANT_HOURS_LOG` (`vehicle`, `start_date`)",
    ],
}

# Pre-existing UNIQUE indexes that must survive this run unchanged. Used
# only to print a before/after diff for the record - the union logic
# below never touches an existing array entry regardless of what it is.
PREEXISTING_UNIQUE = {
    "PURCHASE_ORDERS": "idx_po_number",
    "PLANT_HOURS_LOG": "idx_plant_hours_log_qr_start",
}


def admin_auth(base_url):
    body = json.dumps(
        {
            "identity": os.environ["PB_ADMIN_EMAIL"],
            "password": os.environ["PB_ADMIN_PASSWORD"],
        }
    ).encode()
    req = urllib.request.Request(
        f"{base_url}/api/collections/_superusers/auth-with-password",
        data=body, headers={"Content-Type": "application/json"}, method="POST",
    )
    with urllib.request.urlopen(req) as resp:
        return json.loads(resp.read())["token"]


def api(base_url, token, method, path, body=None):
    """
    A 404 is only soft for GET - "this collection is absent" is a
    legitimate answer to a read. On a write it is not: a PATCH against a
    since-deleted collection returning None would be counted as a
    successful mutation when nothing happened. Mutations raise.
    """
    data = json.dumps(body).encode() if body is not None else None
    req = urllib.request.Request(
        f"{base_url}{path}", data=data, method=method,
        headers={"Authorization": token, "Content-Type": "application/json"},
    )
    try:
        with urllib.request.urlopen(req) as resp:
            txt = resp.read().decode()
            return json.loads(txt) if txt else None
    except urllib.error.HTTPError as e:
        if e.code == 404 and method == "GET":
            return None
        raise RuntimeError(
            f"{method} {path}: {e.code} {e.read().decode() if e.fp else ''}"
        ) from e


def index_name(stmt):
    # `CREATE [UNIQUE] INDEX `name` ON ...` -> name
    marker = "INDEX `"
    start = stmt.index(marker) + len(marker)
    return stmt[start:stmt.index("`", start)]


def main():
    base_url = os.environ.get("PB_URL", "http://127.0.0.1:8090").rstrip("/")
    token = admin_auth(base_url)
    # Say where we are writing. This script mutates production by
    # default target, and a run whose output names no host is a run you
    # cannot audit afterwards.
    print(f"target: {base_url}   mode: {'APPLY' if APPLY else 'DRY RUN'}")

    print("\n=== PRE-EXISTING UNIQUE INDEXES (must survive unchanged) ===")
    before = {}
    for coll, want_name in PREEXISTING_UNIQUE.items():
        cur = api(base_url, token, "GET", f"/api/collections/{coll}")
        if cur is None:
            print(f"   {coll}: COLLECTION NOT FOUND")
            continue
        idxs = cur.get("indexes", [])
        before[coll] = idxs
        matches = [i for i in idxs if index_name(i) == want_name]
        if not matches:
            print(f"   {coll}: MISSING {want_name} !!")
        else:
            print(f"   {coll}: {matches[0]}")

    print("\n=== PLAN ===")
    plan = {}
    for coll, new_stmts in NEW_INDEXES.items():
        cur = api(base_url, token, "GET", f"/api/collections/{coll}")
        if cur is None:
            print(f"   skip {coll}: not found")
            continue
        existing = cur.get("indexes", [])
        existing_names = {index_name(i) for i in existing}
        to_add = [s for s in new_stmts if index_name(s) not in existing_names]
        already = [s for s in new_stmts if index_name(s) in existing_names]
        plan[coll] = (cur, existing, to_add)
        for s in already:
            print(f"   {coll}: {index_name(s)} already present - skip")
        for s in to_add:
            print(f"   {coll}: ADD {index_name(s)}")

    if not APPLY:
        print("\nDRY RUN. Re-run with --apply.")
        return 0

    print("\n=== APPLY ===")
    for coll, (cur, existing, to_add) in plan.items():
        if not to_add:
            print(f"   skip {coll}: nothing to add")
            continue
        union = existing + to_add
        api(base_url, token, "PATCH",
            f"/api/collections/{cur['id']}", {"indexes": union})
        print(f"   {coll}: added {[index_name(s) for s in to_add]}")

    print("\n=== VERIFY ===")
    ok = True
    for coll, want_name in PREEXISTING_UNIQUE.items():
        cur = api(base_url, token, "GET", f"/api/collections/{coll}")
        idxs = cur.get("indexes", []) if cur else []
        matches = [i for i in idxs if index_name(i) == want_name]
        unchanged = matches and matches[0] in before.get(coll, [])
        print(f"   {coll}.{want_name}: "
              f"{'unchanged' if unchanged else 'CHANGED OR MISSING !!'}")
        ok = ok and bool(unchanged)

    for coll, new_stmts in NEW_INDEXES.items():
        cur = api(base_url, token, "GET", f"/api/collections/{coll}")
        idxs = cur.get("indexes", []) if cur else []
        names = {index_name(i) for i in idxs}
        for s in new_stmts:
            present = index_name(s) in names
            print(f"   {coll}.{index_name(s)}: {'present' if present else 'MISSING !!'}")
            ok = ok and present

    print("\nDone." if ok else "\nDone, WITH PROBLEMS ABOVE.")
    return 0 if ok else 1


if __name__ == "__main__":
    try:
        sys.exit(main())
    except KeyError as e:
        print(f"ERROR: missing environment variable {e}", file=sys.stderr)
        sys.exit(2)