MEGAINFRA-WEBAPP/scripts/migrations/2026-09-12-create-document-collections.py

#!/usr/bin/env python3
"""
Create the three sheet-document collections.

  PB_URL=... PB_ADMIN_EMAIL=... PB_ADMIN_PASSWORD=... \\
  python3 scripts/migrations/2026-09-12-create-document-collections.py [--apply]

DRY RUN BY DEFAULT.

  TIMESHEETS          - the uploaded wage/hours sheet
  FUEL_SHEETS         - the uploaded pump sheet
  PLANT_HOURS_SHEETS  - the uploaded plant hours sheet

All three carry the shared document contract from spec section 4.1. They
replace what DATA_IMPORT_BATCHES was doing for the three synchronous
importers, with two differences that matter:

  - site is a real relation, so the future per-user site allowlist can
    filter them
  - rows_created / rows_failed / row_errors persist the outcome the
    current importers compute and then throw away

site is required on TIMESHEETS (a wage sheet belongs to one site) and
optional on the other two (a pump sheet can legitimately span sites; the
row-level site stays authoritative). Do not "tidy" that asymmetry away.

The site relation is cascadeDelete=False on all three, deliberately:
deleting a site must not destroy the uploaded source spreadsheets that
document what was filed against it. Each create is read back and the
landed field is checked, because a relation that silently cascades is
not something you want to discover from a missing audit trail.

Idempotent: skips collections that already exist.
"""
import json
import os
import sys
import urllib.error
import urllib.request

APPLY = "--apply" in sys.argv

SPREADSHEET_MIMES = [
    "text/csv",
    "application/vnd.ms-excel",
    "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
]

STATUS_VALUES = ["pending", "processing", "processed", "failed", "dead"]

# Same posture as every other collection: authenticated users read and
# create, only superusers mutate. Set by scripts/pb-relax-rules.py for
# the rest of the schema.
RULES = {
    "listRule": "@request.auth.id != ''",
    "viewRule": "@request.auth.id != ''",
    "createRule": "@request.auth.id != ''",
    "updateRule": None,
    "deleteRule": None,
}


def document_fields(site_collection_id, site_required):
    """The shared document contract, spec section 4.1."""
    return [
        {
            "name": "site",
            "type": "relation",
            "required": site_required,
            "collectionId": site_collection_id,
            "maxSelect": 1,
            "minSelect": 1 if site_required else 0,
            "cascadeDelete": False,
        },
        {"name": "source_file", "type": "file", "required": False,
         "maxSelect": 1, "maxSize": 20971520, "mimeTypes": SPREADSHEET_MIMES},
        {"name": "content_type", "type": "text", "required": False},
        {"name": "file_hash", "type": "text", "required": False, "max": 64},
        {"name": "date_uploaded", "type": "date", "required": False},
        {"name": "uploaded_by", "type": "text", "required": False},
        {"name": "processing_status", "type": "select", "required": False,
         "maxSelect": 1, "values": STATUS_VALUES},
        {"name": "processed_at", "type": "date", "required": False},
        {"name": "attempts", "type": "number", "required": False},
        {"name": "last_error", "type": "text", "required": False},
        {"name": "rows_total", "type": "number", "required": False},
        {"name": "rows_created", "type": "number", "required": False},
        {"name": "rows_failed", "type": "number", "required": False},
        {"name": "row_errors", "type": "json", "required": False,
         "maxSize": 2000000},
    ]


def indexes(name):
    return [
        f"CREATE INDEX `idx_{name.lower()}_hash` ON `{name}` (`file_hash`)",
        f"CREATE INDEX `idx_{name.lower()}_status` "
        f"ON `{name}` (`processing_status`, `date_uploaded`)",
    ]


NEW_COLLECTIONS = [
    ("TIMESHEETS", True),
    ("FUEL_SHEETS", False),
    ("PLANT_HOURS_SHEETS", False),
]


def admin_auth(base_url):
    body = json.dumps(
        {
            "identity": os.environ["PB_ADMIN_EMAIL"],
            "password": os.environ["PB_ADMIN_PASSWORD"],
        }
    ).encode()
    req = urllib.request.Request(
        f"{base_url}/api/collections/_superusers/auth-with-password",
        data=body, headers={"Content-Type": "application/json"}, method="POST",
    )
    with urllib.request.urlopen(req) as resp:
        return json.loads(resp.read())["token"]


def api(base_url, token, method, path, body=None):
    """
    A 404 is only soft for GET - "this collection is absent" is a
    legitimate answer to a read, and is how we detect a collection that
    already exists. On a write it is not: a POST that 404s returning
    None would be read downstream as a created collection.
    """
    data = json.dumps(body).encode() if body is not None else None
    req = urllib.request.Request(
        f"{base_url}{path}", data=data, method=method,
        headers={"Authorization": token, "Content-Type": "application/json"},
    )
    try:
        with urllib.request.urlopen(req) as resp:
            txt = resp.read().decode()
            return json.loads(txt) if txt else None
    except urllib.error.HTTPError as e:
        if e.code == 404 and method == "GET":
            return None
        raise RuntimeError(
            f"{method} {path}: {e.code} {e.read().decode() if e.fp else ''}"
        ) from e


def verify(collection, site_required, site_collection_id):
    """
    Check what actually landed, not what we asked for. Returns a list of
    human-readable problems; empty means the collection is as specified.
    """
    problems = []
    name = collection["name"]

    fields = {f["name"]: f for f in collection["fields"]}
    expected = {f["name"] for f in document_fields(site_collection_id, site_required)}
    missing = sorted(expected - set(fields))
    if missing:
        problems.append(f"missing fields: {', '.join(missing)}")

    site = fields.get("site")
    if site is None:
        problems.append("site field absent")
    else:
        if site.get("cascadeDelete") is not False:
            problems.append(
                f"site.cascadeDelete is {site.get('cascadeDelete')!r}, must be False - "
                f"deleting a site would destroy its uploaded sheets"
            )
        if bool(site.get("required")) != site_required:
            problems.append(
                f"site.required is {site.get('required')!r}, expected {site_required}"
            )
        if site.get("minSelect") != (1 if site_required else 0):
            problems.append(
                f"site.minSelect is {site.get('minSelect')!r}, "
                f"expected {1 if site_required else 0}"
            )
        if site.get("collectionId") != site_collection_id:
            problems.append(
                f"site.collectionId is {site.get('collectionId')!r}, "
                f"expected {site_collection_id!r}"
            )

    for rule, want in RULES.items():
        if collection.get(rule) != want:
            problems.append(f"{rule} is {collection.get(rule)!r}, expected {want!r}")

    landed = collection.get("indexes") or []
    for want in indexes(name):
        if want not in landed:
            problems.append(f"index missing: {want}")

    return problems


def main():
    base_url = os.environ.get("PB_URL", "http://127.0.0.1:8090").rstrip("/")
    token = admin_auth(base_url)
    # Say where we are writing. The default target is production, and a
    # run whose output names no host is a run you cannot audit later.
    print(f"target: {base_url}   mode: {'APPLY' if APPLY else 'DRY RUN'}")

    sites = api(base_url, token, "GET", "/api/collections/sites")
    if sites is None:
        print("\nABORT: collection `sites` not found - cannot build the relation")
        return 1
    site_id = sites["id"]
    print(f"sites collection id: {site_id}")

    created_count, skipped = 0, 0
    for name, site_required in NEW_COLLECTIONS:
        existing = api(base_url, token, "GET", f"/api/collections/{name}")
        if existing is not None:
            print(f"\nSKIP {name}: already exists (id={existing['id']})")
            skipped += 1
            continue

        print(f"\nCREATE {name}  (site required={site_required})")
        if not APPLY:
            continue

        created = api(
            base_url, token, "POST", "/api/collections",
            {
                "name": name,
                "type": "base",
                "fields": document_fields(site_id, site_required),
                **RULES,
                "indexes": indexes(name),
            },
        )
        if not created or not created.get("id"):
            print(f"   FAILED: create returned no collection for {name}")
            return 1
        print(f"   created id={created['id']}")

        # Read back by id, not by name: reading by name would prove only
        # that *a* collection has that name.
        after = api(base_url, token, "GET", f"/api/collections/{created['id']}")
        if after is None:
            print(f"   FAILED: collection id={created['id']} vanished after create")
            return 1
        problems = verify(after, site_required, site_id)
        if problems:
            for p in problems:
                print(f"   FAILED: {p}")
            print("   Stopping before the next create.")
            return 1
        print(
            f"   OK  site required={site_required} cascadeDelete=False, "
            f"rules as specified, {len(after.get('indexes') or [])} indexes"
        )
        created_count += 1

    if not APPLY:
        print("\nDRY RUN. Re-run with --apply to create.")
        return 0

    print(f"\ncreated {created_count}  skipped {skipped}  of {len(NEW_COLLECTIONS)}")
    return 0


if __name__ == "__main__":
    try:
        sys.exit(main())
    except KeyError as e:
        print(f"ERROR: missing environment variable {e}", file=sys.stderr)
        sys.exit(2)