MEGAINFRA-WEBAPP/scripts/pb-bootstrap-schema.py

#!/usr/bin/env python3
"""Bootstrap PocketBase schema for MEGA INFRASTRUCTURE.

PocketBase v0.39 JSON shape: type-specific options (values, maxSelect,
mimeTypes, collectionId, cascadeDelete, min, max) live at the field
top level — NOT nested under "options".

Reflects the post-ingestion-restructure schema (2026-09):
  - `data_import_batches` (the shared queue pool) and the deprecated
    `plant` alias for `vehicles` are gone.
  - `timesheets`, `fuel_sheets` and `plant_hours_sheets` are the
    uploaded-document collections; each carries its own
    `processing_status` queue fields directly (no separate batch row).
  - `timesheets` (old) was renamed `timesheet_entries` — the actual
    per-employee-per-day time log.
  - `received_items` was renamed `delivery_note_items`.
  - `purchase_orders` absorbed the queue fields `data_import_batches`
    used to carry, plus `file_hash` and `date_uploaded`.
  - `project_code` was removed from every collection except `sites`
    (the one place it is canonical); relations to `sites` carry site
    attribution everywhere else.
  - `qty_outstanding` was removed from `purchase_order_items` — it is
    now computed at read time via src/lib/po-fulfilment.ts.
  - `fuel_prices` and `income_claims` were always live collections but
    were missing from this bootstrap script; both are included now.
"""
import json
import sys
import urllib.request
import urllib.error

PB = "https://pb.mega.infra.apphub.co.za"
EMAIL = "admin@mega.co.za"
PASSWORD = "12345678"


def http(method, path, body=None, token=None):
    url = f"{PB}{path}"
    data = json.dumps(body).encode() if body is not None else None
    req = urllib.request.Request(url, data=data, method=method)
    if body is not None:
        req.add_header("Content-Type", "application/json")
    if token:
        req.add_header("Authorization", token)
    try:
        with urllib.request.urlopen(req) as r:
            txt = r.read().decode()
            return r.status, json.loads(txt) if txt else None
    except urllib.error.HTTPError as e:
        return e.code, json.loads(e.read().decode() or "{}")


code, auth = http("POST", "/api/collections/_superusers/auth-with-password",
                  {"identity": EMAIL, "password": PASSWORD})
if code != 200:
    sys.exit(f"AUTH FAILED: {code} {auth}")
TOKEN = auth["token"]
print(f"auth ok, superuser id={auth['record']['id']}\n")

LOCKED = {  # superuser-only access
    "listRule": None, "viewRule": None,
    "createRule": None, "updateRule": None, "deleteRule": None,
}


def field(name, type_, **kw):
    """Build a field. Extra kwargs (required, values, maxSelect, etc.)
    go at the field top level."""
    f = {"name": name, "type": type_}
    f.update(kw)
    return f


# Shared field set for the three "uploaded document" collections
# (timesheets, fuel_sheets, plant_hours_sheets) — each is a
# resumable/idempotent import queue item in its own right, not a row
# in a shared pool.
def sheet_fields(site_required):
    return [
        field("site", "relation", required=site_required,
              collectionId="__SITES_ID__", cascadeDelete=False, maxSelect=1),
        field("source_file", "file", required=False,
              maxSelect=1, maxSize=20971520,
              mimeTypes=["text/csv", "text/plain", "application/vnd.ms-excel",
                          "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
                          "application/csv"]),
        field("content_type", "text", required=False),
        field("file_hash", "text", required=False),
        field("date_uploaded", "date", required=False),
        field("uploaded_by", "text", required=False),
        field("processing_status", "select", required=False,
              values=["pending", "processing", "processed", "failed", "dead"], maxSelect=1),
        field("processed_at", "date", required=False),
        field("attempts", "number", required=False),
        field("last_error", "text", required=False),
        field("rows_total", "number", required=False),
        field("rows_created", "number", required=False),
        field("rows_failed", "number", required=False),
        field("row_errors", "json", required=False),
    ]


DEFS = {}

DEFS["sites"] = {
    "name": "SITES", "type": "base",
    "fields": [
        field("project_code", "text", required=True, min=1, max=64),
        field("name", "text", required=False),
        field("duration", "text", required=False),
        field("contract_value", "number", required=False),
        field("work_hours", "number", required=False),
        field("hours_per_week", "number", required=False),
        field("wages_hours_per_day", "number", required=False),
        field("latitude", "number", required=False),
        field("longitude", "number", required=False),
        field("job_start_date", "date", required=False),
        # How the site counts its own "work months" — e.g. 21 means each
        # work-month runs the 21st through the 20th of the next calendar
        # month, not the 1st–30th. Unset (or 1) means ordinary calendar
        # months. See src/lib/project-calendar.ts.
        field("work_month_start_day", "number", required=False, min=1, max=31),
    ],
    "indexes": [
        "CREATE UNIQUE INDEX `idx_sites_project_code` ON `sites` (`project_code`)",
    ],
}

DEFS["employees"] = {
    "name": "EMPLOYEES", "type": "base",
    "fields": [
        field("site", "relation", required=False,
              collectionId="__SITES_ID__", cascadeDelete=False, maxSelect=1),
        field("name", "text", required=False),
        field("employee_no", "text", required=False),
        field("position", "text", required=False),
        field("rate_per_hour", "number", required=False),
        field("hours_per_day_standard", "number", required=False),
        field("worker_type", "select", required=False,
              values=["permanent", "casual"], maxSelect=1),
    ],
    "indexes": [],
}

DEFS["suppliers"] = {
    "name": "SUPPLIERS", "type": "base",
    "fields": [
        field("name", "text", required=True, min=1, max=200),
    ],
    "indexes": [],
}

DEFS["purchase_orders"] = {
    "name": "PURCHASE_ORDERS", "type": "base",
    "fields": [
        field("purchase_order_number", "text", required=True, min=1, max=128),
        field("site", "relation", required=True,
              collectionId="__SITES_ID__", cascadeDelete=False, maxSelect=1),
        field("supplier", "text", required=False),
        field("supplier_ref", "relation", required=False,
              collectionId="__SUPPLIERS_ID__", cascadeDelete=False, maxSelect=1),
        field("required_by_date", "date", required=False),
        field("ordered_by", "text", required=False),
        field("items_count", "number", required=False),
        field("total_value", "number", required=False),
        field("purchase_order_file", "file", required=False,
              maxSelect=1, maxSize=20971520,
              mimeTypes=["application/pdf", "image/jpeg", "image/png", "image/webp"]),
        field("raw_ocr_data", "json", required=False),
        field("is_verified", "bool", required=False),
        field("file_hash", "text", required=False),
        field("processing_status", "select", required=False,
              values=["pending", "processing", "processed", "failed", "dead"], maxSelect=1),
        field("attempts", "number", required=False),
        field("last_error", "text", required=False),
        field("next_retry_at", "date", required=False),
        field("last_attempt_at", "date", required=False),
        field("date_uploaded", "date", required=False),
    ],
    "indexes": [
        "CREATE UNIQUE INDEX `idx_po_number` ON `purchase_orders` (`purchase_order_number`)",
        "CREATE INDEX `idx_po_site_status` ON `purchase_orders` (`site`, `processing_status`)",
    ],
}

DEFS["purchase_order_items"] = {
    "name": "PURCHASE_ORDER_ITEMS", "type": "base",
    "fields": [
        field("purchase_order", "relation", required=False,
              collectionId="__PURCHASE_ORDERS_ID__", cascadeDelete=True, maxSelect=1),
        field("material_code", "text", required=False),
        field("cost_code", "text", required=False),
        field("description", "text", required=False),
        field("qty", "number", required=False),
        field("unit", "text", required=False),
        field("rate", "number", required=False),
        field("disc_percent", "number", required=False),
        field("vat_percent", "number", required=False),
        field("amount", "number", required=False),
    ],
    "indexes": [],
}

DEFS["fixed_costs"] = {
    "name": "FIXED_COSTS", "type": "base",
    "fields": [
        field("site", "relation", required=False,
              collectionId="__SITES_ID__", cascadeDelete=False, maxSelect=1),
        field("date_logged", "date", required=False),
        field("description", "text", required=False),
        field("cost", "number", required=False),
        field("notes", "text", required=False),
        # bool, no default: PB applies the field default only when the
        # value is absent. Defaulting to `true` would also be fine for
        # new rows, but we leave the field "unset by default" so explicit
        # writes of `false` cannot be silently overridden by a default.
        # The partition helper treats undefined/null as monthly via a
        # `=== false` check, and the backfill migration patches any
        # pre-existing row to `true`.
        field("is_recurring", "bool", required=False),
    ],
    "indexes": [],
}

DEFS["vehicles"] = {
    "name": "VEHICLES", "type": "base",
    "fields": [
        field("site", "relation", required=False,
              collectionId="__SITES_ID__", cascadeDelete=False, maxSelect=1),
        field("description", "text", required=False),
        field("standard_work_hours", "text", required=False),
        field("fuel_limits", "text", required=False),
        field("qr_code_id", "text", required=False),
        field("cost_per_hour", "number", required=False),
        field("attention_dismissed_at", "date", required=False),
        field("unit_type", "select", required=False,
              values=["fuel", "plant"], maxSelect=1, default="fuel"),
        # Latest GPS fix from a MEGA Clock Fleet scan. Denormalised off
        # `fuel_logs` so the map page can render every vehicle in a
        # single query. Stays null for vehicles never scanned from a
        # kiosk phone (e.g. imported only from a pump sheet).
        field("last_latitude", "number", required=False),
        field("last_longitude", "number", required=False),
        field("last_scanned_at", "date", required=False),
        field("last_location_accuracy_m", "number", required=False),
    ],
    "indexes": [
        "CREATE INDEX `idx_vehicles_last_scanned` ON `vehicles` (`last_scanned_at`)",
    ],
}

DEFS["fuel_prices"] = {
    "name": "FUEL_PRICES", "type": "base",
    "fields": [
        field("effective_from", "date", required=True),
        field("price_per_litre", "number", required=True),
        field("notes", "text", required=False),
    ],
    "indexes": [
        "CREATE UNIQUE INDEX `idx_fuel_prices_effective_from` ON `fuel_prices` (`effective_from`)",
    ],
}

DEFS["timesheets"] = {
    "name": "TIMESHEETS", "type": "base",
    "fields": sheet_fields(site_required=True),
    "indexes": [
        "CREATE INDEX `idx_timesheets_hash` ON `timesheets` (`file_hash`)",
        "CREATE INDEX `idx_timesheets_status` ON `timesheets` (`processing_status`, `date_uploaded`)",
    ],
}

DEFS["timesheet_entries"] = {
    "name": "TIMESHEET_ENTRIES", "type": "base",
    "fields": [
        field("employee", "relation", required=False,
              collectionId="__EMPLOYEES_ID__", cascadeDelete=False, maxSelect=1),
        field("date_logged", "date", required=False),
        field("hours_worked", "number", required=False),
        # Stamped from the employee's site at import time — NOT joined at
        # read time. This ends historical drift when a worker moves sites.
        field("site", "relation", required=True,
              collectionId="__SITES_ID__", cascadeDelete=False, maxSelect=1),
        # Optional forever: empty means captured directly by QR scan, not
        # from an uploaded sheet.
        field("timesheet", "relation", required=False,
              collectionId="__TIMESHEETS_ID__", cascadeDelete=False, maxSelect=1),
    ],
    "indexes": [
        "CREATE INDEX `idx_ts_entries_site_date` ON `timesheet_entries` (`site`, `date_logged`)",
        "CREATE INDEX `idx_ts_entries_emp_date` ON `timesheet_entries` (`employee`, `date_logged`)",
    ],
}

DEFS["clock_events"] = {
    "name": "CLOCK_EVENTS", "type": "base",
    "fields": [
        field("employee", "relation", required=True,
              collectionId="__EMPLOYEES_ID__", cascadeDelete=False, maxSelect=1),
        field("site", "relation", required=True,
              collectionId="__SITES_ID__", cascadeDelete=False, maxSelect=1),
        field("direction", "select", required=True,
              values=["in", "out"], maxSelect=1),
        field("scanned_at", "date", required=True),
        # Optional: set only on "out" events, pointing at the
        # timesheet_entries row that scan created/updated.
        field("timesheet_entry", "relation", required=False,
              collectionId="__TIMESHEET_ENTRIES_ID__", cascadeDelete=False, maxSelect=1),
    ],
    "indexes": [
        "CREATE INDEX `idx_clock_events_emp_scanned` ON `clock_events` (`employee`, `scanned_at`)",
    ],
}

DEFS["fuel_sheets"] = {
    "name": "FUEL_SHEETS", "type": "base",
    # site is optional: a pump sheet can span sites, resolved per-row.
    "fields": sheet_fields(site_required=False),
    "indexes": [
        "CREATE INDEX `idx_fuel_sheets_hash` ON `fuel_sheets` (`file_hash`)",
        "CREATE INDEX `idx_fuel_sheets_status` ON `fuel_sheets` (`processing_status`, `date_uploaded`)",
    ],
}

DEFS["fuel_logs"] = {
    "name": "FUEL_LOGS", "type": "base",
    "fields": [
        field("site", "relation", required=True,
              collectionId="__SITES_ID__", cascadeDelete=False, maxSelect=1),
        field("vehicle", "relation", required=False,
              collectionId="__VEHICLES_ID__", cascadeDelete=False, maxSelect=1),
        field("date_logged", "date", required=False),
        field("litres", "number", required=False),
        field("odo_reading", "number", required=False),
        field("latest_consumption", "number", required=False),
        # GPS fix captured at scan time on the kiosk phone (MEGA Clock
        # Fleet tab). Optional — the spreadsheet import path has no GPS
        # context, so desktop-imported rows stay null. Range validation
        # is the route's job; PB stays permissive.
        field("latitude", "number", required=False),
        field("longitude", "number", required=False),
        field("location_accuracy_m", "number", required=False),
        field("fuel_sheet", "relation", required=False,
              collectionId="__FUEL_SHEETS_ID__", cascadeDelete=False, maxSelect=1),
    ],
    "indexes": [
        "CREATE INDEX `idx_fuel_logs_site_date` ON `fuel_logs` (`site`, `date_logged`)",
        "CREATE INDEX `idx_fuel_logs_vehicle_date` ON `fuel_logs` (`vehicle`, `date_logged`)",
    ],
}

DEFS["plant_hours_sheets"] = {
    "name": "PLANT_HOURS_SHEETS", "type": "base",
    # site is optional, same reasoning as fuel_sheets.
    "fields": sheet_fields(site_required=False),
    "indexes": [
        "CREATE INDEX `idx_plant_hours_sheets_hash` ON `plant_hours_sheets` (`file_hash`)",
        "CREATE INDEX `idx_plant_hours_sheets_status` ON `plant_hours_sheets` (`processing_status`, `date_uploaded`)",
    ],
}

DEFS["plant_hours_log"] = {
    "name": "PLANT_HOURS_LOG", "type": "base",
    "fields": [
        field("date_logged", "date", required=True),
        field("vehicle_qr", "text", required=True),
        field("vehicle", "relation", required=True,
              collectionId="__VEHICLES_ID__", cascadeDelete=False, maxSelect=1, minSelect=1),
        field("site", "relation", required=True,
              collectionId="__SITES_ID__", cascadeDelete=False, maxSelect=1, minSelect=1),
        field("start_date", "date", required=True),
        field("end_date", "date", required=True),
        field("start_time", "text", required=False),
        field("end_time", "text", required=False),
        field("start_hours", "number", required=True),
        field("end_hours", "number", required=True),
        field("hours", "number", required=True),
        field("notes", "text", required=False),
        field("hour_meter_budget", "number", required=True),
        field("cost_per_hour", "number", required=True),
        field("value", "number", required=True),
        field("over_budget", "bool", required=False),
        field("hours_mismatch", "bool", required=False),
        field("vehicle_unit_type_at_import", "select", required=True,
              values=["fuel", "plant"], maxSelect=1),
        field("plant_hours_sheet", "relation", required=False,
              collectionId="__PLANT_HOURS_SHEETS_ID__", cascadeDelete=False, maxSelect=1, minSelect=0),
    ],
    "indexes": [
        "CREATE UNIQUE INDEX `idx_plant_hours_log_qr_start` ON `plant_hours_log` (`vehicle_qr`, `start_date`)",
        "CREATE INDEX `idx_ph_log_vehicle_start` ON `plant_hours_log` (`vehicle`, `start_date`)",
    ],
}

DEFS["income_claims"] = {
    "name": "INCOME_CLAIMS", "type": "base",
    "fields": [
        field("claim_number", "text", required=True),
        field("site", "relation", required=False,
              collectionId="__SITES_ID__", cascadeDelete=False, maxSelect=1),
        field("period_start", "date", required=False),
        field("period_end", "date", required=False),
        field("gross_amount", "number", required=False),
        field("certified_by", "text", required=False),
        field("paid_on", "date", required=False),
        field("status", "select", required=False,
              values=["draft", "certified", "paid"], maxSelect=1),
        field("is_verified", "bool", required=False),
        field("notes", "text", required=False),
    ],
    "indexes": [],
}

DEFS["delivery_notes"] = {
    "name": "DELIVERY_NOTES", "type": "base",
    "fields": [
        field("processing_status", "select", required=False,
              values=["pending", "ocr-complete", "auto-verified",
                      "needs-review", "manual-verified", "failed"], maxSelect=1),
        field("delivery_note_number", "text", required=False),
        field("site", "relation", required=True,
              collectionId="__SITES_ID__", cascadeDelete=False, maxSelect=1),
        field("purchase_order", "relation", required=False,
              collectionId="__PURCHASE_ORDERS_ID__", cascadeDelete=False, maxSelect=1),
        field("date_received", "date", required=False),
        field("received_by", "text", required=False),
        field("slip_image", "file", required=False,
              maxSelect=1, maxSize=20971520,
              mimeTypes=["image/jpeg", "image/png", "image/webp", "application/pdf"]),
        field("raw_ocr_data", "json", required=False),
        field("is_verified", "bool", required=False),
        field("ocr_confidence", "number", required=False),
        field("ocr_completed_at", "date", required=False),
        field("received_by_name", "text", required=False),
        field("file_hash", "text", required=False),
    ],
    "indexes": [
        "CREATE INDEX `idx_dn_site_date` ON `delivery_notes` (`site`, `date_received`)",
        "CREATE INDEX `idx_dn_status` ON `delivery_notes` (`processing_status`)",
    ],
}

DEFS["delivery_note_items"] = {
    "name": "DELIVERY_NOTE_ITEMS", "type": "base",
    "fields": [
        field("delivery_note", "relation", required=False,
              collectionId="__DELIVERY_NOTES_ID__", cascadeDelete=True, maxSelect=1),
        field("purchase_order_item", "relation", required=False,
              collectionId="__PURCHASE_ORDER_ITEMS_ID__", cascadeDelete=False, maxSelect=1),
        field("qty_received", "number", required=False),
        field("condition_notes", "text", required=False),
    ],
    "indexes": [],
}

# Dependency order: a collection referencing another via `relation` must be
# created after the collection it points to, so its `collectionId`
# placeholder can be resolved.
ORDER = [
    "sites",
    "employees",
    "suppliers",
    "purchase_orders",
    "purchase_order_items",
    "fixed_costs",
    "vehicles",
    "fuel_prices",
    "timesheets",
    "timesheet_entries",
    "clock_events",
    "fuel_sheets",
    "fuel_logs",
    "plant_hours_sheets",
    "plant_hours_log",
    "income_claims",
    "delivery_notes",
    "delivery_note_items",
]

ids = {}
for key in ORDER:
    body = dict(DEFS[key])
    body.update(LOCKED)
    for f in body["fields"]:
        if f.get("type") == "relation":
            cid = f.get("collectionId", "")
            # placeholders use collection table name in upper, e.g. __SITES_ID__
            for other_key, other_id in ids.items():
                collection_name_upper = DEFS[other_key]["name"]  # e.g. "SITES"
                cid = cid.replace(f"__{collection_name_upper}_ID__", other_id)
            if "__" in cid:
                sys.exit(f"unresolved placeholder in {key}.{f['name']}: {cid}")
            f["collectionId"] = cid

    code, resp = http("POST", "/api/collections", body, TOKEN)
    if code == 200:
        ids[key] = resp["id"]
        print(f"  ✓ {body['name']:25s} → {resp['id']}")
    elif code == 400:
        code2, existing = http("GET", f"/api/collections/{body['name']}", token=TOKEN)
        if code2 == 200:
            ids[key] = existing["id"]
            print(f"  • {body['name']:25s} → {existing['id']} (already existed)")
        else:
            print(f"  ✗ {body['name']:25s} FAILED: {resp}", file=sys.stderr)
            sys.exit(1)
    else:
        print(f"  ✗ {body['name']:25s} FAILED: {code} {resp}", file=sys.stderr)
        sys.exit(1)

print(f"\nDone. {len(ids)} collections on {PB}.")