MEGAINFRA-WEBAPP/scripts/pb-add-fuel-import-dedup.py

#!/usr/bin/env python3
"""Add fuel import identity, audit counts and a private import lock.

Dry-run is the default (schema reads only after authentication):
    python3 scripts/pb-add-fuel-import-dedup.py
    python3 scripts/pb-add-fuel-import-dedup.py --apply --backup /tmp/fuel-schema.json

Connection/credentials come from pb_env (.env.local or environment). This only
changes schema; it never updates fuel records or steals/deletes import locks.
"""
import argparse
import copy
import json
import os
import re
import sys
import tempfile
from datetime import datetime, timezone
from pathlib import Path

from pb_env import PB, http, superuser_token


LOCKED = dict.fromkeys(("listRule", "viewRule", "createRule", "updateRule", "deleteRule"))
LOG_FIELDS = [
    {"name": name, "type": kind, "required": False}
    for name, kind in (
        ("source_system", "text"), ("source_id", "text"),
        ("source_logged_at", "date"), ("source_created_at", "date"),
        ("source_modified_at", "date"), ("import_key", "text"),
    )
]
LOG_INDEXES = [
    "CREATE UNIQUE INDEX `idx_fuel_logs_source_identity` ON `fuel_logs` "
    "(`source_system`, `source_id`) WHERE `source_id` != ''",
    "CREATE UNIQUE INDEX `idx_fuel_logs_import_key` ON `fuel_logs` "
    "(`import_key`) WHERE `import_key` != ''",
]
LOCK_INDEX = "CREATE UNIQUE INDEX `idx_fuel_import_locks_scope` ON `fuel_import_locks` (`scope`)"


def definitions():
    return {
        "fuel_logs": {"fields": LOG_FIELDS, "indexes": LOG_INDEXES},
        "fuel_sheets": {"fields": [
            {"name": "rows_skipped", "type": "number", "required": False},
            {"name": "rows_review", "type": "number", "required": False},
        ], "indexes": []},
        "fuel_import_locks": {
            "name": "FUEL_IMPORT_LOCKS", "type": "base",
            "fields": [
                {"name": "scope", "type": "text", "required": True, "min": 1},
                {"name": "owner", "type": "text", "required": True, "min": 1},
            ],
            "indexes": [LOCK_INDEX], **LOCKED,
        },
    }


def index_tokens(sql):
    """Compare SQL without confusing identifier quoting/case with string values."""
    tokens = re.findall(r"'(?:''|[^'])*'|`[^`]*`|\"[^\"]*\"|\[[^\]]*\]|[a-zA-Z_][a-zA-Z_0-9]*|!=|<>|\S", sql)
    normalized = []
    for token in tokens:
        if token.startswith("'"):
            normalized.append(token)
        elif token[0] in ('`', '"', '['):
            normalized.append(token[1:-1].lower())
        elif token == "<>":
            normalized.append("!=")
        elif token != ";":
            normalized.append(token.lower())
    if normalized[3:6] == ["if", "not", "exists"]:
        del normalized[3:6]
    elif normalized[2:5] == ["if", "not", "exists"]:
        del normalized[2:5]
    return normalized


def index_name(sql):
    tokens = index_tokens(sql)
    try:
        return tokens[tokens.index("index") + 1]
    except (ValueError, IndexError):
        raise RuntimeError(f"Cannot identify existing index: {sql}") from None


def plan_collection(name, existing, wanted):
    """Return an additive payload; reject incompatible schema before any writes."""
    if existing is None:
        if name != "fuel_import_locks":
            raise RuntimeError(f"Required collection {name} does not exist")
        return copy.deepcopy(wanted)
    if existing.get("type") != "base":
        raise RuntimeError(f"Incompatible collection type: {name}")
    fields = copy.deepcopy(existing.get("fields", []))
    by_name = {field["name"]: field for field in fields}
    missing = []
    for field in wanted["fields"]:
        actual = by_name.get(field["name"])
        if actual is None:
            missing.append(copy.deepcopy(field))
            continue
        for key, value in field.items():
            # PB may omit default required=false/min=0 in exported schemas.
            default = False if key == "required" else None
            if actual.get(key, default) != value:
                raise RuntimeError(f"Incompatible field: {name}.{field['name']}.{key}")
    indexes = list(existing.get("indexes", []))
    by_index = {index_name(sql): sql for sql in indexes}
    missing_indexes = []
    for sql in wanted["indexes"]:
        actual = by_index.get(index_name(sql))
        if actual is None:
            missing_indexes.append(sql)
        elif index_tokens(actual) != index_tokens(sql):
            raise RuntimeError(f"Conflicting index definition: {index_name(sql)}")
    if name == "fuel_import_locks":
        for rule in LOCKED:
            if existing.get(rule) is not None:
                raise RuntimeError(f"fuel_import_locks must be superuser-only: {rule}")
    changes = {}
    if missing:
        changes["fields"] = fields + missing
    if missing_indexes:
        changes["indexes"] = indexes + missing_indexes
    return changes


def read_schema(name, token):
    code, result = http("GET", f"/api/collections/{name}", token=token)
    if code == 404:
        return None
    if code != 200:
        raise RuntimeError(f"Cannot inspect {name}: HTTP {code}: {result}")
    return result


def write_backup(path, schemas):
    snapshot = {"pocketbase_url": PB, "saved_at": datetime.now(timezone.utc).isoformat(),
                "collections": schemas}
    if path:
        # Never overwrite an earlier recovery copy; keep schema/rules private.
        fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
    else:
        fd, path = tempfile.mkstemp(prefix="mega-before-fuel-dedup-", suffix=".json")
    with os.fdopen(fd, "w") as backup:
        json.dump(snapshot, backup, indent=2)
        backup.write("\n")
    return str(Path(path).resolve())


def migrate(token, apply=False, backup=None):
    wanted = definitions()
    before = {name: read_schema(name, token) for name in wanted}
    plans = {name: plan_collection(name, before[name], spec) for name, spec in wanted.items()}
    for name, changes in plans.items():
        action = "create" if before[name] is None else "add schema fields/indexes"
        print(f"{name}: {action if changes else 'already compatible'}")
        if changes:
            existing_fields = {f["name"] for f in (before[name] or {}).get("fields", [])}
            for field in changes.get("fields", []):
                if field["name"] not in existing_fields:
                    print(f"  + {field['name']}: {field['type']}")
            existing_indexes = (before[name] or {}).get("indexes", [])
            for sql in changes.get("indexes", []):
                if sql not in existing_indexes:
                    print(f"  + {sql}")
    if not apply:
        print("Dry-run: no schema or record writes. Use --apply to apply this plan.")
        return
    if not any(plans.values()):
        print("Already compatible; no changes needed.")
        return
    backup_path = write_backup(backup, before)
    print(f"Private schema backup: {backup_path}")
    # Abort on concurrent schema changes instead of overwriting a newer array.
    for name in wanted:
        if read_schema(name, token) != before[name]:
            raise RuntimeError(f"Schema changed during preflight: {name}; rerun migration")
    for name, changes in plans.items():
        if not changes:
            continue
        existing = before[name]
        method = "PATCH" if existing else "POST"
        path = f"/api/collections/{existing['id']}" if existing else "/api/collections"
        code, result = http(method, path, changes, token)
        if code != 200:
            raise RuntimeError(f"{method} {name} failed: HTTP {code}: {result}; backup: {backup_path}")
    for name, spec in wanted.items():
        actual = read_schema(name, token)
        if actual is None or plan_collection(name, actual, spec):
            raise RuntimeError(f"Verification failed: {name}")
        original = before[name]
        if original:
            actual_fields = {f["name"]: f for f in actual["fields"]}
            for field in original.get("fields", []):
                if any(actual_fields.get(field["name"], {}).get(k) != v for k, v in field.items()):
                    raise RuntimeError(f"Existing field changed: {name}.{field['name']}")
            for rule in LOCKED:
                if actual.get(rule) != original.get(rule):
                    raise RuntimeError(f"Existing access rule changed: {name}.{rule}")
            actual_indexes = [index_tokens(sql) for sql in actual.get("indexes", [])]
            if any(index_tokens(sql) not in actual_indexes for sql in original.get("indexes", [])):
                raise RuntimeError(f"Existing indexes changed: {name}")
    print("Verified fuel import schema. No fuel records or lock records changed.")


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    mode = parser.add_mutually_exclusive_group()
    mode.add_argument("--apply", action="store_true", help="Apply additive schema changes")
    mode.add_argument("--dry-run", action="store_true", help="Read and print schema plan (default)")
    parser.add_argument("--backup", help="New private JSON backup path (default: unique /tmp file)")
    args = parser.parse_args()
    migrate(superuser_token(), apply=args.apply, backup=args.backup)


if __name__ == "__main__":
    try:
        main()
    except (OSError, RuntimeError) as exc:
        sys.exit(str(exc))