0a7138f60c
Improve workforce scanning, site transfers and timesheet exports. Audit plant imports, duplicate identities, rates and dashboard totals. Repair delivery PO selection, periodic rechecks and receiving guards. Align financial ledgers, supplier periods and mobile layouts. Add the site map preview, shared theme controls and audit evidence. Verify 3,455 application tests, six fuel export tests, TypeScript and an optimized production build. Clean up a leaked test timer and keep local worktrees and generated reports out of ESLint traversal. Changed-file lint passes; repository-wide lint retains 33 existing errors in unchanged files.
223 lines
9.1 KiB
Python
223 lines
9.1 KiB
Python
#!/usr/bin/env python3
|
|
"""Add fuel import identity, audit counts and a private import lock.
|
|
|
|
Dry-run is the default (schema reads only after authentication):
|
|
python3 scripts/pb-add-fuel-import-dedup.py
|
|
python3 scripts/pb-add-fuel-import-dedup.py --apply --backup /tmp/fuel-schema.json
|
|
|
|
Connection/credentials come from pb_env (.env.local or environment). This only
|
|
changes schema; it never updates fuel records or steals/deletes import locks.
|
|
"""
|
|
import argparse
|
|
import copy
|
|
import json
|
|
import os
|
|
import re
|
|
import sys
|
|
import tempfile
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
from pb_env import PB, http, superuser_token
|
|
|
|
|
|
LOCKED = dict.fromkeys(("listRule", "viewRule", "createRule", "updateRule", "deleteRule"))
|
|
LOG_FIELDS = [
|
|
{"name": name, "type": kind, "required": False}
|
|
for name, kind in (
|
|
("source_system", "text"), ("source_id", "text"),
|
|
("source_logged_at", "date"), ("source_created_at", "date"),
|
|
("source_modified_at", "date"), ("import_key", "text"),
|
|
)
|
|
]
|
|
LOG_INDEXES = [
|
|
"CREATE UNIQUE INDEX `idx_fuel_logs_source_identity` ON `fuel_logs` "
|
|
"(`source_system`, `source_id`) WHERE `source_id` != ''",
|
|
"CREATE UNIQUE INDEX `idx_fuel_logs_import_key` ON `fuel_logs` "
|
|
"(`import_key`) WHERE `import_key` != ''",
|
|
]
|
|
LOCK_INDEX = "CREATE UNIQUE INDEX `idx_fuel_import_locks_scope` ON `fuel_import_locks` (`scope`)"
|
|
|
|
|
|
def definitions():
|
|
return {
|
|
"fuel_logs": {"fields": LOG_FIELDS, "indexes": LOG_INDEXES},
|
|
"fuel_sheets": {"fields": [
|
|
{"name": "rows_skipped", "type": "number", "required": False},
|
|
{"name": "rows_review", "type": "number", "required": False},
|
|
], "indexes": []},
|
|
"fuel_import_locks": {
|
|
"name": "FUEL_IMPORT_LOCKS", "type": "base",
|
|
"fields": [
|
|
{"name": "scope", "type": "text", "required": True, "min": 1},
|
|
{"name": "owner", "type": "text", "required": True, "min": 1},
|
|
],
|
|
"indexes": [LOCK_INDEX], **LOCKED,
|
|
},
|
|
}
|
|
|
|
|
|
def index_tokens(sql):
|
|
"""Compare SQL without confusing identifier quoting/case with string values."""
|
|
tokens = re.findall(r"'(?:''|[^'])*'|`[^`]*`|\"[^\"]*\"|\[[^\]]*\]|[a-zA-Z_][a-zA-Z_0-9]*|!=|<>|\S", sql)
|
|
normalized = []
|
|
for token in tokens:
|
|
if token.startswith("'"):
|
|
normalized.append(token)
|
|
elif token[0] in ('`', '"', '['):
|
|
normalized.append(token[1:-1].lower())
|
|
elif token == "<>":
|
|
normalized.append("!=")
|
|
elif token != ";":
|
|
normalized.append(token.lower())
|
|
if normalized[3:6] == ["if", "not", "exists"]:
|
|
del normalized[3:6]
|
|
elif normalized[2:5] == ["if", "not", "exists"]:
|
|
del normalized[2:5]
|
|
return normalized
|
|
|
|
|
|
def index_name(sql):
|
|
tokens = index_tokens(sql)
|
|
try:
|
|
return tokens[tokens.index("index") + 1]
|
|
except (ValueError, IndexError):
|
|
raise RuntimeError(f"Cannot identify existing index: {sql}") from None
|
|
|
|
|
|
def plan_collection(name, existing, wanted):
|
|
"""Return an additive payload; reject incompatible schema before any writes."""
|
|
if existing is None:
|
|
if name != "fuel_import_locks":
|
|
raise RuntimeError(f"Required collection {name} does not exist")
|
|
return copy.deepcopy(wanted)
|
|
if existing.get("type") != "base":
|
|
raise RuntimeError(f"Incompatible collection type: {name}")
|
|
fields = copy.deepcopy(existing.get("fields", []))
|
|
by_name = {field["name"]: field for field in fields}
|
|
missing = []
|
|
for field in wanted["fields"]:
|
|
actual = by_name.get(field["name"])
|
|
if actual is None:
|
|
missing.append(copy.deepcopy(field))
|
|
continue
|
|
for key, value in field.items():
|
|
# PB may omit default required=false/min=0 in exported schemas.
|
|
default = False if key == "required" else None
|
|
if actual.get(key, default) != value:
|
|
raise RuntimeError(f"Incompatible field: {name}.{field['name']}.{key}")
|
|
indexes = list(existing.get("indexes", []))
|
|
by_index = {index_name(sql): sql for sql in indexes}
|
|
missing_indexes = []
|
|
for sql in wanted["indexes"]:
|
|
actual = by_index.get(index_name(sql))
|
|
if actual is None:
|
|
missing_indexes.append(sql)
|
|
elif index_tokens(actual) != index_tokens(sql):
|
|
raise RuntimeError(f"Conflicting index definition: {index_name(sql)}")
|
|
if name == "fuel_import_locks":
|
|
for rule in LOCKED:
|
|
if existing.get(rule) is not None:
|
|
raise RuntimeError(f"fuel_import_locks must be superuser-only: {rule}")
|
|
changes = {}
|
|
if missing:
|
|
changes["fields"] = fields + missing
|
|
if missing_indexes:
|
|
changes["indexes"] = indexes + missing_indexes
|
|
return changes
|
|
|
|
|
|
def read_schema(name, token):
|
|
code, result = http("GET", f"/api/collections/{name}", token=token)
|
|
if code == 404:
|
|
return None
|
|
if code != 200:
|
|
raise RuntimeError(f"Cannot inspect {name}: HTTP {code}: {result}")
|
|
return result
|
|
|
|
|
|
def write_backup(path, schemas):
|
|
snapshot = {"pocketbase_url": PB, "saved_at": datetime.now(timezone.utc).isoformat(),
|
|
"collections": schemas}
|
|
if path:
|
|
# Never overwrite an earlier recovery copy; keep schema/rules private.
|
|
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
|
else:
|
|
fd, path = tempfile.mkstemp(prefix="mega-before-fuel-dedup-", suffix=".json")
|
|
with os.fdopen(fd, "w") as backup:
|
|
json.dump(snapshot, backup, indent=2)
|
|
backup.write("\n")
|
|
return str(Path(path).resolve())
|
|
|
|
|
|
def migrate(token, apply=False, backup=None):
|
|
wanted = definitions()
|
|
before = {name: read_schema(name, token) for name in wanted}
|
|
plans = {name: plan_collection(name, before[name], spec) for name, spec in wanted.items()}
|
|
for name, changes in plans.items():
|
|
action = "create" if before[name] is None else "add schema fields/indexes"
|
|
print(f"{name}: {action if changes else 'already compatible'}")
|
|
if changes:
|
|
existing_fields = {f["name"] for f in (before[name] or {}).get("fields", [])}
|
|
for field in changes.get("fields", []):
|
|
if field["name"] not in existing_fields:
|
|
print(f" + {field['name']}: {field['type']}")
|
|
existing_indexes = (before[name] or {}).get("indexes", [])
|
|
for sql in changes.get("indexes", []):
|
|
if sql not in existing_indexes:
|
|
print(f" + {sql}")
|
|
if not apply:
|
|
print("Dry-run: no schema or record writes. Use --apply to apply this plan.")
|
|
return
|
|
if not any(plans.values()):
|
|
print("Already compatible; no changes needed.")
|
|
return
|
|
backup_path = write_backup(backup, before)
|
|
print(f"Private schema backup: {backup_path}")
|
|
# Abort on concurrent schema changes instead of overwriting a newer array.
|
|
for name in wanted:
|
|
if read_schema(name, token) != before[name]:
|
|
raise RuntimeError(f"Schema changed during preflight: {name}; rerun migration")
|
|
for name, changes in plans.items():
|
|
if not changes:
|
|
continue
|
|
existing = before[name]
|
|
method = "PATCH" if existing else "POST"
|
|
path = f"/api/collections/{existing['id']}" if existing else "/api/collections"
|
|
code, result = http(method, path, changes, token)
|
|
if code != 200:
|
|
raise RuntimeError(f"{method} {name} failed: HTTP {code}: {result}; backup: {backup_path}")
|
|
for name, spec in wanted.items():
|
|
actual = read_schema(name, token)
|
|
if actual is None or plan_collection(name, actual, spec):
|
|
raise RuntimeError(f"Verification failed: {name}")
|
|
original = before[name]
|
|
if original:
|
|
actual_fields = {f["name"]: f for f in actual["fields"]}
|
|
for field in original.get("fields", []):
|
|
if any(actual_fields.get(field["name"], {}).get(k) != v for k, v in field.items()):
|
|
raise RuntimeError(f"Existing field changed: {name}.{field['name']}")
|
|
for rule in LOCKED:
|
|
if actual.get(rule) != original.get(rule):
|
|
raise RuntimeError(f"Existing access rule changed: {name}.{rule}")
|
|
actual_indexes = [index_tokens(sql) for sql in actual.get("indexes", [])]
|
|
if any(index_tokens(sql) not in actual_indexes for sql in original.get("indexes", [])):
|
|
raise RuntimeError(f"Existing indexes changed: {name}")
|
|
print("Verified fuel import schema. No fuel records or lock records changed.")
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
mode = parser.add_mutually_exclusive_group()
|
|
mode.add_argument("--apply", action="store_true", help="Apply additive schema changes")
|
|
mode.add_argument("--dry-run", action="store_true", help="Read and print schema plan (default)")
|
|
parser.add_argument("--backup", help="New private JSON backup path (default: unique /tmp file)")
|
|
args = parser.parse_args()
|
|
migrate(superuser_token(), apply=args.apply, backup=args.backup)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
main()
|
|
except (OSError, RuntimeError) as exc:
|
|
sys.exit(str(exc))
|