feat(site-map): model upload script with progress-key parity

scripts/site-model/model_keys.py

@@ -0,0 +1,27 @@
+"""Progress keys of a model folder. MUST match src/lib/site-map/keys.ts (modelKeyIndex) —
+both are tested against src/lib/site-map/__fixtures__/mini-model/expected-keys.json."""
+import json
+from pathlib import Path
+
+
+def model_keys(folder):
+    folder = Path(folder)
+    load = lambda name: json.loads((folder / f"{name}.json").read_text())
+    road, services, earth, sched = load("bushveld"), load("services"), load("earthworks"), load("schedules")
+    keys = set()
+    for a in road["areas"]:
+        for layer in road["builds"].get(a["kind"], {}).get("layers", []):
+            keys.add(f"road:{a['section']}:{a['kind']}:{layer['id']}")
+    keys.update(f"kerb:{k['section']}" for k in road["kerbs"])
+    keys.update(f"peg:{p['name']}" for p in road["pegs"])
+    keys.update(f"swpipe:{p['from']}>{p['to']}" for p in services["stormwater"]["pipes"])
+    keys.update(f"sw:{s['name']}" for s in services["stormwater"]["structures"])
+    names = {m["id"]: m["name"] for m in services["sewer"]["manholes"]}
+    keys.update(f"srpipe:{names.get(p['from'], p['from'])}>{names.get(p['to'], p['to'])}" for p in services["sewer"]["pipes"])
+    keys.update(f"sr:{m['name']}" for m in services["sewer"]["manholes"])
+    keys.update(f"earth:{v['from']}-{v['to']}" for v in earth["volumes"])
+    keys.update(f"sign:{c}" for c in sched["signCounts"])
+    keys.update(f"stud:{t}" for t in sched["studCounts"])
+    keys.update(f"fit:{i}" for i in sched["waterFittings"])
+    keys.add("paint")
+    return sorted(keys)

scripts/site-model/test_model_keys.py

@@ -0,0 +1,20 @@
+import importlib.util
+import json
+import unittest
+from pathlib import Path
+
+HERE = Path(__file__).resolve().parent
+FIXTURE = HERE.parent.parent / "src/lib/site-map/__fixtures__/mini-model"
+spec = importlib.util.spec_from_file_location("model_keys", HERE / "model_keys.py")
+model_keys = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(model_keys)
+
+
+class ModelKeysTests(unittest.TestCase):
+    def test_same_keys_as_typescript(self):
+        expected = json.loads((FIXTURE / "expected-keys.json").read_text())
+        self.assertEqual(model_keys.model_keys(FIXTURE), expected)
+
+
+if __name__ == "__main__":
+    unittest.main()

scripts/site-model/test_upload.py

@@ -0,0 +1,27 @@
+import importlib.util
+import sys
+import unittest
+from pathlib import Path
+
+HERE = Path(__file__).resolve().parent
+sys.path.insert(0, str(HERE.parent))           # pb_env
+spec = importlib.util.spec_from_file_location("upload", HERE / "upload.py")
+upload = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(upload)
+
+
+class UploadTests(unittest.TestCase):
+    def test_multipart_contains_fields_and_files(self):
+        body, ctype = upload.multipart({"label": "Rev 1", "status": "active"}, [("files", "bushveld.json", b'{"a":1}', "application/json")])
+        self.assertTrue(ctype.startswith("multipart/form-data; boundary="))
+        boundary = ctype.split("boundary=")[1].encode()
+        self.assertIn(b'name="label"\r\n\r\nRev 1\r\n', body)
+        self.assertIn(b'filename="bushveld.json"\r\nContent-Type: application/json\r\n\r\n{"a":1}\r\n', body)
+        self.assertTrue(body.endswith(b"--" + boundary + b"--\r\n"))
+
+    def test_orphaned_keys(self):
+        self.assertEqual(upload.orphaned({"peg:BJ1", "peg:OLD", "paint"}, {"peg:BJ1", "paint"}), ["peg:OLD"])
+
+
+if __name__ == "__main__":
+    unittest.main()

scripts/site-model/upload.py

@@ -0,0 +1,95 @@
+#!/usr/bin/env python3
+"""Upload a built site model and make it the site's active model.
+
+    python3 scripts/site-model/upload.py --site <site record id> --label "Bushveld Joy Road — RD LO Rev 1" /tmp/site-model-hills
+    add --dry-run to only print the key report.
+
+Creates a site_models row (status active) with the five JSON files, archives the previous
+active model, and prints progress keys recorded on the site that the new model no longer has.
+"""
+import argparse
+import json
+import sys
+import uuid
+import urllib.request
+from datetime import datetime, timezone
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from pb_env import PB, http, superuser_token  # noqa: E402
+from model_keys import model_keys  # noqa: E402
+
+FILES = ["bushveld", "services", "earthworks", "schedules", "surface"]
+
+
+def multipart(fields, files):
+    boundary = uuid.uuid4().hex
+    out = b""
+    for name, value in fields.items():
+        out += f'--{boundary}\r\nContent-Disposition: form-data; name="{name}"\r\n\r\n{value}\r\n'.encode()
+    for field, filename, data, ctype in files:
+        out += f'--{boundary}\r\nContent-Disposition: form-data; name="{field}"; filename="{filename}"\r\nContent-Type: {ctype}\r\n\r\n'.encode()
+        out += data + b"\r\n"
+    out += f"--{boundary}--\r\n".encode()
+    return out, f"multipart/form-data; boundary={boundary}"
+
+
+def orphaned(existing_keys, new_keys):
+    return sorted(existing_keys - new_keys)
+
+
+def existing_progress_keys(token, site):
+    keys, page = set(), 1
+    while True:
+        code, res = http("GET", f"/api/collections/site_progress_events/records?page={page}&perPage=500&fields=key&filter=" +
+                         urllib.request.quote(f"site='{site}'"), token=token)
+        if code != 200:
+            raise RuntimeError(f"Reading progress failed: HTTP {code}")
+        keys.update(r["key"] for r in res["items"])
+        if page >= res["totalPages"]:
+            return keys
+        page += 1
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--site", required=True)
+    ap.add_argument("--label", required=True)
+    ap.add_argument("--dry-run", action="store_true")
+    ap.add_argument("folder")
+    args = ap.parse_args()
+    folder = Path(args.folder)
+    missing = [f for f in FILES if not (folder / f"{f}.json").exists()]
+    if missing:
+        sys.exit(f"Missing model files: {missing}")
+    token = superuser_token()
+    new_keys = set(model_keys(folder))
+    gone = orphaned(existing_progress_keys(token, args.site), new_keys)
+    print(f"{len(new_keys)} progress items in the new model")
+    if gone:
+        print(f"WARNING: {len(gone)} recorded progress keys are not in the new model:")
+        for k in gone[:50]:
+            print("  ", k)
+    if args.dry_run:
+        return
+    road = json.loads((folder / "bushveld.json").read_text())
+    stats = {"asphalt_m2": round(sum(a["area"] for a in road["areas"] if a["kind"] == "asphalt"), 1), "sections": len(road["sections"])}
+    body, ctype = multipart(
+        {"site": args.site, "label": args.label, "status": "active", "built_at": datetime.now(timezone.utc).isoformat(),
+         "sources": json.dumps({"folder": str(folder)}), "stats": json.dumps(stats)},
+        [("files", f"{f}.json", (folder / f"{f}.json").read_bytes(), "application/json") for f in FILES])
+    req = urllib.request.Request(f"{PB}/api/collections/site_models/records", data=body, method="POST")
+    req.add_header("Content-Type", ctype); req.add_header("Authorization", token)
+    with urllib.request.urlopen(req) as r:
+        created = json.loads(r.read().decode())
+    print(f"Created model {created['id']} ({args.label}) — active")
+    code, res = http("GET", "/api/collections/site_models/records?perPage=200&filter=" +
+                     urllib.request.quote(f"site='{args.site}' && status='active' && id!='{created['id']}'"), token=token)
+    for old in res.get("items", []):
+        http("PATCH", f"/api/collections/site_models/records/{old['id']}", {"status": "archived"}, token)
+        print(f"Archived previous model {old['id']}")
+
+
+if __name__ == "__main__":
+    main()