From 300b1d2a07ee8dac893d240da6fde6d886a9bea3 Mon Sep 17 00:00:00 2001 From: DEls Date: Wed, 14 Oct 2026 07:06:13 +0200 Subject: [PATCH] feat(site-map): model upload script with progress-key parity --- scripts/site-model/model_keys.py | 27 ++++++++ scripts/site-model/test_model_keys.py | 20 ++++++ scripts/site-model/test_upload.py | 27 ++++++++ scripts/site-model/upload.py | 95 +++++++++++++++++++++++++++ 4 files changed, 169 insertions(+) create mode 100644 scripts/site-model/model_keys.py create mode 100644 scripts/site-model/test_model_keys.py create mode 100644 scripts/site-model/test_upload.py create mode 100644 scripts/site-model/upload.py diff --git a/scripts/site-model/model_keys.py b/scripts/site-model/model_keys.py new file mode 100644 index 0000000..f6779fb --- /dev/null +++ b/scripts/site-model/model_keys.py @@ -0,0 +1,27 @@ +"""Progress keys of a model folder. MUST match src/lib/site-map/keys.ts (modelKeyIndex) — +both are tested against src/lib/site-map/__fixtures__/mini-model/expected-keys.json.""" +import json +from pathlib import Path + + +def model_keys(folder): + folder = Path(folder) + load = lambda name: json.loads((folder / f"{name}.json").read_text()) + road, services, earth, sched = load("bushveld"), load("services"), load("earthworks"), load("schedules") + keys = set() + for a in road["areas"]: + for layer in road["builds"].get(a["kind"], {}).get("layers", []): + keys.add(f"road:{a['section']}:{a['kind']}:{layer['id']}") + keys.update(f"kerb:{k['section']}" for k in road["kerbs"]) + keys.update(f"peg:{p['name']}" for p in road["pegs"]) + keys.update(f"swpipe:{p['from']}>{p['to']}" for p in services["stormwater"]["pipes"]) + keys.update(f"sw:{s['name']}" for s in services["stormwater"]["structures"]) + names = {m["id"]: m["name"] for m in services["sewer"]["manholes"]} + keys.update(f"srpipe:{names.get(p['from'], p['from'])}>{names.get(p['to'], p['to'])}" for p in services["sewer"]["pipes"]) + keys.update(f"sr:{m['name']}" for m in services["sewer"]["manholes"]) + keys.update(f"earth:{v['from']}-{v['to']}" for v in earth["volumes"]) + keys.update(f"sign:{c}" for c in sched["signCounts"]) + keys.update(f"stud:{t}" for t in sched["studCounts"]) + keys.update(f"fit:{i}" for i in sched["waterFittings"]) + keys.add("paint") + return sorted(keys) diff --git a/scripts/site-model/test_model_keys.py b/scripts/site-model/test_model_keys.py new file mode 100644 index 0000000..80b5caa --- /dev/null +++ b/scripts/site-model/test_model_keys.py @@ -0,0 +1,20 @@ +import importlib.util +import json +import unittest +from pathlib import Path + +HERE = Path(__file__).resolve().parent +FIXTURE = HERE.parent.parent / "src/lib/site-map/__fixtures__/mini-model" +spec = importlib.util.spec_from_file_location("model_keys", HERE / "model_keys.py") +model_keys = importlib.util.module_from_spec(spec) +spec.loader.exec_module(model_keys) + + +class ModelKeysTests(unittest.TestCase): + def test_same_keys_as_typescript(self): + expected = json.loads((FIXTURE / "expected-keys.json").read_text()) + self.assertEqual(model_keys.model_keys(FIXTURE), expected) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/site-model/test_upload.py b/scripts/site-model/test_upload.py new file mode 100644 index 0000000..94be3bb --- /dev/null +++ b/scripts/site-model/test_upload.py @@ -0,0 +1,27 @@ +import importlib.util +import sys +import unittest +from pathlib import Path + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE.parent)) # pb_env +spec = importlib.util.spec_from_file_location("upload", HERE / "upload.py") +upload = importlib.util.module_from_spec(spec) +spec.loader.exec_module(upload) + + +class UploadTests(unittest.TestCase): + def test_multipart_contains_fields_and_files(self): + body, ctype = upload.multipart({"label": "Rev 1", "status": "active"}, [("files", "bushveld.json", b'{"a":1}', "application/json")]) + self.assertTrue(ctype.startswith("multipart/form-data; boundary=")) + boundary = ctype.split("boundary=")[1].encode() + self.assertIn(b'name="label"\r\n\r\nRev 1\r\n', body) + self.assertIn(b'filename="bushveld.json"\r\nContent-Type: application/json\r\n\r\n{"a":1}\r\n', body) + self.assertTrue(body.endswith(b"--" + boundary + b"--\r\n")) + + def test_orphaned_keys(self): + self.assertEqual(upload.orphaned({"peg:BJ1", "peg:OLD", "paint"}, {"peg:BJ1", "paint"}), ["peg:OLD"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/site-model/upload.py b/scripts/site-model/upload.py new file mode 100644 index 0000000..3bb5621 --- /dev/null +++ b/scripts/site-model/upload.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +"""Upload a built site model and make it the site's active model. + + python3 scripts/site-model/upload.py --site --label "Bushveld Joy Road — RD LO Rev 1" /tmp/site-model-hills + add --dry-run to only print the key report. + +Creates a site_models row (status active) with the five JSON files, archives the previous +active model, and prints progress keys recorded on the site that the new model no longer has. +""" +import argparse +import json +import sys +import uuid +import urllib.request +from datetime import datetime, timezone +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from pb_env import PB, http, superuser_token # noqa: E402 +from model_keys import model_keys # noqa: E402 + +FILES = ["bushveld", "services", "earthworks", "schedules", "surface"] + + +def multipart(fields, files): + boundary = uuid.uuid4().hex + out = b"" + for name, value in fields.items(): + out += f'--{boundary}\r\nContent-Disposition: form-data; name="{name}"\r\n\r\n{value}\r\n'.encode() + for field, filename, data, ctype in files: + out += f'--{boundary}\r\nContent-Disposition: form-data; name="{field}"; filename="{filename}"\r\nContent-Type: {ctype}\r\n\r\n'.encode() + out += data + b"\r\n" + out += f"--{boundary}--\r\n".encode() + return out, f"multipart/form-data; boundary={boundary}" + + +def orphaned(existing_keys, new_keys): + return sorted(existing_keys - new_keys) + + +def existing_progress_keys(token, site): + keys, page = set(), 1 + while True: + code, res = http("GET", f"/api/collections/site_progress_events/records?page={page}&perPage=500&fields=key&filter=" + + urllib.request.quote(f"site='{site}'"), token=token) + if code != 200: + raise RuntimeError(f"Reading progress failed: HTTP {code}") + keys.update(r["key"] for r in res["items"]) + if page >= res["totalPages"]: + return keys + page += 1 + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--site", required=True) + ap.add_argument("--label", required=True) + ap.add_argument("--dry-run", action="store_true") + ap.add_argument("folder") + args = ap.parse_args() + folder = Path(args.folder) + missing = [f for f in FILES if not (folder / f"{f}.json").exists()] + if missing: + sys.exit(f"Missing model files: {missing}") + token = superuser_token() + new_keys = set(model_keys(folder)) + gone = orphaned(existing_progress_keys(token, args.site), new_keys) + print(f"{len(new_keys)} progress items in the new model") + if gone: + print(f"WARNING: {len(gone)} recorded progress keys are not in the new model:") + for k in gone[:50]: + print(" ", k) + if args.dry_run: + return + road = json.loads((folder / "bushveld.json").read_text()) + stats = {"asphalt_m2": round(sum(a["area"] for a in road["areas"] if a["kind"] == "asphalt"), 1), "sections": len(road["sections"])} + body, ctype = multipart( + {"site": args.site, "label": args.label, "status": "active", "built_at": datetime.now(timezone.utc).isoformat(), + "sources": json.dumps({"folder": str(folder)}), "stats": json.dumps(stats)}, + [("files", f"{f}.json", (folder / f"{f}.json").read_bytes(), "application/json") for f in FILES]) + req = urllib.request.Request(f"{PB}/api/collections/site_models/records", data=body, method="POST") + req.add_header("Content-Type", ctype); req.add_header("Authorization", token) + with urllib.request.urlopen(req) as r: + created = json.loads(r.read().decode()) + print(f"Created model {created['id']} ({args.label}) — active") + code, res = http("GET", "/api/collections/site_models/records?perPage=200&filter=" + + urllib.request.quote(f"site='{args.site}' && status='active' && id!='{created['id']}'"), token=token) + for old in res.get("items", []): + http("PATCH", f"/api/collections/site_models/records/{old['id']}", {"status": "archived"}, token) + print(f"Archived previous model {old['id']}") + + +if __name__ == "__main__": + main()