diff --git a/scripts/migrations/2026-09-16-migrate-suppliers.py b/scripts/migrations/2026-09-16-migrate-suppliers.py new file mode 100644 index 0000000..58c9635 --- /dev/null +++ b/scripts/migrations/2026-09-16-migrate-suppliers.py @@ -0,0 +1,186 @@ +#!/usr/bin/env python3 +""" +Migrate purchase_orders.supplier (free text) onto real suppliers records. + + PB_URL=https://pb.mega.infra.apphub.co.za \\ + PB_ADMIN_EMAIL=admin@mega.co.za \\ + PB_ADMIN_PASSWORD=... \\ + python3 scripts/migrations/2026-09-16-migrate-suppliers.py [--apply] + +DRY RUN BY DEFAULT. + +For every purchase_orders row with a non-empty, non-placeholder +`supplier` value: + 1. normalize it (trim, collapse whitespace, uppercase -- same rule as + src/lib/purchase-order-schema.ts normalizeSupplierName) + 2. group POs by normalized value + 3. create one suppliers record per group (name = the LONGEST raw + variant in the group -- a proxy for "most complete", e.g. "Global + Precast (Pty) Ltd" over "Global Precast" if both existed in one + group; imperfect but good enough for a one-time seed the operator + can rename) + 4. link every PO in the group to that supplier via supplier_ref + +Against production on 2026-09-11: 24 POs, 19 distinct raw supplier +strings, 18 normalized groups (one merge -- a case-only variant of +"Builders Warehouse"). If your dry run shows a materially different +grouping, that is new production data since then, not a bug in this +script -- read the printed groups before assuming otherwise. Do not +"fix" a low merge count by loosening the normalization; see +docs/superpowers/specs/2026-09-11-suppliers-design.md section 1 for why +that would be a mistake, not an improvement. + +A PO whose supplier value is empty or a known placeholder (see +UNKNOWN_SUPPLIER_PLACEHOLDERS below, which must match +src/lib/utils.ts's isUnknownSupplier alias list) is left with no +supplier_ref -- it goes through the normal "unmatched" flow that the +Task 9 UI change surfaces, not this migration. + +Guard: aborts (writes nothing) if any PO has a non-placeholder, +non-empty supplier value AND no supplier_ref after a run -- that would +mean this script's own grouping logic missed a row, which should be +impossible; the guard exists so a bug here fails loudly rather than +silently leaving a PO unlinked. + +Idempotent: a PO that already has supplier_ref set is skipped. Running +after a successful apply reports 0 to migrate. +""" +import json +import os +import sys +import urllib.error +import urllib.parse +import urllib.request + +APPLY = "--apply" in sys.argv + +# Must match src/lib/utils.ts's UNKNOWN_SUPPLIER_ALIASES exactly. +UNKNOWN_SUPPLIER_PLACEHOLDERS = { + "", "—", "-", "n/a", "na", "unknown", "not specified", + "not specified in document", +} + + +def normalize(s): + return " ".join(s.strip().split()).upper() + + +def is_placeholder(s): + return (s or "").strip().lower() in UNKNOWN_SUPPLIER_PLACEHOLDERS + + +def admin_auth(base_url): + body = json.dumps({ + "identity": os.environ["PB_ADMIN_EMAIL"], + "password": os.environ["PB_ADMIN_PASSWORD"], + }).encode() + req = urllib.request.Request( + f"{base_url}/api/collections/_superusers/auth-with-password", + data=body, headers={"Content-Type": "application/json"}, method="POST") + with urllib.request.urlopen(req) as r: + return json.loads(r.read())["token"] + + +def api(base_url, token, method, path, body=None): + data = json.dumps(body).encode() if body is not None else None + req = urllib.request.Request( + f"{base_url}{path}", data=data, method=method, + headers={"Authorization": token, "Content-Type": "application/json"}) + try: + with urllib.request.urlopen(req) as r: + txt = r.read().decode() + return json.loads(txt) if txt else None + except urllib.error.HTTPError as e: + if e.code == 404 and method == "GET": + return None + raise RuntimeError( + f"{method} {path}: {e.code} {e.read().decode() if e.fp else ''}" + ) from e + + +def get_all(base_url, token, coll, fields, flt=""): + out, page = [], 1 + while True: + path = f"/api/collections/{coll}/records?perPage=500&page={page}&fields={fields}" + if flt: + path += "&filter=" + urllib.parse.quote(flt) + data = api(base_url, token, "GET", path) + out.extend(data["items"]) + if page >= data["totalPages"]: + return out + page += 1 + + +def main(): + base_url = os.environ.get("PB_URL", "http://127.0.0.1:8090").rstrip("/") + token = admin_auth(base_url) + print(f"target: {base_url} mode: {'APPLY' if APPLY else 'DRY RUN'}") + + pos = get_all(base_url, token, "purchase_orders", "id,purchase_order_number,supplier,supplier_ref") + + already_linked = [p for p in pos if p.get("supplier_ref")] + placeholder = [p for p in pos if not p.get("supplier_ref") and is_placeholder(p.get("supplier"))] + to_migrate = [ + p for p in pos + if not p.get("supplier_ref") and not is_placeholder(p.get("supplier")) + ] + + print(f"\npurchase_orders total={len(pos)} " + f"already_linked={len(already_linked)} " + f"placeholder_supplier={len(placeholder)} " + f"to_migrate={len(to_migrate)}") + + groups = {} + for p in to_migrate: + key = normalize(p["supplier"]) + groups.setdefault(key, []).append(p) + + print(f"\n{len(to_migrate)} POs -> {len(groups)} normalized groups:\n") + for key, members in sorted(groups.items()): + raw_variants = sorted({m["supplier"] for m in members}) + tag = " <-- MERGE" if len(raw_variants) > 1 else "" + print(f" {key!r:50} POs={len(members):<3} raw={raw_variants}{tag}") + + if not APPLY: + print("\nDRY RUN. Re-run with --apply to write.") + return 0 + + for key, members in groups.items(): + # Longest raw variant as the seed display name -- see docstring. + display_name = max({m["supplier"] for m in members}, key=len).strip() + # Store the NORMALIZED form, so matchSupplier's exact-match + # lookup (which compares against normalized input) finds it. + supplier = api(base_url, token, "POST", "/api/collections/suppliers/records", + {"name": key}) + print(f"created supplier {supplier['id']} name={key!r} (display seed: {display_name!r})") + for m in members: + api(base_url, token, "PATCH", + f"/api/collections/purchase_orders/records/{m['id']}", + {"supplier_ref": supplier["id"]}) + print(f" linked {len(members)} PO(s)") + + # --- guard / verify --- + pos_after = get_all(base_url, token, "purchase_orders", "id,supplier,supplier_ref") + orphaned = [ + p for p in pos_after + if not p.get("supplier_ref") and not is_placeholder(p.get("supplier")) + ] + if orphaned: + print(f"\nABORT-AFTER-WRITE: {len(orphaned)} PO(s) still have no " + f"supplier_ref despite a non-placeholder supplier value. " + f"This means the grouping logic missed something. IDs: " + f"{[p['id'] for p in orphaned]}") + return 1 + + print(f"\nOK. {len(groups)} supplier(s) created, " + f"{sum(len(m) for m in groups.values())} PO(s) linked, " + f"0 orphaned.") + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except KeyError as e: + print(f"ERROR: missing environment variable {e}", file=sys.stderr) + sys.exit(2) \ No newline at end of file