#!/usr/bin/env python3 """ Backfills qty_raw / unit_raw / unit_converted on delivery_note_items and rewrites qty_received into the purchase order line's unit. Before this ran, qty_received held whatever the slip said — kg for weighbridge tickets — while the PO line it points at is priced per ton. Every fulfilment percentage and received value computed from those rows was 1000x too large. The slip's unit survives only in condition_notes, which the OCR callback wrote as: OCR 95% · desc: G6 Sub Base · unit: kg · match: 100% Rows with no `unit:` segment are left ALONE and reported. Guessing their unit is how the original bug happened. Idempotent — rows that already have a non-empty unit_raw are skipped. Usage: python3 scripts/migrate-dn-item-units.py # dry run python3 scripts/migrate-dn-item-units.py --apply # writes """ import re import sys from pb_env import PB, http, superuser_token UNIT_RE = re.compile(r"unit:\s*([^·]+)") ALIASES = { "kg": ["kg", "kgs", "kilo", "kilos", "kilogram", "kilograms"], "ton": ["t", "ton", "tons", "tonne", "tonnes", "mt", "metric ton"], "m3": ["m3", "m³", "cube", "cubes", "cubic metre", "cubic meter"], "m2": ["m2", "m²", "square metre", "square meter"], "m": ["m", "metre", "meter", "metres", "meters", "lm", "linear metre"], "ea": ["ea", "each", "unit", "units", "no", "nr", "item", "items"], "bag": ["bag", "bags", "pocket", "pockets"], "l": ["l", "litre", "litres", "liter", "liters"], "load": ["load", "loads", "trip", "trips"], } LOOKUP = {a: c for c, al in ALIASES.items() for a in al} FACTORS = { ("kg", "ton"): 1 / 1000, ("ton", "kg"): 1000, ("l", "m3"): 1 / 1000, ("m3", "l"): 1000, } def normalise_unit(raw): """Mirror of normaliseUnit in src/lib/units.ts. Keep the two in step.""" if not isinstance(raw, str): return None cleaned = re.sub(r"\s+", " ", re.sub(r"[^a-z0-9²³ ]", " ", raw.lower())).strip() return LOOKUP.get(cleaned) if cleaned else None def convert(qty, from_unit, to_unit): """Returns (qty, converted, failure_reason).""" f, t = normalise_unit(from_unit), normalise_unit(to_unit) if f is None or t is None: return None, False, "unknown_unit" if f == t: return qty, False, None factor = FACTORS.get((f, t)) if factor is None: return None, False, "incompatible_units" return round(qty * factor, 4), True, None def get_all(token, collection, fields=None, filt=None): out, page = [], 1 while True: q = f"?page={page}&perPage=200" if fields: q += f"&fields={fields}" if filt: q += f"&filter={filt}" code, resp = http("GET", f"/api/collections/{collection}/records{q}", token=token) if code != 200: sys.exit(f"GET {collection} failed: {code} {resp}") out.extend(resp["items"]) if page >= resp["totalPages"]: return out page += 1 def main(apply: bool) -> None: token = superuser_token() print(f"\n→ delivery_note_items on {PB} ({'APPLY' if apply else 'DRY RUN'})") po_items = { p["id"]: p for p in get_all(token, "purchase_order_items", "id,unit,description") } rows = get_all( token, "delivery_note_items", "id,delivery_note,purchase_order_item,qty_received,qty_raw,unit_raw,condition_notes", ) print(f" {len(rows)} rows, {len(po_items)} purchase order items\n") changed = skipped = no_unit = unconvertible = 0 print(f" {'id':17s} {'from':>14s} {'to':>14s} note") print(f" {'-' * 17} {'-' * 14} {'-' * 14} {'-' * 30}") for r in rows: rid = r["id"] if (r.get("unit_raw") or "").strip(): skipped += 1 continue qty = r.get("qty_received") or 0 notes = r.get("condition_notes") or "" m = UNIT_RE.search(notes) if not m: no_unit += 1 print(f" {rid:17s} {qty:>14} {'—':>14} no unit: segment — LEFT ALONE") continue unit_raw = m.group(1).strip() poi = po_items.get(r.get("purchase_order_item") or "") if not poi: # Unlinked line: record what the slip said, leave qty_received. patch = {"qty_raw": qty, "unit_raw": unit_raw, "unit_converted": False} print(f" {rid:17s} {qty:>10} {unit_raw:<3} {'unlinked':>14} qty_raw only") if apply: code, resp = http("PATCH", f"/api/collections/delivery_note_items/records/{rid}", patch, token) if code != 200: sys.exit(f"PATCH {rid} failed: {code} {resp}") changed += 1 continue po_unit = poi.get("unit") or "" new_qty, converted, failure = convert(qty, unit_raw, po_unit) if failure: unconvertible += 1 patch = { "qty_raw": qty, "unit_raw": unit_raw, "unit_converted": False, "qty_received": 0, "purchase_order_item": None, "condition_notes": f"{notes} · unconverted: {failure}", } print(f" {rid:17s} {qty:>10} {unit_raw:<3} {'0':>10} {po_unit:<3} {failure} — link dropped") else: patch = { "qty_raw": qty, "unit_raw": unit_raw, "unit_converted": converted, "qty_received": new_qty, } print(f" {rid:17s} {qty:>10} {unit_raw:<3} {new_qty:>10} {po_unit:<3} {poi.get('description', '')[:26]}") if apply: code, resp = http("PATCH", f"/api/collections/delivery_note_items/records/{rid}", patch, token) if code != 200: sys.exit(f"PATCH {rid} failed: {code} {resp}") changed += 1 print( f"\n changed: {changed} already done: {skipped} " f"no unit recorded: {no_unit} unconvertible: {unconvertible}" ) if not apply: print("\n DRY RUN — nothing written. Re-run with --apply.") if __name__ == "__main__": main("--apply" in sys.argv) print("\nDone.")