MEGAINFRA-WEBAPP/scripts/migrate-dn-item-units.py

#!/usr/bin/env python3
"""
Backfills qty_raw / unit_raw / unit_converted on delivery_note_items and
rewrites qty_received into the purchase order line's unit.

Before this ran, qty_received held whatever the slip said — kg for
weighbridge tickets — while the PO line it points at is priced per ton.
Every fulfilment percentage and received value computed from those rows
was 1000x too large.

The slip's unit survives only in condition_notes, which the OCR callback
wrote as:

    OCR 95% · desc: G6 Sub Base · unit: kg · match: 100%

Rows with no `unit:` segment are left ALONE and reported. Guessing their
unit is how the original bug happened.

Idempotent — rows that already have a non-empty unit_raw are skipped.

Usage:
    python3 scripts/migrate-dn-item-units.py            # dry run
    python3 scripts/migrate-dn-item-units.py --apply    # writes
"""
import re
import sys

from pb_env import PB, http, superuser_token

UNIT_RE = re.compile(r"unit:\s*([^·]+)")

ALIASES = {
    "kg": ["kg", "kgs", "kilo", "kilos", "kilogram", "kilograms"],
    "ton": ["t", "ton", "tons", "tonne", "tonnes", "mt", "metric ton"],
    "m3": ["m3", "m³", "cube", "cubes", "cubic metre", "cubic meter"],
    "m2": ["m2", "m²", "square metre", "square meter"],
    "m": ["m", "metre", "meter", "metres", "meters", "lm", "linear metre"],
    "ea": ["ea", "each", "unit", "units", "no", "nr", "item", "items"],
    "bag": ["bag", "bags", "pocket", "pockets"],
    "l": ["l", "litre", "litres", "liter", "liters"],
    "load": ["load", "loads", "trip", "trips"],
}
LOOKUP = {a: c for c, al in ALIASES.items() for a in al}

FACTORS = {
    ("kg", "ton"): 1 / 1000,
    ("ton", "kg"): 1000,
    ("l", "m3"): 1 / 1000,
    ("m3", "l"): 1000,
}


def normalise_unit(raw):
    """Mirror of normaliseUnit in src/lib/units.ts. Keep the two in step."""
    if not isinstance(raw, str):
        return None
    cleaned = re.sub(r"\s+", " ", re.sub(r"[^a-z0-9²³ ]", " ", raw.lower())).strip()
    return LOOKUP.get(cleaned) if cleaned else None


def convert(qty, from_unit, to_unit):
    """Returns (qty, converted, failure_reason)."""
    f, t = normalise_unit(from_unit), normalise_unit(to_unit)
    if f is None or t is None:
        return None, False, "unknown_unit"
    if f == t:
        return qty, False, None
    factor = FACTORS.get((f, t))
    if factor is None:
        return None, False, "incompatible_units"
    return round(qty * factor, 4), True, None


def get_all(token, collection, fields=None, filt=None):
    out, page = [], 1
    while True:
        q = f"?page={page}&perPage=200"
        if fields:
            q += f"&fields={fields}"
        if filt:
            q += f"&filter={filt}"
        code, resp = http("GET", f"/api/collections/{collection}/records{q}", token=token)
        if code != 200:
            sys.exit(f"GET {collection} failed: {code} {resp}")
        out.extend(resp["items"])
        if page >= resp["totalPages"]:
            return out
        page += 1


def main(apply: bool) -> None:
    token = superuser_token()
    print(f"\n→ delivery_note_items on {PB}  ({'APPLY' if apply else 'DRY RUN'})")

    po_items = {
        p["id"]: p
        for p in get_all(token, "purchase_order_items", "id,unit,description")
    }
    rows = get_all(
        token,
        "delivery_note_items",
        "id,delivery_note,purchase_order_item,qty_received,qty_raw,unit_raw,condition_notes",
    )
    print(f"  {len(rows)} rows, {len(po_items)} purchase order items\n")

    changed = skipped = no_unit = unconvertible = 0
    print(f"  {'id':17s} {'from':>14s}  {'to':>14s}  note")
    print(f"  {'-' * 17} {'-' * 14}  {'-' * 14}  {'-' * 30}")

    for r in rows:
        rid = r["id"]
        if (r.get("unit_raw") or "").strip():
            skipped += 1
            continue

        qty = r.get("qty_received") or 0
        notes = r.get("condition_notes") or ""
        m = UNIT_RE.search(notes)
        if not m:
            no_unit += 1
            print(f"  {rid:17s} {qty:>14} {'—':>14}  no unit: segment — LEFT ALONE")
            continue
        unit_raw = m.group(1).strip()

        poi = po_items.get(r.get("purchase_order_item") or "")
        if not poi:
            # Unlinked line: record what the slip said, leave qty_received.
            patch = {"qty_raw": qty, "unit_raw": unit_raw, "unit_converted": False}
            print(f"  {rid:17s} {qty:>10} {unit_raw:<3}  {'unlinked':>14}  qty_raw only")
            if apply:
                code, resp = http("PATCH", f"/api/collections/delivery_note_items/records/{rid}", patch, token)
                if code != 200:
                    sys.exit(f"PATCH {rid} failed: {code} {resp}")
            changed += 1
            continue

        po_unit = poi.get("unit") or ""
        new_qty, converted, failure = convert(qty, unit_raw, po_unit)

        if failure:
            unconvertible += 1
            patch = {
                "qty_raw": qty,
                "unit_raw": unit_raw,
                "unit_converted": False,
                "qty_received": 0,
                "purchase_order_item": None,
                "condition_notes": f"{notes} · unconverted: {failure}",
            }
            print(f"  {rid:17s} {qty:>10} {unit_raw:<3}  {'0':>10} {po_unit:<3}  {failure} — link dropped")
        else:
            patch = {
                "qty_raw": qty,
                "unit_raw": unit_raw,
                "unit_converted": converted,
                "qty_received": new_qty,
            }
            print(f"  {rid:17s} {qty:>10} {unit_raw:<3}  {new_qty:>10} {po_unit:<3}  {poi.get('description', '')[:26]}")

        if apply:
            code, resp = http("PATCH", f"/api/collections/delivery_note_items/records/{rid}", patch, token)
            if code != 200:
                sys.exit(f"PATCH {rid} failed: {code} {resp}")
        changed += 1

    print(
        f"\n  changed: {changed}   already done: {skipped}   "
        f"no unit recorded: {no_unit}   unconvertible: {unconvertible}"
    )
    if not apply:
        print("\n  DRY RUN — nothing written. Re-run with --apply.")


if __name__ == "__main__":
    main("--apply" in sys.argv)
    print("\nDone.")