MEGAINFRA-WEBAPP/scripts/migrations/2026-09-11-backfill-entity-file-hashes.py

#!/usr/bin/env python3
"""
Backfill file_hash on delivery_notes and purchase_orders by hashing the
file stored on the record itself.

  PB_URL=... PB_ADMIN_EMAIL=... PB_ADMIN_PASSWORD=... \\
  python3 scripts/migrations/2026-09-11-backfill-entity-file-hashes.py [--apply]

DRY RUN BY DEFAULT.

Part of the ingestion restructure (see
docs/superpowers/specs/2026-09-11-ingestion-restructure-design.md).
Task 9 moves delivery-note dedup off `data_import_batches` and onto the DN
row; Task 13 does the same for POs. Both need a hash on the entity.

WHY WE HASH RATHER THAN COPY FROM THE BATCH ROW
-----------------------------------------------
The obvious shortcut is to copy `data_import_batches.file_hash` across via
`processed_record_id`. Production says no:

  * 10 delivery-note batches have no file_hash at all - they predate
    commit 78fa8da which added SHA-256 dedup at upload time. All 10 are
    inside the 30-day dedup window, so copying leaves a live gap.

  * One PO row (uyed9xso3hlfemv) was produced from two DIFFERENT files
    with different hashes, via the dedup-by-PO-number branch. Copying
    gives last-write-wins on an arbitrary one of the two.

Hashing the stored bytes has neither problem, and it does not depend on
the collection we are about to delete.

Idempotent: records that already carry a file_hash are skipped. Records
with no stored file are reported and skipped - never guessed.
"""
import hashlib
import json
import os
import sys
import urllib.error
import urllib.parse
import urllib.request

APPLY = "--apply" in sys.argv

# collection -> the file field whose bytes are the record's identity
TARGETS = {
    "delivery_notes": "slip_image",
    "purchase_orders": "purchase_order_file",
}


def admin_auth(base_url):
    body = json.dumps(
        {
            "identity": os.environ["PB_ADMIN_EMAIL"],
            "password": os.environ["PB_ADMIN_PASSWORD"],
        }
    ).encode()
    req = urllib.request.Request(
        f"{base_url}/api/collections/_superusers/auth-with-password",
        data=body,
        headers={"Content-Type": "application/json"},
        method="POST",
    )
    with urllib.request.urlopen(req) as resp:
        return json.loads(resp.read())["token"]


def api(base_url, token, method, path, body=None):
    """
    A 404 is only soft for GET — "this collection/record is absent" is a
    legitimate answer to a read. On a write it is not: a PATCH against a
    since-deleted record returning None would be counted as a successful
    write. Mutations raise.
    """
    data = json.dumps(body).encode() if body is not None else None
    req = urllib.request.Request(
        f"{base_url}{path}",
        data=data,
        method=method,
        headers={"Authorization": token, "Content-Type": "application/json"},
    )
    try:
        with urllib.request.urlopen(req) as resp:
            txt = resp.read().decode()
            return json.loads(txt) if txt else None
    except urllib.error.HTTPError as e:
        if e.code == 404 and method == "GET":
            return None
        raise RuntimeError(
            f"{method} {path}: {e.code} {e.read().decode() if e.fp else ''}"
        ) from e


def get_all(base_url, token, coll, fields):
    out, page = [], 1
    while True:
        data = api(
            base_url,
            token,
            "GET",
            f"/api/collections/{coll}/records"
            f"?perPage=500&page={page}&fields={fields}",
        )
        out.extend(data["items"])
        if page >= data["totalPages"]:
            return out
        page += 1


def sha256_of_stored_file(base_url, token, coll, rec_id, filename):
    # safe="" so a "/" in a stored filename escapes rather than forging a
    # path segment. Both file fields are maxSelect:1 today, so the value is
    # a bare filename — this is belt and braces.
    #
    # This unauthenticated-file assumption is load-bearing: both fields are
    # `protected: false`, so the Authorization header below is ignored. If
    # either is ever flipped to protected, PB requires a short-lived token
    # from /api/files/token and this 403s.
    url = (
        f"{base_url}/api/files/{coll}/{rec_id}/"
        f"{urllib.parse.quote(filename, safe='')}"
    )
    req = urllib.request.Request(url, headers={"Authorization": token})
    with urllib.request.urlopen(req) as resp:
        return hashlib.sha256(resp.read()).hexdigest()


def main():
    base_url = os.environ.get("PB_URL", "http://127.0.0.1:8090").rstrip("/")
    token = admin_auth(base_url)
    # Say where we are writing. This script mutates production by default
    # target, and a run whose output names no host is a run you cannot
    # audit afterwards.
    print(f"target: {base_url}   mode: {'APPLY' if APPLY else 'DRY RUN'}")

    grand_write = grand_skip = grand_have = grand_err = 0

    for coll, file_field in TARGETS.items():
        rows = get_all(base_url, token, coll, f"id,file_hash,{file_field}")
        writes, no_file, errors, already = [], [], [], 0

        for r in rows:
            if (r.get("file_hash") or "").strip():
                already += 1
                continue
            fname = r.get(file_field) or ""
            if not fname:
                no_file.append(r["id"])
                continue
            try:
                writes.append(
                    (r["id"], sha256_of_stored_file(
                        base_url, token, coll, r["id"], fname))
                )
            except Exception as e:  # noqa: BLE001 - report, never guess
                # Kept apart from `no_file` deliberately. "This record has
                # no file" is a fact about the data; "the download blew up"
                # is a fact about this run. Folding a transient 500 into
                # the former makes a broken run read as a clean one.
                errors.append(f"{r['id']} ({e})")

        print(f"\n=== {coll} ===")
        print(f"  total           : {len(rows)}")
        print(f"  already hashed  : {already}")
        print(f"  to write        : {len(writes)}")
        print(f"  no stored file  : {len(no_file)}")
        print(f"  download errors : {len(errors)}")
        for n in no_file[:20]:
            print(f"     SKIP (no file) {n}")
        for n in errors[:20]:
            print(f"     ERROR {n}")

        grand_write += len(writes)
        grand_skip += len(no_file)
        grand_err += len(errors)
        grand_have += already

        if APPLY:
            for i, (rec_id, digest) in enumerate(writes, 1):
                api(
                    base_url, token, "PATCH",
                    f"/api/collections/{coll}/records/{rec_id}",
                    {"file_hash": digest},
                )
                if i % 10 == 0:
                    print(f"     wrote {i}/{len(writes)}")
            print(f"  wrote           : {len(writes)}/{len(writes)}")

    print(
        f"\ntotals: to_write={grand_write} already={grand_have} "
        f"no_file={grand_skip} errors={grand_err}"
    )

    if grand_err:
        print(
            f"\nFAILED: {grand_err} file(s) could not be downloaded. "
            f"Nothing about this run is trustworthy — re-run it."
        )
        return 1

    if not APPLY:
        print("\nDRY RUN. Re-run with --apply to write.")
        return 0

    # Verify: nothing that has a file should be left without a hash.
    left = 0
    for coll, file_field in TARGETS.items():
        for r in get_all(base_url, token, coll, f"id,file_hash,{file_field}"):
            if (r.get(file_field) or "") and not (r.get("file_hash") or ""):
                left += 1
    print(f"records with a file but no hash after run: {left}")
    return 0 if left == 0 else 1


if __name__ == "__main__":
    try:
        sys.exit(main())
    except KeyError as e:
        print(f"ERROR: missing environment variable {e}", file=sys.stderr)
        sys.exit(2)