MEGAINFRA-WEBAPP/scripts/download-all-delivery-notes.py

#!/usr/bin/env python3
"""
One-shot: download every file attached to a delivery_notes row
(per-note PDF/PNG) and every file attached to a dn_batches row
(whole-document scan the user originally uploaded).

Writes to /tmp/delivery-notes-export/{delivery_notes,dn_batches}/
with filenames that include the row id and the source key for
tracing. Sidecar JSON in each subfolder summarises the export.

Run: python3 scripts/download-all-delivery-notes.py
"""

import json
import os
import sys
import urllib.request

from pb_env import PB, http, superuser_token

OUT_DIR = "/tmp/delivery-notes-export"
CHUNK = 256 * 1024


def safe_filename(parts):
    """Build a filesystem-safe filename from a list of string parts."""
    out = []
    for p in parts:
        s = "".join(c if (c.isalnum() or c in "._-") else "_" for c in str(p))
        s = s.strip("_") or "x"
        out.append(s)
    return "__".join(out)


def download(token, url, dest):
    req = urllib.request.Request(url, headers={"Authorization": token})
    with urllib.request.urlopen(req) as resp, open(dest, "wb") as fh:
        while True:
            buf = resp.read(CHUNK)
            if not buf:
                break
            fh.write(buf)
    return os.path.getsize(dest)


def iter_all(token, collection, fields="", filt=""):
    out, page = [], 1
    while True:
        q = f"?page={page}&perPage=200"
        if fields:
            q += f"&fields={fields}"
        if filt:
            q += f"&filter={filt}"
        code, resp = http("GET", f"/api/collections/{collection}/records{q}", token=token)
        if code != 200:
            sys.exit(f"GET {collection} failed: {code} {resp}")
        out.extend(resp["items"])
        if page >= resp["totalPages"]:
            return out
        page += 1


def main():
    token = superuser_token()
    print(f"\n→ exporting from {PB}")

    # 1. Whole-document scans (dn_batches.file) — these are what the new
    # pipeline consumes.
    os.makedirs(f"{OUT_DIR}/dn_batches", exist_ok=True)
    print("\n→ dn_batches (whole-document scans)")
    code, batches_resp = http("GET", "/api/collections/dn_batches", token=token)
    batches_coll_id = batches_resp["id"]
    batches = iter_all(token, "dn_batches", "id,file,status,notes_detected,notes_promoted,site")
    summary_batches = []
    skipped_batches = 0
    for b in batches:
        fn = b.get("file") or ""
        if not fn:
            skipped_batches += 1
            continue
        url = f"{PB}/api/files/{batches_coll_id}/{b['id']}/{fn}"
        dest = f"{OUT_DIR}/dn_batches/{safe_filename([b['id'], fn])}"
        size = download(token, url, dest)
        summary_batches.append({
            "row_id": b["id"],
            "filename": fn,
            "saved_to": dest,
            "bytes": size,
            "status": b.get("status"),
            "notes_detected": b.get("notes_detected"),
            "notes_promoted": b.get("notes_promoted"),
            "site_id": b.get("site"),
        })
    print(f"  downloaded {len(summary_batches)} batch files; skipped {skipped_batches} with no file")
    with open(f"{OUT_DIR}/dn_batches/_index.json", "w") as fh:
        json.dump(summary_batches, fh, indent=2)

    # 2. Per-note files (delivery_notes.slip_image) — what the old pipeline
    # produced per note.
    os.makedirs(f"{OUT_DIR}/delivery_notes", exist_ok=True)
    print("\n→ delivery_notes (per-note slip images)")
    code, dns_resp = http("GET", "/api/collections/delivery_notes", token=token)
    dns_coll_id = dns_resp["id"]
    dns = iter_all(
        token,
        "delivery_notes",
        "id,slip_image,delivery_note_number,processing_status,purchase_order,is_verified,batch,page_from,page_to",
    )
    summary_dns = []
    skipped_dns = 0
    for d in dns:
        fn = d.get("slip_image") or ""
        if not fn:
            skipped_dns += 1
            continue
        url = f"{PB}/api/files/{dns_coll_id}/{d['id']}/{fn}"
        dest = f"{OUT_DIR}/delivery_notes/{safe_filename([d.get('delivery_note_number') or d['id'], fn])}"
        size = download(token, url, dest)
        summary_dns.append({
            "row_id": d["id"],
            "delivery_note_number": d.get("delivery_note_number"),
            "filename": fn,
            "saved_to": dest,
            "bytes": size,
            "processing_status": d.get("processing_status"),
            "is_verified": d.get("is_verified"),
            "purchase_order": d.get("purchase_order"),
            "batch": d.get("batch"),
            "page_from": d.get("page_from"),
            "page_to": d.get("page_to"),
        })
    print(f"  downloaded {len(summary_dns)} delivery-note files; skipped {skipped_dns} with no file")
    with open(f"{OUT_DIR}/delivery_notes/_index.json", "w") as fh:
        json.dump(summary_dns, fh, indent=2)

    print(f"\nDone. Open {OUT_DIR}/")


if __name__ == "__main__":
    main()