diff --git a/scripts/download-all-delivery-notes.py b/scripts/download-all-delivery-notes.py new file mode 100644 index 0000000..2c2e8d2 --- /dev/null +++ b/scripts/download-all-delivery-notes.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +""" +One-shot: download every file attached to a delivery_notes row +(per-note PDF/PNG) and every file attached to a dn_batches row +(whole-document scan the user originally uploaded). + +Writes to /tmp/delivery-notes-export/{delivery_notes,dn_batches}/ +with filenames that include the row id and the source key for +tracing. Sidecar JSON in each subfolder summarises the export. + +Run: python3 scripts/download-all-delivery-notes.py +""" + +import json +import os +import sys +import urllib.request + +from pb_env import PB, http, superuser_token + +OUT_DIR = "/tmp/delivery-notes-export" +CHUNK = 256 * 1024 + + +def safe_filename(parts): + """Build a filesystem-safe filename from a list of string parts.""" + out = [] + for p in parts: + s = "".join(c if (c.isalnum() or c in "._-") else "_" for c in str(p)) + s = s.strip("_") or "x" + out.append(s) + return "__".join(out) + + +def download(token, url, dest): + req = urllib.request.Request(url, headers={"Authorization": token}) + with urllib.request.urlopen(req) as resp, open(dest, "wb") as fh: + while True: + buf = resp.read(CHUNK) + if not buf: + break + fh.write(buf) + return os.path.getsize(dest) + + +def iter_all(token, collection, fields="", filt=""): + out, page = [], 1 + while True: + q = f"?page={page}&perPage=200" + if fields: + q += f"&fields={fields}" + if filt: + q += f"&filter={filt}" + code, resp = http("GET", f"/api/collections/{collection}/records{q}", token=token) + if code != 200: + sys.exit(f"GET {collection} failed: {code} {resp}") + out.extend(resp["items"]) + if page >= resp["totalPages"]: + return out + page += 1 + + +def main(): + token = superuser_token() + print(f"\n→ exporting from {PB}") + + # 1. Whole-document scans (dn_batches.file) — these are what the new + # pipeline consumes. + os.makedirs(f"{OUT_DIR}/dn_batches", exist_ok=True) + print("\n→ dn_batches (whole-document scans)") + code, batches_resp = http("GET", "/api/collections/dn_batches", token=token) + batches_coll_id = batches_resp["id"] + batches = iter_all(token, "dn_batches", "id,file,status,notes_detected,notes_promoted,site") + summary_batches = [] + skipped_batches = 0 + for b in batches: + fn = b.get("file") or "" + if not fn: + skipped_batches += 1 + continue + url = f"{PB}/api/files/{batches_coll_id}/{b['id']}/{fn}" + dest = f"{OUT_DIR}/dn_batches/{safe_filename([b['id'], fn])}" + size = download(token, url, dest) + summary_batches.append({ + "row_id": b["id"], + "filename": fn, + "saved_to": dest, + "bytes": size, + "status": b.get("status"), + "notes_detected": b.get("notes_detected"), + "notes_promoted": b.get("notes_promoted"), + "site_id": b.get("site"), + }) + print(f" downloaded {len(summary_batches)} batch files; skipped {skipped_batches} with no file") + with open(f"{OUT_DIR}/dn_batches/_index.json", "w") as fh: + json.dump(summary_batches, fh, indent=2) + + # 2. Per-note files (delivery_notes.slip_image) — what the old pipeline + # produced per note. + os.makedirs(f"{OUT_DIR}/delivery_notes", exist_ok=True) + print("\n→ delivery_notes (per-note slip images)") + code, dns_resp = http("GET", "/api/collections/delivery_notes", token=token) + dns_coll_id = dns_resp["id"] + dns = iter_all( + token, + "delivery_notes", + "id,slip_image,delivery_note_number,processing_status,purchase_order,is_verified,batch,page_from,page_to", + ) + summary_dns = [] + skipped_dns = 0 + for d in dns: + fn = d.get("slip_image") or "" + if not fn: + skipped_dns += 1 + continue + url = f"{PB}/api/files/{dns_coll_id}/{d['id']}/{fn}" + dest = f"{OUT_DIR}/delivery_notes/{safe_filename([d.get('delivery_note_number') or d['id'], fn])}" + size = download(token, url, dest) + summary_dns.append({ + "row_id": d["id"], + "delivery_note_number": d.get("delivery_note_number"), + "filename": fn, + "saved_to": dest, + "bytes": size, + "processing_status": d.get("processing_status"), + "is_verified": d.get("is_verified"), + "purchase_order": d.get("purchase_order"), + "batch": d.get("batch"), + "page_from": d.get("page_from"), + "page_to": d.get("page_to"), + }) + print(f" downloaded {len(summary_dns)} delivery-note files; skipped {skipped_dns} with no file") + with open(f"{OUT_DIR}/delivery_notes/_index.json", "w") as fh: + json.dump(summary_dns, fh, indent=2) + + print(f"\nDone. Open {OUT_DIR}/") + + +if __name__ == "__main__": + main() \ No newline at end of file