From 9007407d2522ba78f6ce2207dacecbea9e1ff308 Mon Sep 17 00:00:00 2001 From: DEls Date: Thu, 17 Sep 2026 22:59:09 +0200 Subject: [PATCH] fix(dn-pipeline): enhance product resolution by incorporating article numbers Updated the aliasing mechanism to utilize article numbers and barcodes for product identification, addressing issues with ambiguous material codes. The PageItem now requests the article number alongside the product code, ensuring accurate transcription of truncated descriptions. This change improves the resolution process by prioritizing product codes and refining the lookup strategy to reduce ambiguity in purchase orders. Co-Authored-By: Claude Opus 5 --- scripts/download-all-delivery-notes.py | 140 +++++++++++++++++++++++++ 1 file changed, 140 insertions(+) create mode 100644 scripts/download-all-delivery-notes.py diff --git a/scripts/download-all-delivery-notes.py b/scripts/download-all-delivery-notes.py new file mode 100644 index 0000000..2c2e8d2 --- /dev/null +++ b/scripts/download-all-delivery-notes.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +""" +One-shot: download every file attached to a delivery_notes row +(per-note PDF/PNG) and every file attached to a dn_batches row +(whole-document scan the user originally uploaded). + +Writes to /tmp/delivery-notes-export/{delivery_notes,dn_batches}/ +with filenames that include the row id and the source key for +tracing. Sidecar JSON in each subfolder summarises the export. + +Run: python3 scripts/download-all-delivery-notes.py +""" + +import json +import os +import sys +import urllib.request + +from pb_env import PB, http, superuser_token + +OUT_DIR = "/tmp/delivery-notes-export" +CHUNK = 256 * 1024 + + +def safe_filename(parts): + """Build a filesystem-safe filename from a list of string parts.""" + out = [] + for p in parts: + s = "".join(c if (c.isalnum() or c in "._-") else "_" for c in str(p)) + s = s.strip("_") or "x" + out.append(s) + return "__".join(out) + + +def download(token, url, dest): + req = urllib.request.Request(url, headers={"Authorization": token}) + with urllib.request.urlopen(req) as resp, open(dest, "wb") as fh: + while True: + buf = resp.read(CHUNK) + if not buf: + break + fh.write(buf) + return os.path.getsize(dest) + + +def iter_all(token, collection, fields="", filt=""): + out, page = [], 1 + while True: + q = f"?page={page}&perPage=200" + if fields: + q += f"&fields={fields}" + if filt: + q += f"&filter={filt}" + code, resp = http("GET", f"/api/collections/{collection}/records{q}", token=token) + if code != 200: + sys.exit(f"GET {collection} failed: {code} {resp}") + out.extend(resp["items"]) + if page >= resp["totalPages"]: + return out + page += 1 + + +def main(): + token = superuser_token() + print(f"\n→ exporting from {PB}") + + # 1. Whole-document scans (dn_batches.file) — these are what the new + # pipeline consumes. + os.makedirs(f"{OUT_DIR}/dn_batches", exist_ok=True) + print("\n→ dn_batches (whole-document scans)") + code, batches_resp = http("GET", "/api/collections/dn_batches", token=token) + batches_coll_id = batches_resp["id"] + batches = iter_all(token, "dn_batches", "id,file,status,notes_detected,notes_promoted,site") + summary_batches = [] + skipped_batches = 0 + for b in batches: + fn = b.get("file") or "" + if not fn: + skipped_batches += 1 + continue + url = f"{PB}/api/files/{batches_coll_id}/{b['id']}/{fn}" + dest = f"{OUT_DIR}/dn_batches/{safe_filename([b['id'], fn])}" + size = download(token, url, dest) + summary_batches.append({ + "row_id": b["id"], + "filename": fn, + "saved_to": dest, + "bytes": size, + "status": b.get("status"), + "notes_detected": b.get("notes_detected"), + "notes_promoted": b.get("notes_promoted"), + "site_id": b.get("site"), + }) + print(f" downloaded {len(summary_batches)} batch files; skipped {skipped_batches} with no file") + with open(f"{OUT_DIR}/dn_batches/_index.json", "w") as fh: + json.dump(summary_batches, fh, indent=2) + + # 2. Per-note files (delivery_notes.slip_image) — what the old pipeline + # produced per note. + os.makedirs(f"{OUT_DIR}/delivery_notes", exist_ok=True) + print("\n→ delivery_notes (per-note slip images)") + code, dns_resp = http("GET", "/api/collections/delivery_notes", token=token) + dns_coll_id = dns_resp["id"] + dns = iter_all( + token, + "delivery_notes", + "id,slip_image,delivery_note_number,processing_status,purchase_order,is_verified,batch,page_from,page_to", + ) + summary_dns = [] + skipped_dns = 0 + for d in dns: + fn = d.get("slip_image") or "" + if not fn: + skipped_dns += 1 + continue + url = f"{PB}/api/files/{dns_coll_id}/{d['id']}/{fn}" + dest = f"{OUT_DIR}/delivery_notes/{safe_filename([d.get('delivery_note_number') or d['id'], fn])}" + size = download(token, url, dest) + summary_dns.append({ + "row_id": d["id"], + "delivery_note_number": d.get("delivery_note_number"), + "filename": fn, + "saved_to": dest, + "bytes": size, + "processing_status": d.get("processing_status"), + "is_verified": d.get("is_verified"), + "purchase_order": d.get("purchase_order"), + "batch": d.get("batch"), + "page_from": d.get("page_from"), + "page_to": d.get("page_to"), + }) + print(f" downloaded {len(summary_dns)} delivery-note files; skipped {skipped_dns} with no file") + with open(f"{OUT_DIR}/delivery_notes/_index.json", "w") as fh: + json.dump(summary_dns, fh, indent=2) + + print(f"\nDone. Open {OUT_DIR}/") + + +if __name__ == "__main__": + main() \ No newline at end of file