diff --git a/invoices/_from_user/2568.pdf b/invoices/_from_user/2568.pdf new file mode 100644 index 0000000..9b71ce2 Binary files /dev/null and b/invoices/_from_user/2568.pdf differ diff --git a/invoices/_from_user/doc100189.pdf b/invoices/_from_user/doc100189.pdf new file mode 100644 index 0000000..a841685 Binary files /dev/null and b/invoices/_from_user/doc100189.pdf differ diff --git a/invoices/_non_jei/new155655 Supplier Invoice.pdf b/invoices/_non_jei/new155655 Supplier Invoice.pdf new file mode 100644 index 0000000..850bfdd Binary files /dev/null and b/invoices/_non_jei/new155655 Supplier Invoice.pdf differ diff --git a/invoices/_non_jei/newJEI 155655.pdf b/invoices/_non_jei/newJEI 155655.pdf new file mode 100644 index 0000000..57e4dc5 Binary files /dev/null and b/invoices/_non_jei/newJEI 155655.pdf differ diff --git a/invoices/_non_jei/newJEI SA Invoice 161611.pdf b/invoices/_non_jei/newJEI SA Invoice 161611.pdf new file mode 100644 index 0000000..c6e700c Binary files /dev/null and b/invoices/_non_jei/newJEI SA Invoice 161611.pdf differ diff --git a/invoices/_non_jei/newJEI SA PROFORMA 151778 (1).pdf b/invoices/_non_jei/newJEI SA PROFORMA 151778 (1).pdf new file mode 100644 index 0000000..2cf123e Binary files /dev/null and b/invoices/_non_jei/newJEI SA PROFORMA 151778 (1).pdf differ diff --git a/invoices/_non_jei/newJEI SA PROFORMA 151778 (2).pdf b/invoices/_non_jei/newJEI SA PROFORMA 151778 (2).pdf new file mode 100644 index 0000000..2cf123e Binary files /dev/null and b/invoices/_non_jei/newJEI SA PROFORMA 151778 (2).pdf differ diff --git a/invoices/_non_jei/newJEI SA PROFORMA 151779 (1).pdf b/invoices/_non_jei/newJEI SA PROFORMA 151779 (1).pdf new file mode 100644 index 0000000..a568055 Binary files /dev/null and b/invoices/_non_jei/newJEI SA PROFORMA 151779 (1).pdf differ diff --git a/invoices/newJEI Drilling Invoices (1).pdf b/invoices/newJEI Drilling Invoices (1).pdf new file mode 100644 index 0000000..6e8b491 Binary files /dev/null and b/invoices/newJEI Drilling Invoices (1).pdf differ diff --git a/invoices/newdoc10018920260413143327_001.pdf b/invoices/newdoc10018920260413143327_001.pdf new file mode 100644 index 0000000..a841685 Binary files /dev/null and b/invoices/newdoc10018920260413143327_001.pdf differ diff --git a/scripts/cleanup_lines.py b/scripts/cleanup_lines.py new file mode 100644 index 0000000..9a3b474 --- /dev/null +++ b/scripts/cleanup_lines.py @@ -0,0 +1,73 @@ +""" +Clean up duplicates in PB: +1. Delete JEI_invoice_lines duplicates (keep one per invoice+line_no+stock_code) +2. Delete the bogus JEISA000 stock code and its lines +""" +import os, sys, json, urllib.request, urllib.parse +from collections import defaultdict + +PB_URL = "https://pb.agent.apphub.co.za" +PB_TOKEN = None +env = os.path.join(os.path.dirname(__file__), "..", ".env.local") +if os.path.exists(env): + for line in open(env): + if line.startswith("PB_TOKEN="): + PB_TOKEN = line.strip().split("=", 1)[1] +if not PB_TOKEN: + sys.exit("PB_TOKEN missing") +H = {"Authorization": PB_TOKEN} + + +def get(path): + req = urllib.request.Request(PB_URL + path, headers=H, method="GET") + with urllib.request.urlopen(req, timeout=60) as r: + return json.loads(r.read()) + + +def delete(path): + req = urllib.request.Request(PB_URL + path, headers=H, method="DELETE") + with urllib.request.urlopen(req, timeout=30) as r: + return r.read().decode("utf-8") + + +# 1. Find bogus JEISA000 stock code +sc = get("/api/collections/JEI_stock_codes/records?perPage=500") +jeisa000 = next((c for c in sc["items"] if c["code"] == "JEISA000"), None) +if jeisa000: + print(f"Found bogus JEISA000 id={jeisa000['id']}") + jid = jeisa000["id"] + jfilter = 'stock_code="' + jid + '"' + lines = get("/api/collections/JEI_invoice_lines/records?filter=" + urllib.parse.quote(jfilter) + "&perPage=500") + print(f" Deleting {len(lines['items'])} lines referencing JEISA000...") + for l in lines["items"]: + delete(f"/api/collections/JEI_invoice_lines/records/{l['id']}") + delete(f"/api/collections/JEI_stock_codes/records/{jid}") + print(f" Deleted JEISA000 stock code") +else: + print("JEISA000 not in PB (already cleaned)") + +# 2. Deduplicate invoice lines +print("\nDeduplicating invoice lines...") +lines = get("/api/collections/JEI_invoice_lines/records?perPage=5000") +print(f" Total lines: {len(lines['items'])}") + +# Group by (invoice, line_no, stock_code) +groups = defaultdict(list) +for l in lines["items"]: + key = (l.get("invoice", ""), l.get("line_no", 0), l.get("stock_code", "")) + groups[key].append(l) + +dups_removed = 0 +for key, items in groups.items(): + if len(items) > 1: + # Keep the first, delete the rest + for dup in items[1:]: + delete(f"/api/collections/JEI_invoice_lines/records/{dup['id']}") + dups_removed += 1 + +print(f" Removed {dups_removed} duplicate lines") +print(f" Remaining unique lines: {len(lines['items']) - dups_removed}") + +# Final count +final = get("/api/collections/JEI_invoice_lines/records?perPage=5000") +print(f" Final count: {final['totalItems']}") diff --git a/scripts/extract_invoices.py b/scripts/extract_invoices.py index ec60fc3..d42f17d 100644 --- a/scripts/extract_invoices.py +++ b/scripts/extract_invoices.py @@ -286,12 +286,6 @@ def main(): unparseable_files.append(f"{pdf_path.name}: {e}") print(f"\n--- Parsed {len(records)} invoice records ---") - # Debug: show types of first few lines from first record - if records: - first = records[0] - print(f"DEBUG first record invoice_no={first.get('invoice_no')} lines_count={len(first.get('lines', []))}") - for i, l in enumerate(first.get("lines", [])[:3]): - print(f" line[{i}] type={type(l).__name__} value={repr(l)[:120]}") errors = [] for r in records: if r["total_net"] is None: diff --git a/scripts/import_to_pb.py b/scripts/import_to_pb.py index e9a63de..42e701a 100644 --- a/scripts/import_to_pb.py +++ b/scripts/import_to_pb.py @@ -215,7 +215,21 @@ def create_invoice(rec: dict, dry_run: bool): return None, "fail" +def find_line(invoice_id: str, line_no: int, stock_code_id: str): + q = f'invoice="{invoice_id}" && line_no="{line_no}" && stock_code="{stock_code_id}"' + r = pb_req("GET", f"/api/collections/JEI_invoice_lines/records?filter={urllib.parse.quote(q)}&perPage=1") + if isinstance(r, dict): + items = r.get("items", []) + return items[0] if items else None + return None + + def create_line(line: dict, invoice_id: str, stock_code_id: str, line_no: int, dry_run: bool): + # Idempotency: skip if a line already exists for this (invoice, line_no, stock_code) + existing = find_line(invoice_id, line_no, stock_code_id) + if existing: + return existing["id"], "exists" + body = { "invoice": invoice_id, "line_no": line_no,