fix: cleanup 6x duplicate lines + bogus JEISA000 stock code

- Removed 424 duplicate invoice lines from re-running the import
- Deleted JEISA000 stock code (it was actually your Account Ref)
- Made create_line idempotent in import_to_pb.py so re-runs skip existing
- Removed debug prints from extract_invoices.py

scripts/cleanup_lines.py

@@ -0,0 +1,73 @@
+"""
+Clean up duplicates in PB:
+1. Delete JEI_invoice_lines duplicates (keep one per invoice+line_no+stock_code)
+2. Delete the bogus JEISA000 stock code and its lines
+"""
+import os, sys, json, urllib.request, urllib.parse
+from collections import defaultdict
+
+PB_URL = "https://pb.agent.apphub.co.za"
+PB_TOKEN = None
+env = os.path.join(os.path.dirname(__file__), "..", ".env.local")
+if os.path.exists(env):
+    for line in open(env):
+        if line.startswith("PB_TOKEN="):
+            PB_TOKEN = line.strip().split("=", 1)[1]
+if not PB_TOKEN:
+    sys.exit("PB_TOKEN missing")
+H = {"Authorization": PB_TOKEN}
+
+
+def get(path):
+    req = urllib.request.Request(PB_URL + path, headers=H, method="GET")
+    with urllib.request.urlopen(req, timeout=60) as r:
+        return json.loads(r.read())
+
+
+def delete(path):
+    req = urllib.request.Request(PB_URL + path, headers=H, method="DELETE")
+    with urllib.request.urlopen(req, timeout=30) as r:
+        return r.read().decode("utf-8")
+
+
+# 1. Find bogus JEISA000 stock code
+sc = get("/api/collections/JEI_stock_codes/records?perPage=500")
+jeisa000 = next((c for c in sc["items"] if c["code"] == "JEISA000"), None)
+if jeisa000:
+    print(f"Found bogus JEISA000 id={jeisa000['id']}")
+    jid = jeisa000["id"]
+    jfilter = 'stock_code="' + jid + '"'
+    lines = get("/api/collections/JEI_invoice_lines/records?filter=" + urllib.parse.quote(jfilter) + "&perPage=500")
+    print(f"  Deleting {len(lines['items'])} lines referencing JEISA000...")
+    for l in lines["items"]:
+        delete(f"/api/collections/JEI_invoice_lines/records/{l['id']}")
+    delete(f"/api/collections/JEI_stock_codes/records/{jid}")
+    print(f"  Deleted JEISA000 stock code")
+else:
+    print("JEISA000 not in PB (already cleaned)")
+
+# 2. Deduplicate invoice lines
+print("\nDeduplicating invoice lines...")
+lines = get("/api/collections/JEI_invoice_lines/records?perPage=5000")
+print(f"  Total lines: {len(lines['items'])}")
+
+# Group by (invoice, line_no, stock_code)
+groups = defaultdict(list)
+for l in lines["items"]:
+    key = (l.get("invoice", ""), l.get("line_no", 0), l.get("stock_code", ""))
+    groups[key].append(l)
+
+dups_removed = 0
+for key, items in groups.items():
+    if len(items) > 1:
+        # Keep the first, delete the rest
+        for dup in items[1:]:
+            delete(f"/api/collections/JEI_invoice_lines/records/{dup['id']}")
+            dups_removed += 1
+
+print(f"  Removed {dups_removed} duplicate lines")
+print(f"  Remaining unique lines: {len(lines['items']) - dups_removed}")
+
+# Final count
+final = get("/api/collections/JEI_invoice_lines/records?perPage=5000")
+print(f"  Final count: {final['totalItems']}")

scripts/extract_invoices.py

@@ -286,12 +286,6 @@ def main():
             unparseable_files.append(f"{pdf_path.name}: {e}")
 
     print(f"\n--- Parsed {len(records)} invoice records ---")
-    # Debug: show types of first few lines from first record
-    if records:
-        first = records[0]
-        print(f"DEBUG first record invoice_no={first.get('invoice_no')} lines_count={len(first.get('lines', []))}")
-        for i, l in enumerate(first.get("lines", [])[:3]):
-            print(f"  line[{i}] type={type(l).__name__} value={repr(l)[:120]}")
     errors = []
     for r in records:
         if r["total_net"] is None:

scripts/import_to_pb.py

@@ -215,7 +215,21 @@ def create_invoice(rec: dict, dry_run: bool):
     return None, "fail"
 
 
+def find_line(invoice_id: str, line_no: int, stock_code_id: str):
+    q = f'invoice="{invoice_id}" && line_no="{line_no}" && stock_code="{stock_code_id}"'
+    r = pb_req("GET", f"/api/collections/JEI_invoice_lines/records?filter={urllib.parse.quote(q)}&perPage=1")
+    if isinstance(r, dict):
+        items = r.get("items", [])
+        return items[0] if items else None
+    return None
+
+
 def create_line(line: dict, invoice_id: str, stock_code_id: str, line_no: int, dry_run: bool):
+    # Idempotency: skip if a line already exists for this (invoice, line_no, stock_code)
+    existing = find_line(invoice_id, line_no, stock_code_id)
+    if existing:
+        return existing["id"], "exists"
+
     body = {
         "invoice": invoice_id,
         "line_no": line_no,