JEI_Import_Documents-App/scripts/import_to_pb.py

"""
Import parsed invoice data into PocketBase.

Reads data/review/all.json, upserts JEI_stock_codes and JEI_invoices,
uploads PDFs to JEI_invoices.pdf_file, and creates JEI_invoice_lines.

Idempotent: skips invoices whose invoice_no + document_type already exist.

Usage:
    python scripts/import_to_pb.py [--dry-run]
"""
import os
import re
import sys
import json
import argparse
import urllib.request
import urllib.parse
from pathlib import Path
from urllib.error import HTTPError
from datetime import datetime

PB_URL = os.environ.get("PB_URL", "https://pb.agent.apphub.co.za")
PB_TOKEN = os.environ.get("PB_TOKEN", "")
if not PB_TOKEN:
    env_path = Path(__file__).resolve().parent.parent / ".env.local"
    if env_path.exists():
        for line in open(env_path):
            if line.startswith("PB_TOKEN="):
                PB_TOKEN = line.strip().split("=", 1)[1]
if not PB_TOKEN:
    sys.exit("ERROR: PB_TOKEN not set")

ROOT = Path(__file__).resolve().parent.parent
ALL_JSON = ROOT / "data" / "review" / "all.json"
INVOICES_DIR = ROOT / "invoices"


def pb_req(method: str, path: str, body=None, multipart=None, retries=3):
    """Make an API request with retry on transient errors."""
    url = PB_URL + path
    headers = {"Authorization": PB_TOKEN}
    if multipart is None and body is not None:
        headers["Content-Type"] = "application/json"
        data = json.dumps(body).encode("utf-8")
    elif multipart is not None:
        boundary = "----PBIMPORT" + datetime.now().strftime("%H%M%S%f")
        headers["Content-Type"] = f"multipart/form-data; boundary={boundary}"
        data = b""
        for k, v in (multipart or {}).items():
            if isinstance(v, tuple):
                fname, content, mime = v
                data += f"--{boundary}\r\n".encode()
                data += f'Content-Disposition: form-data; name="{k}"; filename="{fname}"\r\n'.encode()
                data += f"Content-Type: {mime}\r\n\r\n".encode()
                data += content
                data += b"\r\n"
            else:
                data += f"--{boundary}\r\n".encode()
                data += f'Content-Disposition: form-data; name="{k}"\r\n\r\n'.encode()
                data += str(v).encode("utf-8")
                data += b"\r\n"
        data += f"--{boundary}--\r\n".encode()
    else:
        data = None
    last_err = None
    for attempt in range(retries):
        req = urllib.request.Request(url, headers=headers, method=method, data=data)
        try:
            with urllib.request.urlopen(req, timeout=30) as r:
                txt = r.read().decode("utf-8")
                return json.loads(txt) if txt else None
        except HTTPError as e:
            body_txt = e.read().decode("utf-8")
            try:
                return None, json.loads(body_txt)
            except Exception:
                return None, body_txt
        except Exception as e:
            last_err = e
            if attempt < retries - 1:
                import time
                time.sleep(2 ** attempt)
                continue
            raise last_err


def find_collection(name):
    r = pb_req("GET", "/api/collections?perPage=200")
    items = (r or {}).get("items", []) if isinstance(r, tuple) is False else []
    if isinstance(r, dict):
        items = r.get("items", [])
    for c in items:
        if c.get("name") == name:
            return c
    return None


def find_invoice(invoice_no: str, document_type: str):
    q = f'invoice_no="{invoice_no}" && document_type="{document_type}"'
    r = pb_req("GET", f"/api/collections/JEI_invoices/records?filter={urllib.parse.quote(q)}&perPage=1")
    if isinstance(r, dict):
        items = r.get("items", [])
        return items[0] if items else None
    return None


def find_stock_code(code: str):
    q = f'code="{code}"'
    r = pb_req("GET", f"/api/collections/JEI_stock_codes/records?filter={urllib.parse.quote(q)}&perPage=1")
    if isinstance(r, dict):
        items = r.get("items", [])
        return items[0] if items else None
    return None


def upsert_stock_code(rec: dict, dry_run: bool, sc_meta: dict | None = None) -> tuple[str, str]:
    """Returns (stock_code_id, action) where action is 'created', 'updated', or 'exists'.

    sc_meta (optional) provides {first_seen_date, last_seen_date, description_goods}
    derived from aggregating across all invoices.
    """
    code = rec["stock_code"]
    # sc_meta from extract_invoices.py is a list [first_seen, last_seen, description]
    if isinstance(sc_meta, list) and len(sc_meta) >= 3:
        first_seen = sc_meta[0]
        last_seen = sc_meta[1]
        description = sc_meta[2] or code
    else:
        first_seen = None
        last_seen = None
        description = rec.get("description") or code

    existing = find_stock_code(code)
    if existing:
        # PATCH if any field is better than what's stored
        needs_update = False
        patch: dict = {}
        old_desc = (existing.get("description_goods") or "").strip()
        new_desc = (description or "").strip()
        # Always update if: new is longer, OR old is just the code, OR old has artifact (trailing quote, etc.)
        old_is_default = (old_desc == existing.get("code", "")) or not old_desc
        old_has_artifact = old_desc.endswith('"') or old_desc.endswith('"') or "  " in old_desc or old_desc.endswith(",")
        new_is_better = (
            (new_desc and old_is_default) or
            (new_desc and len(new_desc) > len(old_desc) and not old_desc.startswith(new_desc[:20])) or
            (new_desc and old_has_artifact and len(new_desc) >= len(old_desc) - 2)
        )
        if new_is_better and new_desc != old_desc:
            patch["description_goods"] = new_desc
            needs_update = True
        if first_seen and not existing.get("first_seen_date"):
            patch["first_seen_date"] = first_seen
            needs_update = True
        if last_seen and not existing.get("last_seen_date"):
            patch["last_seen_date"] = last_seen
            needs_update = True
        if needs_update and not dry_run:
            r = pb_req("PATCH", f"/api/collections/JEI_stock_codes/records/{existing['id']}", patch)
            if isinstance(r, dict) and r.get("id"):
                return r["id"], "updated"
        return existing["id"], "exists"

    body = {
        "code": code,
        "description_goods": description,
        "first_seen_date": first_seen or None,
        "last_seen_date": last_seen or None,
    }
    if dry_run:
        return "DRY-" + code, "would-create"
    r = pb_req("POST", "/api/collections/JEI_stock_codes/records", body)
    if isinstance(r, dict) and r.get("id"):
        return r["id"], "created"
    print(f"   STOCK CODE FAIL: {r}")
    return None, "fail"


def create_invoice(rec: dict, dry_run: bool):
    existing = find_invoice(rec["invoice_no"], rec["document_type"])
    if existing:
        return existing["id"], "exists"

    body = {
        "invoice_no": rec["invoice_no"],
        "invoice_date": rec["invoice_date"],
        "document_type": rec["document_type"],
        "customer_order_no": rec.get("customer_order_no", ""),
        "buyer_name_variant": rec.get("buyer_name_variant") or "JEI South Africa",
        "supplier_name": rec.get("supplier_name", "JEI DRILLING & CUTTING SOLUTIONS LTD"),
        "account_ref": rec.get("account_ref", ""),
        "total_net": rec["total_net"],
        "carriage_net": rec.get("carriage_net") or 0,
        "invoice_total": rec["invoice_total"],
        "deliver_to_name": rec.get("deliver_to_name", ""),
        "deliver_to_address": rec.get("deliver_to_address", ""),
        "source_filename": rec.get("source_filename", ""),
    }

    # If this is the first page of its source file, attach the PDF
    if rec.get("is_source_first_page"):
        pdf_path = INVOICES_DIR / rec["source_filename"]
        if pdf_path.exists():
            pdf_bytes = pdf_path.read_bytes()
            r = pb_req("POST", "/api/collections/JEI_invoices/records",
                       multipart={**body, "pdf_file": (pdf_path.name, pdf_bytes, "application/pdf")})
        else:
            r = pb_req("POST", "/api/collections/JEI_invoices/records", body)
    else:
        r = pb_req("POST", "/api/collections/JEI_invoices/records", body)

    if isinstance(r, dict) and r.get("id"):
        return r["id"], "created"
    print(f"   INVOICE FAIL: {r}")
    return None, "fail"


def find_line(invoice_id: str, line_no: int, stock_code_id: str):
    q = f'invoice="{invoice_id}" && line_no="{line_no}" && stock_code="{stock_code_id}"'
    r = pb_req("GET", f"/api/collections/JEI_invoice_lines/records?filter={urllib.parse.quote(q)}&perPage=1")
    if isinstance(r, dict):
        items = r.get("items", [])
        return items[0] if items else None
    return None


def create_line(line: dict, invoice_id: str, stock_code_id: str, line_no: int, dry_run: bool):
    # Idempotency: skip if a line already exists for this (invoice, line_no, stock_code)
    existing = find_line(invoice_id, line_no, stock_code_id)
    if existing:
        return existing["id"], "exists"

    body = {
        "invoice": invoice_id,
        "line_no": line_no,
        "stock_code": stock_code_id,
        "quantity": line["quantity"],
        "unit_price": line["unit_price"],
        "net_amount": line["net_amount"],
        "commodity_code": line.get("commodity_code", ""),
        "country_of_origin": line.get("country_of_origin", ""),
    }
    if dry_run:
        return "DRY-LINE", "would-create"
    r = pb_req("POST", "/api/collections/JEI_invoice_lines/records", body)
    if isinstance(r, dict) and r.get("id"):
        return r["id"], "created"
    return None, "fail"


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--dry-run", action="store_true")
    args = ap.parse_args()

    if not ALL_JSON.exists():
        sys.exit(f"Run extract_invoices.py first. Missing: {ALL_JSON}")

    with open(ALL_JSON) as f:
        data = json.load(f)
    if isinstance(data, dict) and "records" in data:
        records = data["records"]
        sc_meta_map = data.get("stock_code_dates", {})
    else:
        records = data
        sc_meta_map = {}

    print(f"Importing {len(records)} records (dry_run={args.dry_run})")

    counts = {"invoices_created": 0, "invoices_exists": 0, "invoices_fail": 0,
              "stock_codes_created": 0, "stock_codes_updated": 0, "stock_codes_exists": 0,
              "lines_created": 0, "lines_fail": 0}

    for rec in records:
        if rec.get("total_net") is None or rec.get("invoice_total") is None:
            print(f"SKIP {rec['invoice_no']}: missing total")
            continue
        if not rec.get("lines"):
            print(f"SKIP {rec['invoice_no']}: no lines parsed")
            continue

        inv_id, inv_action = create_invoice(rec, args.dry_run)
        if inv_action == "created":
            counts["invoices_created"] += 1
        elif inv_action == "exists":
            counts["invoices_exists"] += 1
        else:
            counts["invoices_fail"] += 1
            continue

        for idx, line in enumerate(rec["lines"], start=1):
            sc_meta = sc_meta_map.get(line["stock_code"])
            sc_id, sc_action = upsert_stock_code(line, args.dry_run, sc_meta)
            if sc_action == "created":
                counts["stock_codes_created"] += 1
            elif sc_action == "updated":
                counts["stock_codes_updated"] += 1
            elif sc_action == "exists":
                counts["stock_codes_exists"] += 1
            if sc_id is None:
                counts["lines_fail"] += 1
                continue
            _, line_action = create_line(line, inv_id, sc_id, idx, args.dry_run)
            if line_action == "created":
                counts["lines_created"] += 1
            else:
                counts["lines_fail"] += 1

        status = "OK" if inv_action == "created" else "EXISTS"
        print(f"{status} {rec['invoice_no']} {rec['document_type']} - {len(rec['lines'])} lines")

    print(f"\nDONE: {counts}")


if __name__ == "__main__":
    main()