diff --git a/docs/n8n/po-email-intake.md b/docs/n8n/po-email-intake.md index 91780d0..cd1b056 100644 --- a/docs/n8n/po-email-intake.md +++ b/docs/n8n/po-email-intake.md @@ -188,6 +188,12 @@ the app, are recorded as `raw_ocr_data.site_check.result = "mismatch"` and the f OCR's header project reference is still never used. Nellmapius Ext 25's code was corrected from `001` to `C26/27` (the code on its POs) the same day. The history below is kept for context. +> **This check still runs on POs filed by the email intake, and it still uses the cost code.** +> A PO whose every line is billed to one other site's code will be *moved* to that site after +> OCR, whatever the intake booked it against. That is the app's own rule and it has not changed — +> the email intake only decides the site *before* the row exists. A PO whose lines name two +> projects, like MEGCPO9550, is recorded as a `mismatch` and the filed site stays. + Before that, MEGA's finalisation step deliberately **never re-mapped the site after OCR** — `src/app/api/purchase-orders/process-batch/route.ts` states it outright: *"`site` was set at upload by the person who uploaded the … and is NEVER touched here"*, because OCR only matches a @@ -206,25 +212,32 @@ itself, in this order: If none of those match, the email is tagged `PO Needs Review` and left in the inbox for a human. It is never filed against a guessed site. -### The script reads the site out of the PDF +### The script books against the name written on the PO -`scripts/po_email_intake.py` has the same goal but a better source of truth: the PO document -itself. It looks for the site in this order: +`scripts/po_email_intake.py` reads the site from the PO document itself, and it books against the +**site name**, in this order: -1. **the project code on the PO's own lines** — every line of a MEGA PO starts with the site's - project code (`C26/27, 0001`). All the lines must name the same site; lines naming two - projects are ambiguous and are left for a human, exactly as `process-batch` treats them -2. **the site named in the PO's `Description`** — MEGA writes it as `Tools (Nellmapius EXT25)` -3. **the site named in the PO's `Subject`** — `MEGCREQ11913 (Nellmapius EXT25)` -4. the email subject, as a last resort +1. **the site named in the PO's `Subject`** — `MEGCREQ11913 (Nellmapius EXT25)` +2. **the site named in the PO's `Description`** — `Tools (Nellmapius EXT25)` +3. **the site named in the email subject** — `RE: Nellmapius Ext 25 PO Tools/cement/Bricks` -Only the line-item block is scanned for project codes, so the `Head office` in a PO's contact -block cannot be mistaken for the `HEAD` site code. +A PO that names two sites, or none, is flagged for review rather than booked against one of them. + +**The project code on the PO's lines is deliberately not used.** It is a cost allocation, not a +statement of where the work is going. MEGCPO9550 is a **Cedar Gate** delivery — its subject says +`(Cedar Gate)` and its email subject says *"RE: Cedar Gate Paving & Kerbs PO"* — yet four of its +seven lines are billed to `C26/08` (Castle Gate) and three to `C26/13` (Cedar Gate). Booking by +code would have put a Cedar Gate PO against Castle Gate. The code is still read, and kept on the +filed row as `po_cost_codes`, because the app's own finalisation step uses it as evidence. + +A site may be written loosely — MEGA writes *"Munyaka"* where the app has *Munyaka 2*, *"West +Park EXT6"* where the app has *Wespark*, and MEGCPO9475's own subject misspells *Nellmpius*. The +qualifier and the two known spellings are handled in `name_variants()` and `SITE_ALIASES`. **This closes the gap below for the script.** On 2026-11-05 a PO arrived whose email subject was -just *"Chamberlain PO"* — Chamberlain is the supplier, not a site — while the PDF gave the site -three ways: `Description: Tools (Nellmapius EXT25)` and `C26/27` on every line. The script filed -it to **Nellmapius Ext 25**; before this change it was flagged for review. +just *"Chamberlain PO"* — Chamberlain is the supplier, not a site — while the PDF gave the site in +its `Description`: `Tools (Nellmapius EXT25)`. The script filed it to **Nellmapius Ext 25**; before +this change it was flagged for review. **Known limitation of the n8n workflow, seen in real mail:** the site often appears only *inside* the PDF, not in the email. A real example from the mailbox — the subject was *"RE: Bushveld Joy x 12 diff --git a/scripts/po_email_intake.py b/scripts/po_email_intake.py index ff13183..1c4d129 100755 --- a/scripts/po_email_intake.py +++ b/scripts/po_email_intake.py @@ -46,6 +46,16 @@ RUN_LOG = STATE_DIR / "runs.jsonl" SITES_FILE = STATE_DIR / "sites.json" PDF_BIN = "/opt/homebrew/bin/pdftotext" DOC_TYPES = {"application/pdf"} +# A trailing instance qualifier on a site name: "Munyaka 2", "Nellmapius Ext 25". +QUALIFIER = re.compile(r"\s+(?:ext\.?\s*\d*|\d+)$", re.I) +# MEGA does not always write a site's name the way the app has it. These are +# the spellings seen on real POs, mapped to the site they mean: +# "West Park EXT6" -> Wespark (MEGCPO9554, 2026-11-06) +# "Nellmpius EXT25" -> Nellmapius Ext 25 (MEGCPO9475, 2026-11-05) +SITE_ALIASES = { + "WESTPARK": "Wespark", + "NELLMPIUS": "Nellmapius Ext 25", +} # --------------------------------------------------------------------------- helpers @@ -245,41 +255,55 @@ def load_sites(token: str) -> list[dict]: return sites -def resolve_site_by_code(codes: list[str], sites: list[dict]) -> dict | None: - """The site named by the PO's own line-item project codes. +def name_variants(name: str) -> list[str]: + """The site name, and the same name without its instance qualifier. - A PO whose lines all carry one project code belongs to that site. Lines naming - two different sites are ambiguous -- MEGA's own finalisation step treats them - the same way -- so they are left for a human instead of being guessed at. + MEGA writes "Munyaka 2" but a PO may say only "Munyaka", and "Nellmapius + Ext 25" may appear as "Nellmapius". The site is still named. """ - matched = [] - for code in codes: - for site in sites: - if code.upper() == (site.get("code") or "").upper(): - matched.append(site) - break - if len(matched) == 1: - return matched[0] - return None + out = [name.strip()] + stem = name.strip() + while True: + stripped = QUALIFIER.sub("", stem) + if stripped == stem or not stripped: + break + stem = stripped + out.append(stem) + return out -def resolve_site(text: str, sites: list[dict]) -> dict | None: - """Find the MEGA site named in a PO's subject line.""" +def resolve_site_by_name(text: str, sites: list[dict]) -> dict | None: + """The MEGA site whose name is written in the text. + + This is what a PO is booked against. The project code on the PO's lines is + not used: it is a cost allocation, not a statement of where the work is + going -- MEGCPO9550 is a Cedar Gate delivery with four of its seven lines + billed to Castle Gate's project code. The name is written on the PO + itself, in its subject line or its description. + + A PO that names two sites is ambiguous, and is left for a person rather + than booked against one of them. + """ flat = norm(text) if not flat: return None + matched: list[dict] = [] for site in sites: - key = norm(site.get("code")) - if len(key) >= 3 and re.search(rf"\b{re.escape(site.get('code',''))}\b", text, re.I): - return site - for site in sites: - name = norm(site.get("name")) - if len(name) >= 4 and name in flat: - return site - for site in sites: - key = norm(site.get("code")) + for variant in name_variants(site.get("name") or ""): + key = norm(variant) + if len(key) >= 4 and key in flat: + matched.append(site) + break + for alias, target in SITE_ALIASES.items(): + key = norm(alias) if len(key) >= 4 and key in flat: - return site + for site in sites: + if site.get("name") == target: + matched.append(site) + break + unique = {site.get("id"): site for site in matched} + if len(unique) == 1: + return next(iter(unique.values())) return None @@ -447,14 +471,14 @@ def main() -> int: outcomes.append("duplicate_number") continue - # The site comes from the document itself, in order of how reliable each - # signal is: the project code on its own lines, then the site named in the - # PO's description, then its subject, then the email subject. + # The site comes from the name written on the PO: its subject line + # first, then its description, then the email subject. The project + # code on the PO's lines is a cost allocation, not a statement of + # where the work is going, so it never decides the site. site = ( - resolve_site_by_code(parsed["cost_codes"], sites) - or resolve_site(parsed["description"], sites) - or resolve_site(parsed["subject_line"], sites) - or resolve_site(msg["subject"], sites) + resolve_site_by_name(parsed["subject_line"], sites) + or resolve_site_by_name(parsed["description"], sites) + or resolve_site_by_name(msg["subject"], sites) ) if not site: report["needs_review"].append( @@ -462,7 +486,7 @@ def main() -> int: "po_number": po_number, "subject": msg["subject"], "file": filename, - "reason": "no MEGA site found in the PO subject line or the email", + "reason": "no MEGA site name found on the PO or in the email", } ) outcomes.append("needs_review") @@ -476,6 +500,7 @@ def main() -> int: "received": msg["received"], "site_code": site.get("code"), "site_name": site.get("name"), + "po_cost_codes": parsed["cost_codes"], "po_number_hint": po_number, "file_name": filename, "backfilled_at": now_iso(), diff --git a/scripts/test_po_email_intake.py b/scripts/test_po_email_intake.py index 556f41c..cc331e1 100644 --- a/scripts/test_po_email_intake.py +++ b/scripts/test_po_email_intake.py @@ -9,9 +9,9 @@ from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent)) from po_email_intake import ( # noqa: E402 + name_variants, parse_po_text, - resolve_site, - resolve_site_by_code, + resolve_site_by_name, ) # MEGA's sites, as in scripts/data/po-email-intake/sites.json @@ -21,6 +21,12 @@ SITES = [ {"id": "v61qv4s1omqqkc6", "name": "Mega Head Office", "code": "HEAD"}, {"id": "dmqybquea190z22", "name": "Nellmapius Ext 25", "code": "C26/27"}, {"id": "1d4hqov69qzajcp", "name": "Munyaka 2", "code": "C26/14"}, + {"id": "ne80o2ow1vblsnq", "name": "Cosmo City", "code": "C30/25"}, + {"id": "1q3fvsiq7vv28np", "name": "Halfway Gardens", "code": "C31/25"}, + {"id": "r4q8yrip5wbmxrp", "name": "Wespark", "code": "C26/10"}, + {"id": "uaqxp126by39jkb", "name": "Montana", "code": "C25/41"}, + {"id": "majjyyc7agr51aa", "name": "Mushroom farm", "code": "C11/23"}, + {"id": "newcastle", "name": "Castle Gate", "code": "C26/08"}, ] # The real layout of a MEGA PO, trimmed but faithful to rptViewer output. @@ -103,59 +109,86 @@ class ParsePoText(unittest.TestCase): self.assertEqual(parsed["required_by"], "10/11/2026") def test_reads_the_project_code_off_each_line(self): + # Kept as evidence on the filed row; it no longer decides the site. self.assertEqual(parse_po_text(PO_9547)["cost_codes"], ["C26/27"]) - def test_the_letterhead_is_not_mistaken_for_a_project_code(self): - # "Head office" in the contact block is a MEGA Head Office code, HEAD. - # It sits in the header, before the line items, so it must not be read - # as the site of this PO. - codes = parse_po_text(PO_9547)["cost_codes"] - self.assertNotIn("HEAD", codes) - self.assertEqual(resolve_site_by_code(codes, SITES)["name"], "Nellmapius Ext 25") - def test_a_document_with_no_line_items_has_no_project_code(self): self.assertEqual(parse_po_text("Purchase Order(s) #MEGCPO1\n")["cost_codes"], []) -class ResolveSite(unittest.TestCase): - def test_one_project_code_resolves_to_that_site(self): - site = resolve_site_by_code(["C26/27"], SITES) +class NameVariants(unittest.TestCase): + def test_drops_a_trailing_instance_number(self): + self.assertEqual(name_variants("Munyaka 2"), ["Munyaka 2", "Munyaka"]) + + def test_drops_a_trailing_ext_and_number(self): + self.assertEqual( + name_variants("Nellmapius Ext 25"), + ["Nellmapius Ext 25", "Nellmapius"], + ) + + def test_a_name_without_a_qualifier_is_left_alone(self): + self.assertEqual(name_variants("Castle Gate"), ["Castle Gate"]) + + +class ResolveSiteByName(unittest.TestCase): + """The site is booked against the name written on the PO.""" + + def test_a_name_in_the_po_subject_line_resolves(self): + site = resolve_site_by_name("MEGCREQ11958 (Castle Gate)", SITES) + self.assertEqual(site["name"], "Castle Gate") + + def test_a_name_in_the_po_description_resolves(self): + site = resolve_site_by_name("Tools (Nellmapius EXT25)", SITES) self.assertEqual(site["name"], "Nellmapius Ext 25") - def test_two_sites_in_the_lines_are_left_for_a_human(self): - self.assertIsNone(resolve_site_by_code(["C26/27", "C26/09"], SITES)) - - def test_no_project_code_resolves_to_nothing(self): - self.assertIsNone(resolve_site_by_code([], SITES)) - - def test_the_description_names_the_site(self): - site = resolve_site("Tools (Nellmapius EXT25)", SITES) - self.assertEqual(site["name"], "Nellmapius Ext 25") - - def test_a_site_named_in_the_subject_line_still_resolves(self): - site = resolve_site("MEGCREQ11928 a CO on MEGCREQ11927 (Nellmapius EXT25)", SITES) - self.assertEqual(site["name"], "Nellmapius Ext 25") + def test_a_loose_name_still_resolves(self): + site = resolve_site_by_name("MEGCREQ11879 Munyaka", SITES) + self.assertEqual(site["name"], "Munyaka 2") def test_the_hills_is_still_found(self): - site = resolve_site("MEGCREQ11911 The Hills Ext12", SITES) + site = resolve_site_by_name("MEGCREQ11911 The Hills Ext12", SITES) self.assertEqual(site["name"], "The Hills") - def test_head_office_is_found_by_its_code(self): - self.assertEqual(resolve_site("HEAD", SITES)["name"], "Mega Head Office") - def test_a_supplier_name_is_not_a_site(self): - self.assertIsNone(resolve_site("Chamberlain PO", SITES)) + self.assertIsNone(resolve_site_by_name("Chamberlain PO", SITES)) + + def test_no_name_at_all_is_not_a_site(self): + self.assertIsNone(resolve_site_by_name("MEGCREQ11948 a CO on MEGCREQ11947", SITES)) + + def test_a_project_code_is_not_a_site_name(self): + # The code on the lines is a cost allocation, not the site. + self.assertIsNone(resolve_site_by_name("C26/27", SITES)) + + def test_the_name_beats_a_cost_code_naming_another_site(self): + # MEGCPO9550: a Cedar Gate delivery with four of its seven lines billed + # to Castle Gate's project code. The name decides. + self.assertEqual( + resolve_site_by_name("MEGCREQ11956 (Cedar Gate)", SITES)["name"], + "Cedar Gate", + ) + + def test_a_po_naming_two_sites_is_left_for_a_person(self): + self.assertIsNone(resolve_site_by_name("Munyaka 2 delivery for Castle Gate", SITES)) + + def test_a_different_spelling_of_the_name_still_resolves(self): + # MEGA writes "West Park" where the app has "Wespark". + site = resolve_site_by_name("MEGCREQ11960 (West Park EXT6)", SITES) + self.assertEqual(site["name"], "Wespark") + + def test_a_misspelling_in_the_po_still_resolves(self): + # MEGCPO9475's own subject misspells the site: "Nellmpius EXT25". + site = resolve_site_by_name("MEGCREQ11843 (Nellmpius EXT25)", SITES) + self.assertEqual(site["name"], "Nellmapius Ext 25") class SiteOfAPo(unittest.TestCase): - """The order the intake uses: project code, then description, then subject.""" + """The order the intake uses: the PO's subject, then its description, then the email.""" def resolve(self, parsed, email_subject=""): return ( - resolve_site_by_code(parsed["cost_codes"], SITES) - or resolve_site(parsed["description"], SITES) - or resolve_site(parsed["subject_line"], SITES) - or resolve_site(email_subject, SITES) + resolve_site_by_name(parsed["subject_line"], SITES) + or resolve_site_by_name(parsed["description"], SITES) + or resolve_site_by_name(email_subject, SITES) ) def test_a_po_whose_email_subject_names_no_site_is_still_filed(self): @@ -164,11 +197,10 @@ class SiteOfAPo(unittest.TestCase): self.assertIsNotNone(site, "MEGCPO9547 names no site in its email subject") self.assertEqual(site["name"], "Nellmapius Ext 25") - def test_a_po_naming_two_sites_in_its_lines_is_left_for_review(self): + def test_a_po_naming_no_site_anywhere_needs_review(self): parsed = parse_po_text(PO_9547) - parsed["cost_codes"] = ["C26/27", "C26/09"] + parsed["subject_line"] = "MEGCREQ11948 a CO on MEGCREQ11947" parsed["description"] = "" - parsed["subject_line"] = "" self.assertIsNone(self.resolve(parsed, "Chamberlain PO"))