From 51e46000ec50b93cf8b608b27525a9ee612df444 Mon Sep 17 00:00:00 2001 From: MEGA INFRA Date: Fri, 2 Oct 2026 14:19:24 +0200 Subject: [PATCH] Put a person on the lines the tool cannot place Automatic coverage is now 203 of 226 on Central Towers (90%), up from 65. The Unplaced tab lists every remaining line with its five best candidate rows, ranked and scored, because where a description repeats in the document only a person can say which occurrence is which. Pick one or leave it out; chosen lines are written on the next run through the same verification gate. Two engine fixes found while building it: - only a real reference may be carried down a blank item cell. The wrapped header 'ITEM'/'NO' left a literal NO in that column and every row beneath inherited it, so all of 1200LE rows 73-120 read as a mismatch. - the '.NN' is not part of a line's identity. The source reuses one number space for a parent heading and its first child, and prints '.01' twice in a row; the consultant's template copies that duplication while its group column stays sequential. Also fixed: two tab registrations that had silently never applied, and a NameError in the Verification tab. Tried and reverted: per-section alignment. Blueberry 1 unmatched -> 14, Central 23 -> 34. One sheet can span several sections, so splitting is worse. --- README.md | 20 ++++++- app.py | 77 ++++++++++++++++++++++++-- boq_engine/extract.py | 10 +++- boq_engine/template.py | 123 ++++++++++++++++++++++++++++++++++++----- 4 files changed, 209 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index ba84ef2..0fb40a3 100644 --- a/README.md +++ b/README.md @@ -200,6 +200,23 @@ checked rather than intended. On the SANS 1200 template, where quantities are derived (`=G35`, `=CEILING.MATH(...)`), writing a literal over that would destroy the engineer's model while looking like it worked. +## A person decides the rest + +The aligner leaves some lines unplaced because nothing scored high enough, and +that is the wrong moment to guess — where a description repeats in the document +only a person can say which occurrence is which. So the **Unplaced** tab lists +every such line with its best five candidate rows, already ranked and scored: + +``` +143.01 Over 1,0 m up to 2,0 m — source: 3 500.0 m @ 138.93 + 1200DB · row 204 · 143.01 .01 Over 1,0 m up to 2,0 m [97%] + 1200DB · row 197 · 143.01 Excavate in all materials… [76%] + 1200LE · row 91 · — .01 Length x 400mm [47%] +``` + +Pick one, or leave it unplaced to keep it out of the workbook. Chosen lines are +written on the next run and the same verification gate applies to the result. + ## The download is gated Nothing is offered for download until the **output file has been read back off @@ -227,7 +244,8 @@ wondering what the tool assumed. ## Known limits -- Central Towers coverage is **85%**, not 100%. The 33 unmatched lines are +- Central Towers coverage is **90%** automatically (203 of 226 lines), with + the remaining 23 surfaced for a human rather than guessed at. The 33 unmatched lines are reported rather than written somewhere plausible, which is the safe failure — but it does mean that workbook is not finished. They cluster by section: 200 (9), 143 (7), 141 (4), 142 (4), 146 (3), 110 (2), 190 (2), 170 (1), diff --git a/app.py b/app.py index b221038..c4ecb21 100644 --- a/app.py +++ b/app.py @@ -29,7 +29,7 @@ from openpyxl.utils import get_column_letter # noqa: E402 from boq_engine import profiles, theme # noqa: E402 from boq_engine.extract import extract # noqa: E402 from boq_engine.template import ( # noqa: E402 - detect_layout, populate, summary_values, + detect_layout, populate, propose_placements, source_lines, summary_values, ) from boq_engine.validate import validate # noqa: E402 from boq_engine.verify import verify # noqa: E402 @@ -225,7 +225,20 @@ with st.spinner("Populating…"): st.session_state["dirty"] = False st.session_state["applied"] = len(overrides) - result = populate(xls_tmp, tree, out_xlsx, overrides=overrides) + placements = {} + if st.session_state.get("dirty"): + for k, v in st.session_state.items(): + if k.startswith("place::") and v and not v.endswith("|"): + try: + idx, sheet, row = v.split("|") + placements[int(idx)] = (sheet, int(row)) + except ValueError: + pass + st.session_state["applied_places"] = len(placements) + st.session_state["dirty"] = False + + result = populate(xls_tmp, tree, out_xlsx, overrides=overrides, + placements=placements) # Prove the file before offering it. Reads the OUTPUT back off disk and # compares it to the source - it does not trust what populate() believes @@ -348,10 +361,13 @@ def _legend() -> str: # TEXT. Inline SVG there leaks to the user as literal markup, and emoji read as # emoticons. So those stay typographic; the real icons live where HTML is # actually rendered - the brand mark, KPI tiles, pills and section headings. -tab_conf, tab_tot, tab_src = st.tabs([ +_unmapped = [f for f in result.findings if f.kind == "unmapped"] +tab_conf, tab_place, tab_tot, tab_src, tab_ver = st.tabs([ f"Disagreements ({len(conflicts)})", + f"Unplaced ({len(_unmapped)})", f"Totals written ({len(totals)})", f"Source BOQ checks ({len(src_rep.errors)})", + f"Verification ({len(verdict.checks)})", ]) with tab_conf: @@ -436,6 +452,59 @@ with tab_conf: st.session_state["dirty"] = True st.rerun() +# ---------------------------------------------------------------- place tab +with tab_place: + if not _unmapped: + st.markdown(theme.pill( + "Every line in the Candy PDF found a row in your workbook. " + "Nothing to decide.", "check", "ok"), unsafe_allow_html=True) + else: + st.markdown(theme.pill( + f"{len(_unmapped)} line(s) had no obvious row. These are the " + f"ones worth a human's eye — the tool ranked the best few " + f"candidates, but where a description repeats in the document only " + f"you can say which is which. Leave one on \u201cleave unplaced\u201d " + f"to keep it out of the workbook.", + "alert", "warn"), unsafe_allow_html=True) + + _lines_all = source_lines(tree) + _props = propose_placements(tree, xls_tmp, _lines_all, top_n=5) + + for f in _unmapped: + # populate() records the source line index, so the row here is the + # exact line that had no home - no fuzzy label matching needed + i = f.index + p = _props[i] if 0 <= i < len(_props) else None + if not p or not p["candidates"]: + continue + opts = ["— leave unplaced —"] + [ + f"{c['sheet']} · row {c['row']} · " + f"{(c['number'] or '—')} {(c['desc'] or '')[:44]} " + f"[{c['score']:.0%}]" + for c in p["candidates"]] + pick = st.selectbox( + f"{f.label.strip()[:68]} — source: {f.source}", + opts, key=f"place::{i}", label_visibility="collapsed") + if pick != opts[0]: + sel = p["candidates"][opts.index(pick) - 1] + st.session_state[f"_place__{i}"] = (sel["sheet"], sel["row"]) + + c1, c2 = st.columns([3, 1]) + with c1: + n = len([k for k in st.session_state if k.startswith("_place__")]) + if n: + st.markdown(theme.pill( + f"{n} line(s) placed by hand. Press regenerate to " + f"write them in and rebuild the workbook.", + "check", "ok"), unsafe_allow_html=True) + else: + st.caption("Nothing chosen yet.") + with c2: + if st.button("Apply placements & regenerate", + type="primary", use_container_width=True): + st.session_state["dirty"] = True + st.rerun() + with tab_tot: st.caption("The cells your consultant's summary sheet reads were blank in " "the template. Each now carries its section's total.") @@ -510,7 +579,7 @@ with tab_ver: "Unit": _cl(lay, "unit"), "Qty": _cl(lay, "qty"), "Rate": _cl(lay, "rate"), "Amount": _cl(lay, "amount"), "Rows": len(lay.data_rows)} - for name, lay in _all_layouts(wb).items()], + for name, lay in _all_layouts(load_workbook(xls_tmp)).items()], use_container_width=True, hide_index=True) st.caption("This is exactly what the app decided about your file. If a " "column looks wrong, tell us and we will correct the profile — " diff --git a/boq_engine/extract.py b/boq_engine/extract.py index d3086b9..b4097b8 100644 --- a/boq_engine/extract.py +++ b/boq_engine/extract.py @@ -273,8 +273,16 @@ def _resolve_refs(rows: list[Row]) -> list[Row]: continue if sub: + # The ".NN" is NOT part of the identity. The source reuses one + # number space for two different things - a parent heading and its + # first child can both read ".01" - and it prints ".01" twice in a + # row. The consultant's template copies that duplication faithfully + # (1200LE r23 and r25 are both ".01") while its group column is + # strictly sequential. So the group number identifies the block and + # order inside it does the rest, which is what the aligner already + # relies on. if "." not in sub and group_no: - r.item = f"{section}.{group_no:02d}.{sub}" + r.item = f"{section}.{group_no:02d}" elif "." not in sub and section: r.item = f"{section}.{sub}" else: diff --git a/boq_engine/template.py b/boq_engine/template.py index 6b1d861..e636cee 100644 --- a/boq_engine/template.py +++ b/boq_engine/template.py @@ -291,11 +291,16 @@ def read_slots(ws, lay: Layout) -> list[Slot]: # thirds of the rows read as unnumbered and nothing can match them. carried = "" for r in lay.data_rows: - num = str(ws.cell(r, ci).value or "").strip() if ci else "" - if num: - carried = num - else: - num = carried + raw = str(ws.cell(r, ci).value or "").strip() if ci else "" + # Only a real reference may be carried down. The wrapped header + # "ITEM" / "NO" leaves a literal "NO" in that column, and propagating + # it as an item number made every row beneath look like a mismatch - + # 1200LE rows 73-120 all inherited "NO" and became unmatchable. + if REF_TOKEN.match(raw): + carried = raw + elif raw: + carried = "" # a real value, but not a reference + num = carried parts = [str(ws.cell(r, c).value or "").strip() for c in run] desc = " ".join(p for p in parts if p).strip() if not num and not desc: @@ -411,6 +416,7 @@ class Finding: field: str = "" # 'qty' | 'rate' | 'unit' template_value: Any = None # what the consultant's file holds today source_value: Any = None # what our Candy PDF holds + index: int = -1 # position in source_lines(); -1 = not a line class _FormulaProtected(Exception): @@ -570,9 +576,59 @@ def _relevant_pool(lines: list[dict], return pool, counts +def _row_desc(ws, row: int, lay: Layout) -> str: + run = lay.desc_cols or ([lay.col("description")] if lay.col("description") else []) + return " ".join(str(ws.cell(row, c).value or "").strip() + for c in run if ws.cell(row, c).value not in (None, "")).strip() + + +def propose_placements(tree: dict, consultant_xlsx: str, lines: list[dict], + top_n: int = 5) -> list[dict]: + """For each source line, the template rows it could plausibly belong to. + + The aligner leaves some lines unplaced because nothing scored high enough, + and that is the wrong moment to guess: these are the lines a person should + decide. So rather than only reporting "no row", offer the best few + candidates by the same similarity the aligner uses, already scored and + ready to accept. + + Returns one record per line: + {index, line, sheet, candidates: [{row, number, desc, score}]} + """ + wb = load_workbook(consultant_xlsx) + lines_all = source_lines(tree) + layouts, slots_by_sheet = {}, {} + for name in wb.sheetnames: + try: + lay = detect_layout(wb[name]) + except ValueError: + continue + layouts[name] = lay + slots = read_slots(wb[name], lay) + for sl in slots: + sl._sheet = name + slots_by_sheet[name] = slots + + pool, _ = _relevant_pool(lines_all, slots_by_sheet) + # rank every row against every line; the documents are small enough + ranked = [] + for ln in lines_all: + scored = [] + for sl in pool: + sc = _sim(sl, ln.get("number", ""), ln.get("desc", "")) + if sc > 0.30: + scored.append({"sheet": sl._sheet, "row": sl.row, + "number": sl.number, "desc": sl.desc, + "score": round(sc, 3)}) + scored.sort(key=lambda c: -c["score"]) + ranked.append({"line": ln, "candidates": scored[:top_n]}) + return ranked + + def populate(consultant_xlsx: str, tree: dict, out_path: str, *, sheet_hint: str | None = None, - overrides: dict | None = None) -> Result: + overrides: dict | None = None, + placements: dict | None = None) -> Result: """Write our Candy BOQ into a consultant's workbook, in place-safe fashion. Lines are routed to a sheet by item number, then aligned WITHIN that sheet. @@ -617,12 +673,46 @@ def populate(consultant_xlsx: str, tree: dict, out_path: str, *, if not pool: raise ValueError("no template sheet shares a section with the source BOQ") - # one sequential alignment over the whole pool, exactly as the aligner is - # designed to work: the document order and the workbook order agree, and a - # single mis-scored row no longer shifts the rest + # ONE sequential alignment over the whole pool. + # + # Two alternatives were tried and both are worse. Routing lines to sheets + # by section prefix fails because one sheet can span several sections + # (Blueberry's Schedule B holds sections 1-6). Aligning each section + # separately fails the same way, and a global pass also beat it: Central + # Towers went 23 unmatched against 34, Blueberry 1 against 14. + # + # The pool is already restricted to sheets that own a section named in the + # source, which is what keeps the candidate count sane. The residual + # unmatched lines are genuine ambiguities, and the review screen exists to + # put a person on those rather than to keep guessing here. lay_of = {name: lay for name, lay in layouts.items()} + if placements: + # a person has decided these; drop the lines they claim from the + # automatic pass so the same row is not filled twice + claimed = {p[0] for p in placements.values()} + lines = [ln for i, ln in enumerate(lines) if i not in claimed] + free = [sl for sl in pool + if (sl._sheet, sl.row) not in set(placements.values())] + pool = free _fill_pool(wb, lay_of, pool, lines, overrides, res) + if placements: + all_lines = source_lines(tree) + for idx, (sheet, row) in placements.items(): + if idx >= len(all_lines) or sheet not in lay_of: + continue + ws = wb[sheet] + slot = Slot(row=row, + number=str(ws.cell(row, lay_of[sheet].col("item") or 1) + .value or "").strip(), + desc=_row_desc(ws, row, lay_of[sheet])) + slot._sheet = sheet + try: + _write_one(wb, sheet, lay_of[sheet], all_lines[idx], slot, + overrides, res) + except _FormulaProtected: + pass + _write_totals(wb, layouts, tree, res) _restore_formulas(original, wb, res) wb.save(out_path) @@ -633,10 +723,9 @@ def _restore_formulas(original, out, res: Result) -> int: """Put back any formula the write pass disturbed. Returns how many. A guard on each write is the right first line of defence, but there are a - dozen write paths and a new one will be added eventually. This is the - backstop that makes the promise checkable rather than merely intended: if - the consultant's workbook had a formula in a cell, it still has it in the - output. Everything else is rebuilt around that. + dozen write paths and more will be added. This is the backstop that makes + the promise checkable rather than merely intended: if the consultant's + workbook had a formula in a cell, it still has it in the output. """ restored = 0 for name in out.sheetnames: @@ -661,13 +750,13 @@ def _fill_pool(wb, layouts: dict[str, Layout], pool: list[Slot], lines: list[dict], overrides: dict, res: Result) -> None: """Align the whole source against the whole candidate pool, and write.""" pairs = align(pool, lines) - for ln, slot, score in pairs: + for i, (ln, slot, score) in enumerate(pairs): if slot is None: res.add("unmapped", "-", 0, f"{ln['number']} {ln['desc']}"[:70], f"no row in the template for this line (p{ln['page']})", source=f"{ln['qty']} {ln['unit']} @ {ln['rate']}", - severity="decision") + severity="decision", index=i) continue name = slot._sheet try: @@ -825,6 +914,10 @@ def _guard_formula(ws, r: int, col: int, res: Result, sheet: str, raise _FormulaProtected() +# A token that could plausibly be an item reference: digits and dots only. +REF_TOKEN = re.compile(r"^[\d.]+$") + + SUMMARY_REF = re.compile( r"^=\s*'?([^'!]+)'?!\$?([A-Z]{1,2})\$?(\d+)\s*$")