From 68cf65e544a235dc508d08d1f247e62e22e2eddf Mon Sep 17 00:00:00 2001 From: MEGA INFRA Date: Sun, 4 Oct 2026 02:06:23 +0200 Subject: [PATCH] Fix values landing one row above the line they belong to Caught by the engineer: '.04 Appointment of Community Liaison Officer' showed RATE ONLY in the spreadsheet while the PDF has Month, 8, R20 000.00, R160 000.00. The values had landed on the row ABOVE it. Two causes, both the same underlying mistake. 1. A BOQ row here is often TWO rows - the one with the item number, unit and quantity, and a following row carrying the wrapped tail of the description: r41 C=.04 Appointment of Community Liaison Officer Month 8 20 000 r42 for the duration of the contract period Those tails were separate candidates, and the aligner matched the line to the PREVIOUS line's tail. A row with no item number of its own and nothing in the priced columns is now joined onto the line above. 2. The row classifier still tested the single column the header claimed for DESCRIPTION (E) while the text is in D - the same header-lies problem I had fixed for matching but not for classification. Row 41 was never counted as a line, so it was never even a candidate. Both now use the description RUN. Also worth being precise about: the 'RATE ONLY' in that cell was the consultant's own pre-existing text in his template, not something this tool wrote. The bug was the values going to the wrong row. Central Towers 192 written / 23 unplaced -> 196 / 22, and four more genuine disagreements are now visible because values land on the right rows. Blueberry unchanged at 103 / 1. Zero formulas damaged on both, gate not blocked. --- README.md | 25 ++++++++++++++ boq_engine/template.py | 77 ++++++++++++++++++++++++++++++------------ 2 files changed, 80 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index d339763..5329c15 100644 --- a/README.md +++ b/README.md @@ -232,6 +232,31 @@ checked rather than intended. On the SANS 1200 template, where quantities are derived (`=G35`, `=CEILING.MATH(...)`), writing a literal over that would destroy the engineer's model while looking like it worked. +## A BOQ row is often two rows + +On the SANS 1200 template each line occupies **two** rows: the one carrying the +item number, unit and quantity, and a following row holding the wrapped tail of +the description — + +``` +r41 C=.04 Appointment of Community Liaison Officer Month 8 20 000.00 +r42 for the duration of the contract period +``` + +Left as separate candidates those tails are decoys. The aligner matched +`.04 Appointment of Community Liaison Officer` to the **previous** line's tail +and the values landed one row above the label they belonged to — a wrong +figure on a tender document. A row with no item number of its own and nothing +in the priced columns is now joined onto the line above. + +Related, and the same root cause twice over: the header claims DESCRIPTION is +in column E while the text is in D. That was already handled when *matching*, +but the **row classifier** still tested column E alone — so row 41 was never +counted as a line at all and never became a candidate. Both now use the +description *run*. Central Towers went 192 written / 23 unplaced to +**196 / 22**, with 4 more genuine disagreements now visible because the values +are landing on the right rows. + ## Tariffs carried inside a sum Candy sometimes prints a **word where a rate would go** — `INCL`, `Incl VAT`, diff --git a/boq_engine/template.py b/boq_engine/template.py index 10dc01c..495dbe3 100644 --- a/boq_engine/template.py +++ b/boq_engine/template.py @@ -172,6 +172,8 @@ def detect_layout(ws, hint: "SheetProfile | None" = None) -> Layout: lay.notes.append(f"header wraps over rows {header_row}" f"-{header_row + span - 1}") + _verify_against_data(ws, lay) + _derive_desc_run(lay) _rebuild_rows(ws, lay, start=header_row + span) for name in ("item", "description", "unit", "qty", "rate", "amount"): @@ -180,8 +182,6 @@ def detect_layout(ws, hint: "SheetProfile | None" = None) -> Layout: if "payment" in cols: lay.notes.append("PAYMENT column detected and will be left untouched") - _verify_against_data(ws, lay) - _derive_desc_run(lay) return lay @@ -205,14 +205,23 @@ def _derive_desc_run(lay: Layout) -> None: def _rebuild_rows(ws, lay: Layout, start: int | None = None) -> None: - """Re-scan which rows are data and which are totals, from a column map.""" + """Re-scan which rows are data and which are totals, from a column map. + + Tests the description RUN, not the single column the header claimed. On + the SANS 1200 template the header says DESCRIPTION is in E while the text + is in D, so testing E alone classified real line rows as blank - and + ".04 Appointment of Community Liaison Officer" never became a candidate at + all. + """ first = lay.header_row + 1 if start is None else start c_item = lay.cols.get("item") - c_desc = lay.cols.get("description") c_qty = lay.cols.get("qty") + run = lay.desc_cols or ([lay.cols["description"]] + if "description" in lay.cols else []) for r in range(first, ws.max_row + 1): a = ws.cell(r, c_item).value if c_item else None - d = ws.cell(r, c_desc).value if c_desc else None + d = " ".join(str(ws.cell(r, c).value or "").strip() + for c in run).strip() or None q = ws.cell(r, c_qty).value if c_qty else None label = f"{a or ''} {d or ''}".strip() if label and TOTAL_RE.match(label): @@ -325,33 +334,57 @@ class Slot: def read_slots(ws, lay: Layout) -> list[Slot]: + """One slot per real line. + + A BOQ row in these templates is often TWO rows: the one carrying the item + number, unit and quantity, and a following row holding the wrapped tail of + the description - "Appointment of Community Liaison Officer" on one row and + "for the duration of the contract period" on the next. + + Left as separate slots, those tails become decoys. The aligner matched + ".04 Appointment of Community Liaison Officer" to the PREVIOUS line's tail + row, and the values landed one row above the label they belonged to. A row + that has no item number of its own and nothing in the priced columns is a + continuation, so its text is joined onto the line above. + """ ci = lay.col("item") - run = lay.desc_cols or ([lay.col("description")] if lay.col("description") else []) + cu, cq, cr = lay.col("unit"), lay.col("qty"), lay.col("rate") + run = lay.desc_cols or ([lay.col("description")] + if lay.col("description") else []) out: list[Slot] = [] - # A spreadsheet implies its hierarchy: once "110.01" is printed, the blank - # cells below it still belong to 110.01. Without carrying that down, two - # thirds of the rows read as unnumbered and nothing can match them. carried = "" + for r in lay.data_rows: raw = str(ws.cell(r, ci).value or "").strip() if ci else "" - # Only a real reference may be carried down. The wrapped header - # "ITEM" / "NO" leaves a literal "NO" in that column, and propagating - # it as an item number made every row beneath look like a mismatch - - # 1200LE rows 73-120 all inherited "NO" and became unmatchable. + # Only a real reference may be carried down a blank item cell. The + # wrapped header "ITEM" / "NO" leaves a literal "NO" in that column, + # and propagating it made every row beneath look like a mismatch. if REF_TOKEN.match(raw): carried = raw elif raw: - carried = "" # a real value, but not a reference + carried = "" num = carried - parts = [str(ws.cell(r, c).value or "").strip() for c in run] - desc = " ".join(p for p in parts if p).strip() - if not num and not desc: + + desc = _row_desc(ws, r, lay) + priced = any(ws.cell(r, c).value not in (None, "") + for c in (cu, cq, cr) if c) + + if not desc and not priced and not raw: + continue # wholly empty spacer row + + if out and not priced and not raw and desc: + # the wrapped tail of the line above + if out[-1].desc: + out[-1].desc = f"{out[-1].desc} {desc}".strip() + else: + out[-1].desc = desc continue - # indent level from the leading whitespace the consultant preserved - raw = next((str(ws.cell(r, c).value or "") for c in run - if str(ws.cell(r, c).value or "") != - str(ws.cell(r, c).value or "").lstrip()), "") - indent = len(raw) - len(raw.lstrip()) if raw else 0 + + raw_desc = next((str(ws.cell(r, c).value or "") for c in run + if str(ws.cell(r, c).value or "") + != str(ws.cell(r, c).value or "").lstrip()), "") + indent = (len(raw_desc) - len(raw_desc.lstrip()) + if raw_desc != raw_desc.lstrip() else 0) out.append(Slot(row=r, number=num, desc=desc, indent=indent)) return out