diff --git a/app.py b/app.py
index 42381e5..f58ca5a 100644
--- a/app.py
+++ b/app.py
@@ -354,11 +354,20 @@ if verdict.blocked:
st.caption("Send these to us; this is a bug in the converter, not in "
"your file.")
else:
- st.markdown(theme.pill(
- f"Verified safe to send. {_tail}Your file was populated using "
- f"its own formulas and layout, and every write was read back and "
- f"checked. Download it and send it with the Candy PDF.",
- "shield", "ok"), unsafe_allow_html=True)
+ if verdict.needs_review:
+ st.markdown(theme.pill(
+ f"Mostly there — read the MEGA CHECK sheet. A few sections "
+ f"do not add up to the Candy PDF, and the sheet at the front lists "
+ f"which ones and the rows to look at. Work down that list, then "
+ f"send it with the PDF.",
+ "alert", "warn"), unsafe_allow_html=True)
+ else:
+ st.markdown(theme.pill(
+ f"Ready to send. {_tail}Every section we could compare adds "
+ f"up to the Candy PDF, and every write was read back out of the "
+ f"file and checked. The MEGA CHECK sheet at the front shows the "
+ f"section totals side by side.",
+ "check", "ok"), unsafe_allow_html=True)
g1, g2 = st.columns([1, 1])
with g1:
@@ -368,7 +377,7 @@ else:
file_name=os.path.basename(out_xlsx),
mime="application/vnd.openxmlformats-"
"officedocument.spreadsheetml.sheet")
- with g2:
+with g2:
st.markdown(theme.pill(
f"Source PDF totals R{verdict.source_total:,.2f} before "
f"VAT. The populated workbook keeps every one of the "
diff --git a/boq_engine/reconcile.py b/boq_engine/reconcile.py
new file mode 100644
index 0000000..40ff284
--- /dev/null
+++ b/boq_engine/reconcile.py
@@ -0,0 +1,454 @@
+"""
+Section-level reconciliation: what the PDF says, what the workbook now says.
+
+The placement check tells us *a* line is wrong. What the person at the
+spreadsheet actually needs is different: he opens the file, looks at the totals,
+and jumps to the section that doesn't add up.
+
+So this compares, section by section:
+
+ PDF printed total vs the workbook as populated
+
+Pairing the two is the subtle part. The consultant's SUMMARY sheet lists its
+sections in a DIFFERENT order from the PDF's closing summary — 1200DM, 1200DK,
+1200L where the PDF has sections 141, 142, 170 — so pairing by position shifts
+every figure and produces a confident, wrong report. Instead each PDF section is
+matched to the sheet that actually holds its lines, counted by containment.
+
+Every template already carries a summary sheet, so the section-to-sheet map
+comes out of their own file. No consultant-specific configuration.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+
+from openpyxl.styles import Alignment, Border, Font, PatternFill, Side
+
+TOL = 0.02
+XREF = re.compile(r"^=?\+?'?([^'!]+)'?!(\$?[A-Z]{1,3}\$?\d+)")
+
+
+@dataclass
+class Reconcile:
+ """Everything the check sheet needs, including what could NOT be checked."""
+ checks: list = field(default_factory=list)
+ unmatched: list = field(default_factory=list) # (label, total) not paired
+ orphans: list = field(default_factory=list) # (sheet, row, why)
+
+
+@dataclass
+class SectionCheck:
+ label: str
+ sheet: str
+ cell: str = ""
+ expected: float | None = None # what the PDF printed
+ actual: float | None = None # what we wrote
+ rows: list = field(default_factory=list) # (row, why) needing a look
+
+ @property
+ def delta(self) -> float | None:
+ if self.expected is None or self.actual is None:
+ return None
+ return round(self.actual - self.expected, 2)
+
+ @property
+ def ok(self) -> bool:
+ return self.delta is not None and abs(self.delta) <= TOL
+
+ @property
+ def no_figure(self) -> bool:
+ """The PDF prints zero here, so it has no opinion to disagree with.
+
+ Candy prints 0.00 for a section it is handing the tenderer to price -
+ Blueberry's "4 CABLE DUCTS" is twelve lines that all carry qty 0.0.
+ The consultant then puts their OWN quantities in. Reporting that as a
+ R348,746.25 difference points at our work when the truth is the source
+ simply has nothing to compare against.
+ """
+ return self.expected is not None and abs(self.expected) <= TOL
+
+ @property
+ def status(self) -> str:
+ if self.no_figure:
+ return "no figure in PDF"
+ if self.ok:
+ return "balances"
+ if self.actual is None:
+ return "not checked"
+ return "CHECK"
+
+
+# --------------------------------------------------------------------------
+# pairing
+# --------------------------------------------------------------------------
+def _summary_refs(wb) -> dict:
+ """sheet -> total cell, from the workbook's own summary sheet."""
+ for name in wb.sheetnames:
+ if "summar" not in name.lower():
+ continue
+ out = {}
+ for row in wb[name].iter_rows():
+ for c in row:
+ if isinstance(c.value, str) and c.value.startswith("="):
+ m = XREF.match(c.value)
+ if m and m.group(1) in wb.sheetnames:
+ out[m.group(1)] = m.group(2).replace("$", "")
+ if out:
+ return out
+ return {}
+
+
+def _sheet_texts(out_wb) -> dict:
+ from boq_engine.template import detect_layout, read_slots
+ texts = {}
+ for name in out_wb.sheetnames:
+ try:
+ lay = detect_layout(out_wb[name])
+ except ValueError:
+ continue
+ parts = [sl.desc for sl in read_slots(out_wb[name], lay) if sl.desc]
+ if parts:
+ texts[name] = " || ".join(parts).lower()
+ return texts
+
+
+def _sect_key(label: str) -> str:
+ """The part of a section label that actually identifies it.
+
+ Candy restarts its numbering inside every schedule, so "1" can be three
+ different sections. The TITLE is what distinguishes them.
+ """
+ s = re.sub(r"[^a-z0-9]+", " ", (label or "").lower()).strip()
+ return re.sub(r"\b\d[\d. ]*\b", "", s).strip() or s
+
+
+def _row_sections(result, tree, out_wb) -> dict:
+ """(sheet, row) -> section key, for every row we can attribute.
+
+ Two sources, in order of trust:
+ 1. what the writer recorded placing there - exact, because the writer
+ knows this for certain and nobody has to guess it afterwards;
+ 2. the row's own description, matched against the source descriptions.
+ """
+ from boq_engine.template import source_lines
+
+ by_desc: dict[str, str] = {}
+ by_sec_desc: dict[str, set] = {}
+ for ln in source_lines(tree):
+ key = _sect_key(ln.get("sec", ""))
+ d = (ln["desc"] or "").lower().strip()
+ if d:
+ by_desc.setdefault(d, key)
+ by_sec_desc.setdefault(key, set()).add(d[:26])
+
+ out: dict = {}
+ for p in getattr(result, "placements", []) or []:
+ d = (p.get("desc") or "").lower().strip()
+ key = by_desc.get(d)
+ if key:
+ out[(p["sheet"], p["row"])] = key
+
+ for sheet, text in _sheet_row_texts(out_wb).items():
+ for row, d in text.items():
+ if (sheet, row) in out or not d:
+ continue
+ for key, descs in by_sec_desc.items():
+ if d[:26] in descs:
+ out[(sheet, row)] = key
+ break
+ return out
+
+
+def _sheet_row_texts(out_wb) -> dict:
+ """(sheet, row) -> lowercased description, from the written file."""
+ from boq_engine.template import detect_layout, read_slots
+
+ wb = out_wb
+ out: dict = {}
+ for name in wb.sheetnames:
+ try:
+ lay = detect_layout(wb[name])
+ except ValueError:
+ continue
+ for sl in read_slots(wb[name], lay):
+ d = (sl.desc or "").lower().strip()
+ if d:
+ out.setdefault(name, {})[sl.row] = d
+ return out
+
+
+def _pair(tree, out_wb, expected_values) -> tuple:
+ """(pairs, unmatched) - (label, sheet, total) per section, by content.
+
+ Pairing by POSITION produces a confident, wholly wrong report: the
+ consultant's SUMMARY lists 1200DM/1200DK/1200L where the PDF lists
+ sections 141/142/170. So each section is matched to the sheet that actually
+ holds its lines, counted by containment.
+
+ Sections that find no home are RETURNED, not dropped. A section that is
+ silently skipped is worse than one reported as unmatched, because the
+ check sheet then implies the untested sections balance - which is the exact
+ false reassurance this sheet exists to remove.
+ """
+ from boq_engine.template import source_lines
+
+ by_section: dict[str, list] = {}
+ for ln in source_lines(tree):
+ by_section.setdefault(_sect_key(ln.get("sec", "")), []).append(
+ ln["desc"].lower())
+
+ texts = _sheet_texts(out_wb)
+ taken: set = set()
+ pairs = []
+ unmatched = []
+ for label, total in expected_values:
+ key = _sect_key(label)
+ cand = [d for d in by_section.get(key, []) if d and len(d) > 6][:40]
+ scored = []
+ for name, text in texts.items():
+ hits = sum(1 for d in cand if d[:26] in text)
+ if hits:
+ scored.append((hits, name))
+ if not scored:
+ unmatched.append((label, round(float(total), 2)))
+ continue
+ # One consultant sheet can legitimately hold several sections -
+ # Blueberry's SCHEDULE B - ROADS carries sections 1 to 6. So a sheet is
+ # not consumed by its first match, and an UNCLAIMED sheet always beats
+ # a claimed one even on a lower hit count: Central Towers keeps 141, 142
+ # and 143 on their own sheets (DB PdW / 1200LG / MK PdW) instead of all
+ # three collapsing onto 1200DB, which merely happens to mention all
+ # three. A claimed sheet is still the fallback, which is what rescues
+ # the rest of Blueberry's Schedule B.
+ scored.sort(key=lambda hv: (hv[1] in taken, -hv[0], hv[1]))
+ best_hits, best = scored[0]
+ if best not in taken:
+ taken.add(best)
+ pairs.append((label, best, round(float(total), 2)))
+ return pairs, unmatched
+
+
+# --------------------------------------------------------------------------
+# reconciliation
+# --------------------------------------------------------------------------
+def reconcile(tree, out_wb, result, expected_values) -> list:
+ from boq_engine.template import detect_layout
+
+ refs = _summary_refs(out_wb)
+ layouts = {}
+ for name in out_wb.sheetnames:
+ try:
+ layouts[name] = detect_layout(out_wb[name])
+ except ValueError:
+ pass
+
+ checks = []
+ orphans: list = []
+ pairs, unmatched = _pair(tree, out_wb, expected_values)
+
+ # Which section does each ROW belong to. Summing a whole sheet is wrong the
+ # moment one sheet holds more than one section - Blueberry's Schedule B
+ # carries sections 1 through 6, and reporting its full total against
+ # "1 SITE CLEARANCE" (R160,057.99) as R4,422,018 sends the consultant to
+ # the right section with a figure that is simply not that section's.
+ row_sec = _row_sections(result, tree, out_wb)
+
+ for label, sheet, total in pairs:
+ chk = SectionCheck(label, sheet, refs.get(sheet, ""), expected=total)
+ lay = layouts.get(sheet)
+ if lay is not None:
+ ws = out_wb[sheet]
+ cq, cr = lay.col("qty"), lay.col("rate")
+ if cq and cr:
+ key = _sect_key(label)
+ acc, seen = 0.0, 0
+ for r in lay.data_rows:
+ got = row_sec.get((sheet, r))
+ # A row we cannot attribute counts for NOTHING. Counting it
+ # against every section made 141, 142 and 143 all report
+ # the same 100,722.77 - three different sections cannot
+ # each own the same money.
+ if got is None or got != key:
+ continue
+ q, rate = ws.cell(r, cq).value, ws.cell(r, cr).value
+ if isinstance(q, (int, float)) and isinstance(rate, (int, float)):
+ acc += float(q) * float(rate)
+ seen += 1
+ # a section with no identifiable rows must say nothing rather
+ # than claim 0.00 - "0.00" reads as "we checked and it is nil"
+ if seen:
+ chk.actual = round(acc, 2)
+ checks.append(chk)
+
+ for sheet, row, why, sec in _attention_rows(result, out_wb):
+ target = None
+ if sheet and sheet != "-":
+ for chk in checks:
+ if chk.sheet == sheet:
+ target = chk
+ break
+ if target is None and sec:
+ # A line with no row to go to has no sheet either. It still belongs
+ # to a section though, and hanging it off THAT is the difference
+ # between a person seeing it and it vanishing.
+ key = _sect_key(sec)
+ for chk in checks:
+ if _sect_key(chk.label) == key:
+ target = chk
+ break
+ if target is not None:
+ target.rows.append((row if row else "add", why))
+ else:
+ # nothing to attach to. Kept anyway - a line nobody can place is
+ # the single thing most worth a person's attention, and dropping it
+ # because we could not categorise it is the wrong way round.
+ orphans.append((sheet if sheet and sheet != "-" else "-",
+ row if row else "add", why))
+ return Reconcile(checks, unmatched, orphans)
+
+
+def _attention_rows(result, out_wb) -> list:
+ """Rows a person still has to deal with, with the reason.
+
+ A row is a duplicate here by design: the totals tell him WHICH SECTION is
+ out, and this list tells him which rows inside it. Both are needed, and
+ neither is much use on its own.
+ """
+ from boq_engine.verify import _placement_report
+
+ rows = []
+ for sheet, row, want, why in _placement_report(result, out_wb):
+ rows.append((sheet, row, f"value may belong to another line - {why}", ""))
+ for f in result.findings:
+ if f.kind == "unmapped":
+ rows.append(("-", 0,
+ f"'{f.label.strip()[:60]}' had no row to go to "
+ f"— {f.source}".strip(),
+ getattr(f, "section", "")))
+ return rows
+
+
+# --------------------------------------------------------------------------
+# the check sheet
+# --------------------------------------------------------------------------
+ORANGE_F = PatternFill("solid", fgColor="2A2118")
+BAND = PatternFill("solid", fgColor="2A2F3C")
+OK_FILL = PatternFill("solid", fgColor="1F3A33")
+BAD_FILL = PatternFill("solid", fgColor="3A2A2A")
+WARN_FILL = PatternFill("solid", fgColor="3A3730")
+
+THIN = Side(style="thin", color="3A4050")
+BOX = Border(left=THIN, right=THIN, top=THIN, bottom=THIN)
+
+
+def write_check_sheet(wb, rec, *, out_name: str = ""):
+ """Put the reconciliation and the punch list at the front of the book."""
+ checks = rec.checks if isinstance(rec, Reconcile) else rec
+ unmatched = rec.unmatched if isinstance(rec, Reconcile) else []
+ orphans = rec.orphans if isinstance(rec, Reconcile) else []
+ if "MEGA CHECK" in wb.sheetnames:
+ del wb["MEGA CHECK"]
+ ws = wb.create_sheet("MEGA CHECK", 0)
+
+ ws["A1"] = "MEGA INFRA \u2014 what still needs a person"
+ ws["A1"].font = Font(bold=True, size=14, color="FEA944")
+ ws["A2"] = ("Section totals printed in the Candy PDF, against this workbook "
+ "as populated. A section that balances needs nothing.")
+ ws["A2"].font = Font(size=9, italic=True, color="999999")
+ if out_name:
+ ws["A3"] = out_name
+ ws["A3"].font = Font(size=9, color="666666")
+
+ r = 5
+ if unmatched:
+ # Say the quiet part out loud. A table of N clean sections reads as
+ # "this book is checked" unless it is stated that M sections were never
+ # tested at all - which is the exact false reassurance this sheet is
+ # supposed to remove.
+ c = ws.cell(r, 1, (
+ "⚠ %d section(s) in the PDF could NOT be matched to a sheet in "
+ "this workbook, so they are NOT covered by the table below: %s. "
+ "Please check those by hand." % (
+ len(unmatched),
+ ", ".join(str(l)[:28] for l, _ in unmatched[:6]) +
+ ("…" if len(unmatched) > 6 else ""))))
+ c.font = Font(size=10, bold=True, color="FEA944")
+ c.fill = WARN_FILL
+ r += 2
+
+ for i, h in enumerate(["SECTION", "SHEET", "PDF SAYS",
+ "WORKBOOK SAYS", "DIFFERENCE", "STATUS"], start=1):
+ c = ws.cell(r, i, h)
+ c.font = Font(bold=True, size=9, color="E1E1E1")
+ c.fill = BAND
+ c.border = BOX
+ c.alignment = Alignment(horizontal="left")
+
+ n_bad = 0
+ for chk in checks:
+ r += 1
+ d = chk.delta
+ st = chk.status
+ if st == "CHECK":
+ n_bad += 1
+ for i, v in enumerate([chk.label, chk.sheet, chk.expected,
+ chk.actual, d, st], start=1):
+ c = ws.cell(r, i, v)
+ c.border = BOX
+ c.font = Font(size=10)
+ if i in (3, 4, 5) and isinstance(v, (int, float)):
+ c.number_format = "#,##0.00"
+ if i == 6:
+ c.font = Font(size=10, bold=(st == "CHECK"),
+ color={"balances": "00CD8A",
+ "no figure in PDF": "999999",
+ "not checked": "999999"}.get(st, "FEA944"))
+ fill = (OK_FILL if st == "balances"
+ else (WARN_FILL if st in ("no figure in PDF", "not checked")
+ else BAD_FILL))
+ for i in range(1, 7):
+ ws.cell(r, i).fill = fill
+
+ r += 2
+ ws.cell(r, 1, "ROWS TO LOOK AT").font = Font(bold=True, size=11,
+ color="FEA944")
+ r += 1
+ for i, h in enumerate(["SHEET", "ROW", "WHY"], start=1):
+ c = ws.cell(r, i, h)
+ c.font = Font(bold=True, size=9, color="E1E1E1")
+ c.fill = BAND
+ c.border = BOX
+ n_rows = 0
+ for chk in checks:
+ for row, why in chk.rows:
+ r += 1
+ n_rows += 1
+ for i, v in enumerate([chk.sheet, row, why], start=1):
+ c = ws.cell(r, i, v)
+ c.border = BOX
+ c.font = Font(size=10)
+
+ for sheet, row, why in orphans:
+ r += 1
+ n_rows += 1
+ for i, v in enumerate([sheet, row, why], start=1):
+ c = ws.cell(r, i, v)
+ c.border = BOX
+ c.font = Font(size=10, color="FEA944")
+ c.fill = WARN_FILL
+
+ if n_rows == 0:
+ r += 1
+ c = ws.cell(r, 1, "Nothing — every line landed where it belongs.")
+ c.font = Font(size=10, italic=True, color="00CD8A")
+ else:
+ r += 1
+ c = ws.cell(r, 1, "%d line(s) above need a person." % n_rows)
+ c.font = Font(size=9, italic=True, color="999999")
+
+ for col, w in (("A", 46), ("B", 14), ("C", 16), ("D", 18), ("E", 15), ("F", 12)):
+ ws.column_dimensions[col].width = w
+ ws.sheet_view.showGridLines = False
+ return n_bad, n_rows
diff --git a/boq_engine/template.py b/boq_engine/template.py
index c6f4316..385e9ba 100644
--- a/boq_engine/template.py
+++ b/boq_engine/template.py
@@ -565,6 +565,11 @@ class Finding:
template_value: Any = None # what the consultant's file holds today
source_value: Any = None # what our Candy PDF holds
index: int = -1 # position in source_lines(); -1 = not a line
+ # The top-level section this line came from, as a bare number ("140").
+ # A line the aligner could not place has no row, so without this there is
+ # nothing to hang it off in the MEGA CHECK punch list and it simply
+ # disappears - which is the one case a person most needs to be told about.
+ section: str = ""
class _FormulaProtected(Exception):
@@ -620,11 +625,31 @@ class Result:
# source-side flattening: PDF tree -> the list we try to place
# --------------------------------------------------------------------------
def source_lines(tree: dict, *, with_prices: bool = True) -> list[dict]:
- """Every priced line in the Candy PDF, in document order."""
+ """Every priced line in the Candy PDF, in document order.
+
+ Each line also carries `sec`: the label of the SUMMARY section it sits
+ under, and `path`: every heading above it.
+
+ The ancestry is not decoration. Candy RESTARTS its numbering inside each
+ schedule, so Blueberry has a section "1" in Schedule A, another "1" under
+ Schedule B, and a "4" under a third schedule. Keying anything on the bare
+ number therefore merges unrelated sections - which is how the reconciliation
+ silently lost 6 of Blueberry's 8 sections and left a check sheet that
+ implied the untested ones were fine.
+
+ `sec` tracks the DIRECT child of a schedule, not the deepest heading: a
+ summary sheet lists "2 ROADS" and never its children "2.1 EARTHWORKS", so
+ recording the deepest one would put the line in a section the summary has
+ no row for and match nothing.
+ """
out: list[dict] = []
- def walk(node: dict) -> None:
+ def walk(node: dict, path: list[str], sec: str, top: bool) -> None:
kind = node.get("kind")
+ if kind == "root":
+ for c in node.get("children", []):
+ walk(c, path, sec, True)
+ return
if kind == "line":
if node.get("qty") is None and node.get("rate") is None:
return
@@ -642,12 +667,27 @@ def source_lines(tree: dict, *, with_prices: bool = True) -> list[dict]:
"amount": node.get("amount"),
"rate_only": bool(node.get("rate_only")),
"page": node.get("page", 0),
+ "sec": sec,
+ "path": " > ".join(path),
})
return
+ here, new_sec, new_top = path, sec, False
+ if kind in ("section", "schedule", "group"):
+ label = f"{node.get('number','')} {node.get('title','')}".strip()
+ here = path + [label]
+ if top:
+ # This node is itself an entry the summary sheet lists.
+ new_sec = label
+ # Only a SCHEDULE promotes its children to summary entries. A
+ # top-level group contributes its own label alone - its children
+ # ("1.1 Fixed-Charge Items") are detail the summary never names,
+ # and recording them as the section is what left 26 phantom
+ # sections and matched none of the 8 that exist.
+ new_top = (kind == "schedule")
for c in node.get("children", []):
- walk(c)
+ walk(c, here, new_sec, new_top)
- walk(tree)
+ walk(tree, [], "", False)
return out
@@ -912,6 +952,22 @@ def populate(consultant_xlsx: str, tree: dict, out_path: str, *,
_write_totals(wb, layouts, tree, res)
_restore_formulas(original, wb, res)
+
+ # A "MEGA CHECK" sheet at the front: every section the PDF priced against
+ # what this workbook now says, plus the rows that still need a person. The
+ # person at the spreadsheet works from the totals, so the totals are where
+ # the punch list belongs.
+ try:
+ from boq_engine.reconcile import reconcile, write_check_sheet
+ rec = reconcile(tree, wb, res, summary_values(tree))
+ n_bad, n_rows = write_check_sheet(wb, rec, out_name=out_path)
+ res.layout_notes.append(
+ f"MEGA CHECK written: {len(rec.checks)} of "
+ f"{len(rec.checks) + len(rec.unmatched)} section(s) compared, "
+ f"{n_bad} do not balance, {n_rows} line(s) flagged for a person")
+ except Exception as exc: # never block on this
+ res.layout_notes.append(f"MEGA CHECK not written: {exc}")
+
wb.save(out_path)
return res
@@ -953,7 +1009,8 @@ def _fill_pool(wb, layouts: dict[str, Layout], pool: list[Slot],
f"{ln['number']} {ln['desc']}"[:70],
f"no row in the template for this line (p{ln['page']})",
source=f"{ln['qty']} {ln['unit']} @ {ln['rate']}",
- severity="decision", index=i)
+ severity="decision", index=i,
+ section=ln.get("sec", ""))
continue
name = slot._sheet
try:
@@ -970,7 +1027,8 @@ def _write_one(wb, name: str, lay: Layout, ln: dict,
f"{ln['number']} {ln['desc']}"[:70],
f"no row in '{name}' for this line (p{ln['page']})",
source=f"{ln['qty']} {ln['unit']} @ {ln['rate']}",
- severity="decision")
+ severity="decision",
+ section=ln.get("sec", ""))
return
ws = wb[name]
diff --git a/boq_engine/verify.py b/boq_engine/verify.py
index cd31320..4172d2c 100644
--- a/boq_engine/verify.py
+++ b/boq_engine/verify.py
@@ -64,8 +64,30 @@ class Verdict:
@property
def blocked(self) -> bool:
+ """Damage to the consultant's own workbook. Never acceptable."""
return bool(self.failures)
+ @property
+ def needs_review(self) -> bool:
+ """Work that is incomplete or possibly misplaced, but not damaging.
+
+ This is a different thing from damage. A rate on the wrong line, or a
+ line with no row at all, is a mistake; a missing formula is vandalism.
+ The first is handed over with a punch list, because that is what the
+ person at the spreadsheet is there to resolve. The second removes the
+ download.
+
+ BOTH the misplaced and the unplaced checks count. A line that was never
+ placed leaves no trace in the workbook at all — nothing looks wrong
+ until a total is short — so it is the case most worth shouting about,
+ and the gate used to stay quiet on it.
+ """
+ return any(
+ not c.passed and c.name.startswith(
+ ("every rate sits", "every source line"))
+ for c in self.checks
+ )
+
def as_dict(self) -> dict:
return {
"blocked": self.blocked,
@@ -258,24 +280,38 @@ def verify(src_xlsx: str, out_xlsx: str, tree: dict, result,
except ValueError:
layouts[name] = None
- # read back exactly the cells populate() says it wrote
- spot, wrong = 0, []
- for sheet, row, col, value in getattr(result, "written_cells", []):
+ # Read back exactly the cells populate() says it wrote, and compare the
+ # VALUE - not merely that something is there.
+ #
+ # Checking "not empty" and calling it "every write landed" let a rate
+ # overwritten with a wrong number pass silently, which is the whole class
+ # of damage this gate exists to catch. It was proved by planting one: the
+ # verifier stayed green on a workbook we had deliberately corrupted.
+ cells = getattr(result, "written_cells", [])
+ wrong = []
+ for sheet, row, col, value in cells:
if sheet not in out.sheetnames:
- wrong.append(f"{sheet}!{row} (sheet gone)")
+ wrong.append(f"{sheet}!r{row} (sheet gone)")
continue
- spot += 1
- if spot > 600:
- break
got = out[sheet].cell(row, col).value
if isinstance(value, str) and value.startswith("="):
if not (isinstance(got, str) and got.startswith("=")):
wrong.append(f"{sheet}!r{row}c{col} formula missing")
elif got is None:
wrong.append(f"{sheet}!r{row}c{col} empty")
+ elif isinstance(value, (int, float)) and not isinstance(value, bool):
+ # the money columns. exact enough to catch a wrong figure, loose
+ # enough for float round-tripping through the file
+ if not isinstance(got, (int, float)) or isinstance(got, bool):
+ wrong.append(f"{sheet}!r{row}c{col} is {got!r}, wrote {value!r}")
+ elif abs(float(got) - float(value)) > 0.005:
+ wrong.append(f"{sheet}!r{row}c{col} is {got!r}, wrote {value!r}")
+ elif isinstance(value, str) and isinstance(got, str):
+ if _norm_txt(value) != _norm_txt(got):
+ wrong.append(f"{sheet}!r{row}c{col} is {got!r}, wrote {value!r}")
v.add("every write landed", not wrong,
- f"read back {spot} of {len(getattr(result, 'written_cells', []))} "
- f"written cells; {len(wrong)} wrong"
+ f"read back all {len(cells)} written cells and compared values; "
+ f"{len(wrong)} wrong"
+ (f": {wrong[:4]}" if wrong else ""),
fatal=True)
@@ -287,7 +323,13 @@ def verify(src_xlsx: str, out_xlsx: str, tree: dict, result,
f"placed on it; {len(misplaced)} row(s) disagree")
for sh, rw, want, why in misplaced[:6]:
detail += f"\n {sh} r{rw}: wanted {want!r} but the row says {why}"
- v.add("every rate sits on the line it belongs to", not misplaced, detail)
+ # REVIEW, not fatal. Damage to the consultant's own file blocks the
+ # download; a line that landed on the wrong row does not, because the
+ # MEGA CHECK sheet says which sections are out and which rows to look at,
+ # and the person receiving the file works from the totals. Blocking here
+ # would mean no file at all for a 10-minute fix.
+ v.add("every rate sits on the line it belongs to", not misplaced, detail,
+ fatal=False)
# ---- 5. COMPLETE: every source line accounted for ----------------------
unmapped = [f for f in result.findings if f.kind == "unmapped"]