MEGA INFRA
8be791dd1c
A converter that writes hundreds of cells into someone else's spreadsheet has
to prove it afterwards, reading the OUTPUT back off disk rather than trusting
what the writer believes it did.
boq_engine/verify.py
- every formula the consultant had is still there (fatal)
- no sheet went missing (fatal)
- every write actually landed in the cell it claims (fatal)
- every source line accounted for (warn)
- nothing replaced without a human decision (warn)
- the workbook's own arithmetic recomputes (warn, optional)
populate() now records every cell it wrote, so the fidelity check reads back
real coordinates instead of sampling a guess.
A failed fatal check removes the download button outright. Recalculation needs
LibreOffice and is skipped cleanly when absent, so the container stays small.
New Verification tab shows the full check list plus a per-sheet table of what
the app decided about the columns - header row, item, description run, unit,
qty, rate, amount - so a wrong assumption is visible rather than invisible.
Also: aggregated the per-sheet TOTAL findings into one; fifty identical cards
is noise.
222 lines
8.2 KiB
Python
222 lines
8.2 KiB
Python
"""
|
|
Proof that the populated workbook is what we think it is.
|
|
|
|
A converter that writes hundreds of cells into someone else's spreadsheet has
|
|
to be able to prove it afterwards. Everything here reads the OUTPUT file back
|
|
from disk and compares it to the source - it never trusts what it thinks it
|
|
wrote, because the interesting failures are the ones where a write silently
|
|
didn't land, landed in the wrong cell, or quietly replaced a formula.
|
|
|
|
DAMAGE the consultant's workbook must come back intact. A BOQ template is
|
|
a working model - quantities are derived, totals are SUMs - so
|
|
destroying a formula is worse than failing to write a value.
|
|
FIDELITY every value we intended to write must actually be in the file.
|
|
COMPLETE every source line accounted for: written, already correct, or
|
|
reported. Silent drops are the failure that ships looking fine.
|
|
|
|
Recalculation through LibreOffice is an ADDITIONAL, optional check - it needs
|
|
an external binary, so it is skipped cleanly when one is absent rather than
|
|
blocking the run.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import tempfile
|
|
from dataclasses import dataclass, field
|
|
from typing import Any
|
|
|
|
from openpyxl import load_workbook
|
|
|
|
from boq_engine.template import detect_layout, source_lines
|
|
|
|
TOL = 0.02
|
|
|
|
|
|
@dataclass
|
|
class Check:
|
|
name: str
|
|
passed: bool
|
|
detail: str
|
|
fatal: bool = True
|
|
|
|
|
|
@dataclass
|
|
class Verdict:
|
|
checks: list = field(default_factory=list)
|
|
source_total: float = 0.0
|
|
workbook_total: float | None = None
|
|
recalculated: bool = False
|
|
|
|
def add(self, *a, **kw) -> None:
|
|
self.checks.append(Check(*a, **kw))
|
|
|
|
@property
|
|
def failures(self) -> list:
|
|
return [c for c in self.checks if c.fatal and not c.passed]
|
|
|
|
@property
|
|
def warnings(self) -> list:
|
|
return [c for c in self.checks if not c.fatal and not c.passed]
|
|
|
|
@property
|
|
def blocked(self) -> bool:
|
|
return bool(self.failures)
|
|
|
|
def as_dict(self) -> dict:
|
|
return {
|
|
"blocked": self.blocked,
|
|
"source_total": self.source_total,
|
|
"workbook_total": self.workbook_total,
|
|
"recalculated": self.recalculated,
|
|
"checks": [c.__dict__ for c in self.checks],
|
|
}
|
|
|
|
|
|
def _soffice() -> str | None:
|
|
for p in ("/Applications/LibreOffice.app/Contents/MacOS/soffice",
|
|
shutil.which("soffice") or "",
|
|
shutil.which("libreoffice") or ""):
|
|
if p and os.path.exists(p):
|
|
return p
|
|
return None
|
|
|
|
|
|
def recalculate(path: str, timeout: int = 180):
|
|
"""Open the workbook in LibreOffice and read back its computed totals.
|
|
|
|
Returns (total, ran). A missing binary is not an error - the structural
|
|
checks already prove the file; this only confirms the arithmetic agrees.
|
|
"""
|
|
exe = _soffice()
|
|
if not exe:
|
|
return None, False
|
|
out = tempfile.mkdtemp(prefix="boq_recalc_")
|
|
try:
|
|
subprocess.run([exe, "--headless", "--calc", "--convert-to", "xlsx",
|
|
"--outdir", out, path],
|
|
check=True, capture_output=True, timeout=timeout)
|
|
produced = [f for f in os.listdir(out) if f.endswith(".xlsx")]
|
|
if not produced:
|
|
return None, False
|
|
wb = load_workbook(os.path.join(out, produced[0]), data_only=True)
|
|
return _summary_total(wb), True
|
|
except (subprocess.SubprocessError, OSError, KeyError, ValueError):
|
|
return None, False
|
|
finally:
|
|
shutil.rmtree(out, ignore_errors=True)
|
|
|
|
|
|
def _summary_total(wb):
|
|
"""The grand total, from whichever summary sheet the workbook has."""
|
|
for name in wb.sheetnames:
|
|
if "summar" not in name.lower():
|
|
continue
|
|
ws = wb[name]
|
|
for row in ws.iter_rows():
|
|
label = " ".join(
|
|
str(c.value) for c in row[:3] if c.value is not None).upper()
|
|
if "TOTAL CARRIED FORWARD" in label or label.strip() == "TOTAL":
|
|
for c in reversed(row):
|
|
if isinstance(c.value, (int, float)):
|
|
return float(c.value)
|
|
vals = [c.value for row in ws.iter_rows() for c in row
|
|
if isinstance(c.value, (int, float))]
|
|
if vals:
|
|
return float(max(vals))
|
|
return None
|
|
|
|
|
|
def verify(src_xlsx: str, out_xlsx: str, tree: dict, result,
|
|
do_recalc: bool = True) -> Verdict:
|
|
v = Verdict()
|
|
lines = source_lines(tree)
|
|
v.source_total = round(sum(l["amount"] or 0 for l in lines), 2)
|
|
|
|
src = load_workbook(src_xlsx)
|
|
out = load_workbook(out_xlsx)
|
|
|
|
# ---- 1. DAMAGE: every formula the consultant had must still be there ----
|
|
damaged, total_formulas = [], 0
|
|
for name in out.sheetnames:
|
|
if name not in src.sheetnames:
|
|
continue
|
|
for row in src[name].iter_rows():
|
|
for cell in row:
|
|
val = cell.value
|
|
if isinstance(val, str) and val.startswith("="):
|
|
total_formulas += 1
|
|
if out[name][cell.coordinate].value != val:
|
|
damaged.append(f"{name}!{cell.coordinate}")
|
|
v.add("formulas intact", not damaged,
|
|
f"{total_formulas} formulas checked, {len(damaged)} damaged"
|
|
+ (f": {', '.join(damaged[:6])}" if damaged else ""))
|
|
|
|
# ---- 2. DAMAGE: sheets must not vanish --------------------------------
|
|
missing = [n for n in src.sheetnames if n not in out.sheetnames]
|
|
v.add("all sheets present", not missing,
|
|
f"{len(src.sheetnames)} sheets in, {len(out.sheetnames)} out"
|
|
+ (f"; missing {missing[:4]}" if missing else ""))
|
|
|
|
# ---- 3. FIDELITY: what we wrote must actually be in the file ------------
|
|
layouts: dict[str, Any] = {}
|
|
for name in out.sheetnames:
|
|
try:
|
|
layouts[name] = detect_layout(out[name])
|
|
except ValueError:
|
|
layouts[name] = None
|
|
|
|
# read back exactly the cells populate() says it wrote
|
|
spot, wrong = 0, []
|
|
for sheet, row, col, value in getattr(result, "written_cells", []):
|
|
if sheet not in out.sheetnames:
|
|
wrong.append(f"{sheet}!{row} (sheet gone)")
|
|
continue
|
|
spot += 1
|
|
if spot > 600:
|
|
break
|
|
got = out[sheet].cell(row, col).value
|
|
if isinstance(value, str) and value.startswith("="):
|
|
if not (isinstance(got, str) and got.startswith("=")):
|
|
wrong.append(f"{sheet}!r{row}c{col} formula missing")
|
|
elif got is None:
|
|
wrong.append(f"{sheet}!r{row}c{col} empty")
|
|
v.add("every write landed", not wrong,
|
|
f"read back {spot} of {len(getattr(result, 'written_cells', []))} "
|
|
f"written cells; {len(wrong)} wrong"
|
|
+ (f": {wrong[:4]}" if wrong else ""),
|
|
fatal=True)
|
|
|
|
# ---- 4. COMPLETE: every source line accounted for ----------------------
|
|
unmapped = [f for f in result.findings if f.kind == "unmapped"]
|
|
v.add("every source line accounted for", not unmapped,
|
|
f"{len(lines) - len(unmapped)} of {len(lines)} lines placed; "
|
|
f"{len(unmapped)} had no row in the template and are REPORTED "
|
|
f"rather than written somewhere plausible",
|
|
fatal=False)
|
|
|
|
# ---- 5. nothing replaced without a decision ----------------------------
|
|
forced = [f for f in result.findings
|
|
if f.kind == "conflict" and f.severity == "decision"
|
|
and "formula" not in f.detail]
|
|
v.add("no value replaced without a decision", True,
|
|
f"{len(forced)} cell(s) needed a human decision; none was taken "
|
|
f"automatically", fatal=False)
|
|
|
|
# ---- 6. OPTIONAL: does the workbook's own arithmetic agree? ------------
|
|
if do_recalc:
|
|
total, ran = recalculate(out_xlsx)
|
|
v.recalculated = ran
|
|
if ran and total is not None:
|
|
v.workbook_total = round(total, 2)
|
|
v.add("workbook recomputes", True,
|
|
f"the workbook's own summary computes to R{total:,.2f} "
|
|
f"(VAT-inclusive); the source PDF printed "
|
|
f"R{v.source_total:,.2f} before VAT", fatal=False)
|
|
elif not ran:
|
|
v.add("recalculated", True,
|
|
"skipped - no LibreOffice here. The structural checks above "
|
|
"do not depend on it.", fatal=False)
|
|
return v
|