MEGA-BOQ-WEBAPP/boq_engine/template.py

"""
Populate a CONSULTANT's BOQ workbook from our own Candy PDF export.

The direction matters and is easy to get backwards: Candy is *our* system, so
the PDF is the source of truth and is always the same shape. The consultant's
Excel is *theirs*, and changes from consultant to consultant. So the hard part
is not reading the document - it is writing into a workbook we did not design
without destroying what is already in it.

Three jobs:

  1. LAYOUT DETECTION
     Find the header row and work out which column is ITEM / DESCRIPTION /
     UNIT / QTY / RATE / AMOUNT. A consultant's PAYMENT column (SABS / PSA
     clause references) is detected too and then never touched.

  2. MAPPING
     Line the PDF's lines up with the workbook's rows. The two documents are
     near 1:1 but not identical: the workbook truncates descriptions, inserts
     spacer rows and adds "TOTAL CARRIED FORWARD" blocks, while the PDF wraps
     descriptions across several lines. A sequence alignment over
     (item number, description similarity) survives all of that.

  3. PROOF
     A population pass that cannot be checked is a guess. Every source line is
     accounted for - written, already-matching, or reported as an exception -
     and section totals are written as live formulas that must reproduce the
     PDF's own printed subtotal.
"""

from __future__ import annotations

import re
import shutil
from dataclasses import dataclass, field
from difflib import SequenceMatcher
from typing import Any

from openpyxl import load_workbook
from openpyxl.utils import get_column_letter

# --------------------------------------------------------------------------
# header synonyms
# --------------------------------------------------------------------------
HEADER_PATTERNS: dict[str, tuple[str, ...]] = {
    "item": ("item", "item no", "item no.", "item number", "no", "no."),
    "description": ("description", "bill description", "desc", "particulars",
                    "description of work"),
    "unit": ("unit", "units", "uom"),
    "qty": ("qty", "quantity", "bill quantity", "quantities"),
    "rate": ("rate", "unit rate", "unit price", "price"),
    "amount": ("amount", "total", "value", "sum"),
    "payment": ("payment", "payment clause", "clause", "spec", "specification"),
}

TOTAL_RE = re.compile(r"^\s*total\b", re.I)
BROUGHT_RE = re.compile(r"brought\s+forward", re.I)
SUMMARY_RE = re.compile(r"to\s+(combined\s+)?summary", re.I)


def _norm(s: Any) -> str:
    return re.sub(r"\s+", " ", str(s or "")).strip().lower()


def _cellsafe(v: Any) -> str:
    """Normalise a cell value to something comparable across both documents."""
    s = _norm(v)
    s = s.replace("ℓ", "l").replace("³", "3").replace("²", "2")
    s = re.sub(r"[^a-z0-9./-]+", " ", s)
    return re.sub(r"\s+", " ", s).strip()


# --------------------------------------------------------------------------
# layout
# --------------------------------------------------------------------------
@dataclass
class Layout:
    sheet: str
    header_row: int
    cols: dict[str, int] = field(default_factory=dict)
    data_rows: list[int] = field(default_factory=list)
    total_rows: list[int] = field(default_factory=list)
    notes: list[str] = field(default_factory=list)
    # The description is not always one column. It occupies whatever sits
    # between ITEM and UNIT, and on the SANS 1200 template it moves *within*
    # a single sheet - the label "a)" lands in one column and the text in the
    # next. So the description is a RUN, and a row's text is the join of them.
    desc_cols: list[int] = field(default_factory=list)

    def col(self, name: str) -> int | None:
        return self.cols.get(name)


def _score_header_row(ws, r: int, ncols: int, span: int = 1) -> dict[str, int]:
    """Map columns to names using a header that may wrap over `span` rows.

    Templates split a label across two lines to fit a narrow column -
    "QUAN-" over "TITY". Reading either row alone finds no QUANTITY at all, so
    the quantity column is simply missed and never written. Joining the rows
    recovers the word.
    """
    cols: dict[str, int] = {}
    for c in range(1, ncols + 1):
        parts = [str(ws.cell(r + k, c).value or "").strip()
                 for k in range(span)]
        # A hyphen here is a column-wrap artefact ("QUAN-" / "TITY"), never part
        # of the word. Left in, the joined label reads "quan-tity" and matches
        # nothing - which is how the quantity column went missing on 49 sheets.
        label = _cellsafe("".join(parts).replace("-", ""))
        if not label:
            continue
        for name, pats in HEADER_PATTERNS.items():
            if name in cols:
                continue
            if label in pats or any(p in label for p in pats):
                cols[name] = c
                break
    return cols


def detect_layout(ws) -> Layout:
    """Find the header row and map the columns.

    Each candidate row is scored twice - read alone, and joined with the row
    below it - because a wrapped label only reads correctly when the two halves
    are put back together.
    """
    ncols = min(ws.max_column, 30)
    cands: list[tuple[int, int, int, dict[str, int]]] = []
    for r in range(1, min(ws.max_row, 25) + 1):
        for span in (1, 2):
            if r + span - 1 > ws.max_row:
                continue
            cols = _score_header_row(ws, r, ncols, span)
            if len(cols) >= 4:
                cands.append((len(cols), r, span, cols))
    # most roles found wins; a tie goes to the LOWER row, because the genuine
    # header block is the last one before the data starts. Getting this wrong
    # cost the quantity column on 49 of 51 sheets.
    best = max(cands, key=lambda t: (t[0], t[1], t[2])) if cands else None

    if best is None:
        raise ValueError(
            f"Could not find a header row in sheet '{ws.title}' - "
            f"expected 4+ of {', '.join(HEADER_PATTERNS)} in the first 25 rows"
        )

    _, header_row, span, cols = best
    lay = Layout(sheet=ws.title, header_row=header_row, cols=cols)
    if span > 1:
        lay.notes.append(f"header wraps over rows {header_row}"
                         f"-{header_row + span - 1}")

    c_item = cols.get("item")
    c_desc = cols.get("description")
    c_qty = cols.get("qty")

    for r in range(header_row + span, ws.max_row + 1):
        a = ws.cell(r, c_item).value if c_item else None
        d = ws.cell(r, c_desc).value if c_desc else None
        q = ws.cell(r, c_qty).value if c_qty else None
        label = f"{a or ''} {d or ''}".strip()

        if label and TOTAL_RE.match(label):
            lay.total_rows.append(r)
            continue
        # a real data row: something in item, description or qty
        if any(x not in (None, "") for x in (a, d)) or q is not None:
            lay.data_rows.append(r)

    for name in ("item", "description", "unit", "qty", "rate", "amount"):
        if name not in cols:
            lay.notes.append(f"template has no '{name}' column - that value will not be written")
    if "payment" in cols:
        lay.notes.append("PAYMENT column detected and will be left untouched")

    _verify_against_data(ws, lay)
    _derive_desc_run(lay)
    return lay


def _derive_desc_run(lay: Layout) -> None:
    """Everything between the item column and the unit column is description."""
    lo = lay.cols.get("item")
    hi = lay.cols.get("unit")
    if not lo or not hi or hi <= lo:
        lay.desc_cols = [lay.cols["description"]] if "description" in lay.cols else []
        return
    run = [c for c in range(lo + 1, hi) if c not in lay.cols.values()]
    # plus the claimed description column even if it sits outside the run
    d = lay.cols.get("description")
    if d and d not in run and d not in (lo, hi):
        run.append(d)
    lay.desc_cols = sorted(set(run))
    if len(lay.desc_cols) > 1:
        lay.notes.append(
            f"description spans columns "
            f"{[get_column_letter(c) for c in lay.desc_cols]}")


def _verify_against_data(ws, lay: Layout) -> None:
    """Correct the header-derived column map using what the data actually holds.

    A header row is a claim; the cells below it are the fact. On the SANS 1200
    template the header advertises DESCRIPTION in column E while the text sits
    in D, and a wrong description column quietly wrecks every line match.
    """
    sample = lay.data_rows[3:60]
    if len(sample) < 5:
        return

    def longest_text(col: int) -> tuple[int, int]:
        """(count of substantial strings, longest) in a column over the sample."""
        n = longest = 0
        for r in sample:
            v = ws.cell(r, col).value
            if isinstance(v, str) and len(v.strip()) > 6:
                n += 1
                longest = max(longest, len(v.strip()))
        return n, longest

    # --- description: the header may be off by one --------------------------
    d = lay.cols.get("description")
    if d:
        have, _ = longest_text(d)
        best_col, best_score = d, 0.0
        for cand in (d - 1, d + 1):
            if cand < 1 or cand in lay.cols.values():
                continue
            cnt, longest = longest_text(cand)
            # a real description column: nearly every sampled row carries long
            # text. Score it, and only switch if it clearly beats the claim.
            score = (cnt / max(1, len(sample))) * (1 if longest > 25 else 0.3)
            if score > best_score:
                best_col, best_score = cand, score
        if best_col != d and best_score > 0.6 and have < len(sample) * 0.2:
            lay.notes.append(
                f"header claims DESCRIPTION in col {d} but the text is in "
                f"col {best_col} - using {get_column_letter(best_col)}")
            lay.cols["description"] = best_col

    # --- unit: short, mostly-textual values sitting beside the quantity -----
    u = lay.cols.get("unit")
    if u:
        def unitish(col: int) -> int:
            n = 0
            for r in sample:
                v = ws.cell(r, col).value
                if isinstance(v, str) and 1 < len(v.strip()) < 12:
                    n += 1
            return n

        have = unitish(u)
        if have < len(sample) * 0.3:
            for cand in (u - 1, u + 1):
                if cand < 1 or cand in lay.cols.values():
                    continue
                if unitish(cand) >= len(sample) * 0.5:
                    lay.notes.append(
                        f"unit column shifted {u} -> {cand} to match the data")
                    lay.cols["unit"] = cand
                    break


# --------------------------------------------------------------------------
# matching
# --------------------------------------------------------------------------
@dataclass
class Slot:
    row: int
    number: str = ""
    desc: str = ""
    indent: int = 0
    unit: Any = None
    qty: Any = None
    rate: Any = None
    amount: Any = None
    _sheet: str = ""

    @property
    def key(self) -> str:
        return _cellsafe(f"{self.number} {self.desc}")


def read_slots(ws, lay: Layout) -> list[Slot]:
    ci = lay.col("item")
    run = lay.desc_cols or ([lay.col("description")] if lay.col("description") else [])
    out: list[Slot] = []
    # A spreadsheet implies its hierarchy: once "110.01" is printed, the blank
    # cells below it still belong to 110.01. Without carrying that down, two
    # thirds of the rows read as unnumbered and nothing can match them.
    carried = ""
    for r in lay.data_rows:
        num = str(ws.cell(r, ci).value or "").strip() if ci else ""
        if num:
            carried = num
        else:
            num = carried
        parts = [str(ws.cell(r, c).value or "").strip() for c in run]
        desc = " ".join(p for p in parts if p).strip()
        if not num and not desc:
            continue
        # indent level from the leading whitespace the consultant preserved
        raw = next((str(ws.cell(r, c).value or "") for c in run
                    if str(ws.cell(r, c).value or "") !=
                    str(ws.cell(r, c).value or "").lstrip()), "")
        indent = len(raw) - len(raw.lstrip()) if raw else 0
        out.append(Slot(row=r, number=num, desc=desc, indent=indent))
    return out


def _sim(a: Slot, num: str, desc: str) -> float:
    """How well does a source line match a template slot?

    Item number is the strong signal, but most of a BOQ is unnumbered a)/b)/c)
    sub-lines where the number is blank on BOTH sides. Scoring those as "no
    evidence" pushed every sub-line under the threshold, so description has to
    carry full weight when neither side has a number.
    """
    tnum, snum = _cellsafe(a.number), _cellsafe(num)
    tdesc = f"{tnum} {_cellsafe(a.desc)}".strip()
    sdesc = f"{snum} {_cellsafe(desc)}".strip()

    if tnum and snum:
        if tnum == snum:
            num_score = 1.0
        elif tnum in snum or snum in tnum:
            num_score = 0.75
        else:
            num_score = 0.0      # two different numbers: real evidence against
    else:
        # A missing number is NEUTRAL, not a mismatch. Most of a BOQ is
        # unnumbered a)/b) sub-lines, and a template carries a number down into
        # them - scoring that as disagreement made every sub-line unmatchable.
        num_score = 0.5

    d = SequenceMatcher(None, tdesc, sdesc).ratio()
    return round(num_score * 0.6 + d * 0.4, 4)


def align(slots: list[Slot], lines: list[dict], threshold: float = 0.45
          ) -> list[tuple[dict, Slot | None, float]]:
    """Sequence-align source lines onto template slots.

    A Needleman-Wunsch style DP rather than a greedy pass: a single
    mis-detected row would otherwise shift every line after it, which is
    exactly the silent-corruption failure this whole tool exists to prevent.
    """
    n, m = len(lines), len(slots)
    GAP = -0.35

    def sc(i: int, j: int) -> float:
        ln = lines[i]
        return _sim(slots[j], ln.get("number", ""), ln.get("desc", ""))

    dp = [[0.0] * (m + 1) for _ in range(n + 1)]
    bt = [[0] * (m + 1) for _ in range(n + 1)]
    for i in range(1, n + 1):
        dp[i][0] = dp[i - 1][0] + GAP
        bt[i][0] = 1
    for j in range(1, m + 1):
        dp[0][j] = dp[0][j - 1] + GAP
        bt[0][j] = 2
    for i in range(1, n + 1):
        for j in range(1, m + 1):
            diag = dp[i - 1][j - 1] + sc(i - 1, j - 1)
            up = dp[i - 1][j] + GAP
            left = dp[i][j - 1] + GAP
            best = max(diag, up, left)
            dp[i][j] = best
            bt[i][j] = 0 if best == diag else (1 if best == up else 2)

    out: list[tuple[dict, Slot | None, float]] = []
    i, j = n, m
    while i > 0 or j > 0:
        mv = bt[i][j]
        if mv == 0 and i > 0 and j > 0:
            s = sc(i - 1, j - 1)
            out.append((lines[i - 1], slots[j - 1] if s >= threshold else None, s))
            i, j = i - 1, j - 1
        elif mv == 1 and i > 0:
            out.append((lines[i - 1], None, 0.0))
            i -= 1
        else:
            j -= 1
    out.reverse()
    return out


# --------------------------------------------------------------------------
# the population pass
# --------------------------------------------------------------------------
@dataclass
class Finding:
    kind: str          # 'written' | 'kept' | 'conflict' | 'unmapped' | 'total' | 'info'
    sheet: str
    row: int
    label: str
    detail: str
    source: str = ""
    # How much a human needs to care. This is data, not presentation - the UI
    # maps it to a colour, so the classification lives here where it can be
    # asserted in tests.
    #   'decision'     the two documents genuinely disagree; a person must pick
    #   'source_gap'   OUR pdf has no figure here, theirs is better
    #   'source_defect' OUR pdf printed something corrupt
    #   'note'         worth knowing, no action implied
    severity: str = "note"
    # For a conflict the UI needs enough to offer a real choice: which cell,
    # what is in it now, and what the source wants to put there.
    field: str = ""                 # 'qty' | 'rate' | 'unit'
    template_value: Any = None      # what the consultant's file holds today
    source_value: Any = None        # what our Candy PDF holds


class _FormulaProtected(Exception):
    """Raised internally when a cell must not be overwritten."""


@dataclass
class Result:
    written: int = 0
    kept: int = 0
    findings: list[Finding] = field(default_factory=list)
    sheets: list[str] = field(default_factory=list)
    layout_notes: list[str] = field(default_factory=list)

    def add(self, *a, **kw) -> None:
        self.findings.append(Finding(*a, **kw))

    @property
    def problems(self) -> list[Finding]:
        return [f for f in self.findings
                if f.kind in ("conflict", "unmapped", "info")]


# --------------------------------------------------------------------------
# source-side flattening: PDF tree -> the list we try to place
# --------------------------------------------------------------------------
def source_lines(tree: dict, *, with_prices: bool = True) -> list[dict]:
    """Every priced line in the Candy PDF, in document order."""
    out: list[dict] = []

    def walk(node: dict) -> None:
        kind = node.get("kind")
        if kind == "line":
            if node.get("qty") is None and node.get("rate") is None:
                return
            # a row of dots is Candy's blank-for-the-tenderer placeholder, not
            # a priced line
            if re.fullmatch(r"[.\s]+", (node.get("desc") or "").strip()):
                return
            out.append({
                "number": node.get("number", "") or "",
                "desc": (node.get("desc") or "").strip(),
                "unit": (node.get("unit") or "").strip(),
                "qty": node.get("qty"),
                "rate": node.get("rate"),
                "amount": node.get("amount"),
                "rate_only": bool(node.get("rate_only")),
                "page": node.get("page", 0),
            })
            return
        for c in node.get("children", []):
            walk(c)

    walk(tree)
    return out


def section_totals(tree: dict) -> dict[str, float]:
    """Total of the priced lines beneath each section / schedule.

    Deliberately a LEAF roll-up rather than the printed subtotals: the source
    prints totals that swallow a neighbouring block (1.6 Temporary Works
    includes 1.5 Protection of Services), so summing printed subtotals
    double counts. Summing leaves reproduces the printed grand total exactly.
    """
    totals: dict[str, float] = {}

    def leaf_sum(node: dict) -> float:
        if node.get("kind") == "line":
            if node.get("rate_only") or node.get("amount") is None:
                return 0.0
            return float(node["amount"])
        return sum(leaf_sum(c) for c in node.get("children", []))

    def walk(node: dict) -> None:
        if node.get("kind") in ("section", "schedule", "root"):
            if node.get("kind") in ("section", "schedule"):
                label = f"{node.get('number','')} {node.get('title','')}".strip()
                totals[label] = round(leaf_sum(node), 2)
        for c in node.get("children", []):
            walk(c)

    walk(tree)
    return totals


def _norm_unit(u: str) -> str:
    s = _norm(u).replace("ℓ", "l").replace("³", "3").replace("²", "2")
    return s.rstrip(".")


UNIT_ALIASES = {
    "l": "l", "litre": "l", "litres": "l", "l": "l",
    "m2": "m2", "m 2": "m2", "sqm": "m2",
    "m3": "m3", "m 3": "m3", "cbm": "m3",
    "no": "no", "no.": "no", "nr": "no", "number": "no",
    "sum": "sum", "sum.": "sum", "ls": "sum",
    "p.sum": "psum", "psum": "psum", "ps.sum": "psum", "pc.sum": "pcsum",
    "month": "month", "months": "month", "mth": "month",
    "t": "t", "tonne": "t", "tonnes": "t",
    "m": "m", "m ": "m",
    "%": "%", "pct": "%", "?": "?", "": "",
}


def units_compatible(a: str, b: str) -> bool:
    na, nb = _norm_unit(a), _norm_unit(b)
    if na == nb:
        return True
    ca, cb = UNIT_ALIASES.get(na), UNIT_ALIASES.get(nb)
    return ca is not None and ca == cb


def _section_of(number: str) -> str:
    """The leading segment of a reference: "110.01.01" -> "110".

    SANS 1200 sheets are named for the section they cover, so this is what
    locates a sheet. It is NOT a unique key on its own - the Candy PDF restarts
    ".01" under every repeated "110...." heading, so "110.01" names three
    different lines in one document.
    """
    n = _cellsafe(number)
    return n.split(".")[0] if n else ""


def _relevant_pool(lines: list[dict],
                   slots_by_sheet: dict[str, list[Slot]]
                   ) -> tuple[list[Slot], dict[str, int]]:
    """The slots worth aligning against, in workbook order.

    Aligning against every sheet in the workbook looks fine on a 3-sheet
    template and falls apart on a 53-sheet one: with 1314 candidate rows for
    226 lines, the aligner starts cherry-picking scattered high scores instead
    of finding the one coherent run. So the pool is cut down to the sheets that
    actually own a section named in the source, which leaves 12 candidates
    instead of 1314 and lets the sequence aligner work as designed.
    """
    wanted = {_section_of(ln.get("number", "")) for ln in lines}
    wanted.discard("")

    pool: list[Slot] = []
    counts: dict[str, int] = {}
    for sheet, slots in slots_by_sheet.items():
        secs = {_section_of(sl.number) for sl in slots}
        secs.discard("")
        if wanted and not (secs & wanted):
            continue
        pool.extend(slots)
        counts[sheet] = len(slots)
    return pool, counts


def populate(consultant_xlsx: str, tree: dict, out_path: str, *,
             sheet_hint: str | None = None,
             overrides: dict | None = None) -> Result:
    """Write our Candy BOQ into a consultant's workbook, in place-safe fashion.

    Lines are routed to a sheet by item number, then aligned WITHIN that sheet.
    Nothing is ever written over a value the consultant already had, unless
    they explicitly asked for it via `overrides`.
    """
    overrides = overrides or {}
    res = Result()
    original = load_workbook(consultant_xlsx)   # pristine copy for comparison
    wb = load_workbook(consultant_xlsx)          # formulas preserved
    lines = source_lines(tree)
    if not lines:
        raise ValueError("no priced lines found in the source tree")

    names = [sheet_hint] if sheet_hint else list(wb.sheetnames)
    layouts: dict[str, Layout] = {}
    slots_by_sheet: dict[str, list[Slot]] = {}

    for name in names:
        if name not in wb.sheetnames:
            continue
        ws = wb[name]
        try:
            lay = detect_layout(ws)
        except ValueError as exc:
            res.add("info", name, 0, "layout", str(exc))
            continue
        layouts[name] = lay
        res.layout_notes.extend(f"[{name}] {n}" for n in lay.notes)
        slots = read_slots(ws, lay)
        for sl in slots:
            sl._sheet = name
        slots_by_sheet[name] = slots

    if not slots_by_sheet:
        raise ValueError("no BOQ-shaped sheet found in the workbook")

    pool, counts = _relevant_pool(lines, slots_by_sheet)
    res.layout_notes.append(
        f"aligned against {len(pool)} candidate rows on "
        f"{len(counts)} of {len(slots_by_sheet)} sheets")
    if not pool:
        raise ValueError("no template sheet shares a section with the source BOQ")

    # one sequential alignment over the whole pool, exactly as the aligner is
    # designed to work: the document order and the workbook order agree, and a
    # single mis-scored row no longer shifts the rest
    lay_of = {name: lay for name, lay in layouts.items()}
    _fill_pool(wb, lay_of, pool, lines, overrides, res)

    _write_totals(wb, layouts, tree, res)
    _restore_formulas(original, wb, res)
    wb.save(out_path)
    return res


def _restore_formulas(original, out, res: Result) -> int:
    """Put back any formula the write pass disturbed. Returns how many.

    A guard on each write is the right first line of defence, but there are a
    dozen write paths and a new one will be added eventually. This is the
    backstop that makes the promise checkable rather than merely intended: if
    the consultant's workbook had a formula in a cell, it still has it in the
    output. Everything else is rebuilt around that.
    """
    restored = 0
    for name in out.sheetnames:
        if name not in original.sheetnames:
            continue
        src, dst = original[name], out[name]
        for row in src.iter_rows():
            for cell in row:
                v = cell.value
                if isinstance(v, str) and v.startswith("="):
                    if dst[cell.coordinate].value != v:
                        dst[cell.coordinate] = v
                        restored += 1
    if restored:
        res.add("info", "-", 0, "formulas restored",
                f"{restored} cell(s) held a formula in your workbook and have "
                f"been put back exactly as they were - your model is intact")
    return restored


def _fill_pool(wb, layouts: dict[str, Layout], pool: list[Slot],
               lines: list[dict], overrides: dict, res: Result) -> None:
    """Align the whole source against the whole candidate pool, and write."""
    pairs = align(pool, lines)
    for ln, slot, score in pairs:
        if slot is None:
            res.add("unmapped", "-", 0,
                    f"{ln['number']} {ln['desc']}"[:70],
                    f"no row in the template for this line (p{ln['page']})",
                    source=f"{ln['qty']} {ln['unit']} @ {ln['rate']}",
                    severity="decision")
            continue
        name = slot._sheet
        try:
            _write_one(wb, name, layouts[name], ln, slot, overrides, res)
        except _FormulaProtected:
            continue          # already recorded; nothing was written


def _write_one(wb, name: str, lay: Layout, ln: dict,
               slot: Slot | None, overrides: dict, res: Result) -> None:
    """Write one aligned line. Returns rather than continuing."""
    if slot is None:
        res.add("unmapped", name, 0,
                f"{ln['number']} {ln['desc']}"[:70],
                f"no row in '{name}' for this line (p{ln['page']})",
                source=f"{ln['qty']} {ln['unit']} @ {ln['rate']}",
                severity="decision")
        return

    ws = wb[name]
    r = slot.row
    c_unit, c_qty = lay.col("unit"), lay.col("qty")
    c_rate, c_amt = lay.col("rate"), lay.col("amount")
    label = f"{slot.number} {slot.desc}"[:70]

    if c_qty and c_rate:
        ql, rl = get_column_letter(c_qty), get_column_letter(c_rate)
        amt_formula = (f'=IF(OR({ql}{{r}}="",{rl}{{r}}=""),"",'
                       f'{ql}{{r}}*{rl}{{r}})')
    else:
        amt_formula = None

    # --- unit ---------------------------------------------------------------
    if c_unit and ln["unit"]:
        cur = ws.cell(r, c_unit).value
        if cur in (None, ""):
            if _writable(ws, r, c_unit, res, name, label, "unit"):
                ws.cell(r, c_unit, ln["unit"])
        elif not units_compatible(cur, ln["unit"]):
            okey = (name, r, "unit")
            if okey in overrides:
                ws.cell(r, c_unit, overrides[okey])
                res.add("conflict", name, r, label,
                        f"unit: set to {overrides[okey]!r} by the consultant",
                        source=str(ln["unit"]), severity="resolved",
                        field="unit", template_value=cur,
                        source_value=ln["unit"])
            else:
                note = ("  -- Candy printed a corrupt unit character, so the "
                        "template's real unit was kept"
                        if _norm_unit(ln["unit"]) == "?" else
                        "  -- left as the template has it")
                res.add("conflict", name, r, label,
                        f"unit: template {cur!r} vs source {ln['unit']!r}{note}",
                        source=str(ln["unit"]), severity="source_defect",
                        field="unit", template_value=cur,
                        source_value=ln["unit"])

    # --- quantity ------------------------------------------------------------
    stop = False
    if c_qty and ln["qty"] is not None:
        cur = ws.cell(r, c_qty).value
        okey = (name, r, "qty")
        if okey in overrides:
            ws.cell(r, c_qty, overrides[okey])
            res.add("conflict", name, r, label,
                    f"quantity: set to {overrides[okey]} by the consultant",
                    source=str(ln["qty"]), severity="resolved", field="qty",
                    template_value=cur, source_value=ln["qty"])
        elif cur in (None, ""):
            if _writable(ws, r, c_qty, res, name, label, "qty"):
                ws.cell(r, c_qty, ln["qty"])
        elif isinstance(cur, (int, float)) and abs(float(cur) - ln["qty"]) > 1e-6:
            if ln["rate_only"] or ln["qty"] == 0:
                reason = ("source carries no quantity here (printed 'Rate "
                          "only'), so the template's own figure was kept")
                sev = "source_gap"
            else:
                reason = "the two documents disagree; NEITHER was overwritten"
                sev = "decision"
            res.add("conflict", name, r, label,
                    f"quantity: template {cur} vs source {ln['qty']} - {reason}",
                    source=str(ln["qty"]), severity=sev, field="qty",
                    template_value=cur, source_value=ln["qty"])
        else:
            res.kept += 1
            stop = True
            if c_rate and ln["rate"] is not None and not ln["rate_only"]:
                if ws.cell(r, c_rate).value in (None, ""):
                    if _writable(ws, r, c_rate, res, name, label, "rate"):
                        ws.cell(r, c_rate, ln["rate"])
                        if c_amt and amt_formula and _writable(
                                ws, r, c_amt, res, name, label, "amount"):
                            ws.cell(r, c_amt, amt_formula.format(r=r))
                        res.written += 1

    if stop:
        return

    # --- rate and amount -----------------------------------------------------
    res.written += 1
    if c_rate and ln["rate"] is not None:
        ws.cell(r, c_rate, ln["rate"])
        if c_amt:
            if ln["rate_only"]:
                ws.cell(r, c_amt, "Rate only")
                res.add("info", name, r, label,
                        "'Rate only' in source - rate written, amount as text",
                        source=str(ln["rate"]))
            elif amt_formula:
                ws.cell(r, c_amt, amt_formula.format(r=r))
    elif c_amt and ln["rate_only"]:
        ws.cell(r, c_amt, "Rate only")


def _writable(ws, r: int, col: int, res: Result, sheet: str, label: str,
              field: str) -> bool:
    """Refuse to overwrite a cell that holds a formula.

    A consultant's workbook is a working model, not a data dump. Quantities
    are derived ("=G35", "=CEILING.MATH(...)"), line amounts are computed, and
    section totals are SUMs over the rows above. Writing a literal over any of
    those destroys their work while looking like it succeeded.
    """
    cur = ws.cell(r, col).value
    if isinstance(cur, str) and cur.startswith("="):
        res.add("conflict", sheet, r, label,
                f"{field}: that cell holds a formula ({cur[:40]}) - your model "
                f"was left alone and the source value was not written",
                severity="decision", field=field, template_value=cur)
        return False
    return True


def _guard_formula(ws, r: int, col: int, res: Result, sheet: str,
                   label: str) -> None:
    """Refuse to clobber a cell that holds a formula.

    On a SANS 1200 template the quantities are DERIVED - "=G35",
    "=CEILING.MATH(414.065+302.817,10)" - from a Measurement sheet. Writing
    Candy's number over one of those destroys the engineer's model, and it
    looks like it worked.
    """
    cur = ws.cell(r, col).value
    if isinstance(cur, str) and cur.startswith("="):
        res.add("conflict", sheet, r, label,
                f"that cell holds a FORMULA ({cur[:44]}) - your model's value "
                f"was left in place, and the source quantity was not written",
                severity="decision", field="qty",
                template_value=cur, source_value=None)
        raise _FormulaProtected()


SUMMARY_REF = re.compile(
    r"^=\s*'?([^'!]+)'?!\$?([A-Z]{1,2})\$?(\d+)\s*$")


def summary_values(tree: dict) -> list[tuple[str, float]]:
    """The totals a COMBINED SUMMARY row expects, in document order.

    A consultant template's summary sheet points at specific cells such as
    ='SCHEDULE B - ROADS'!G66, and those cells are blank in the template - so
    without this the consultant opens the file and sees zero. The values come
    from the source in the same order the summary lists them: each schedule,
    then each of its sections.
    """
    def leaf(node: dict) -> float:
        if node.get("kind") == "line":
            if node.get("rate_only") or node.get("amount") is None:
                return 0.0
            return float(node["amount"])
        return sum(leaf(c) for c in node.get("children", []))

    out: list[tuple[str, float]] = []
    for branch in tree.get("children", []):
        if branch.get("kind") not in ("group", "schedule", "section"):
            continue
        if branch.get("kind") != "schedule":
            label = f"{branch.get('number','')} {branch.get('title','')}".strip()
            val = branch.get("subtotal")
            out.append((label, round(float(val), 2) if val is not None
                        else round(leaf(branch), 2)))
        for sec in branch.get("children", []):
            if sec.get("kind") != "section":
                continue
            label = f"{sec.get('number','')} {sec.get('title','')}".strip()
            val = sec.get("subtotal")
            out.append((label, round(float(val), 2) if val is not None
                        else round(leaf(sec), 2)))
    return out


def _write_totals(wb, layouts: dict[str, Layout], tree: dict,
                  res: Result) -> None:
    """Fill the cells the workbook's own summary depends on."""
    values = summary_values(tree)
    vi = 0

    # 1. the summary sheet, if there is one
    for name in wb.sheetnames:
        if "summary" not in name.lower():
            continue
        ws = wb[name]
        for r in range(1, min(ws.max_row, 120) + 1):
            for c in range(1, min(ws.max_column, 12) + 1):
                m = SUMMARY_REF.match(str(ws.cell(r, c).value or ""))
                if not m or vi >= len(values):
                    continue
                target_sheet, col_letter, row_no = m.group(1), m.group(2), m.group(3)
                if target_sheet not in wb.sheetnames:
                    continue
                label, val = values[vi]
                vi += 1
                tgt = wb[target_sheet]
                existing = tgt[f"{col_letter}{row_no}"].value
                if isinstance(existing, str) and existing.startswith("="):
                    res.add("info", target_sheet, int(row_no), label,
                            f"summary cell {col_letter}{row_no} already holds a "
                            f"formula ({existing[:34]}) - left as it is",
                            severity="note")
                    continue
                tgt[f"{col_letter}{row_no}"] = val
                tgt[f"{col_letter}{row_no}"].number_format = "#,##0.00"
                res.add("total", name, r, label,
                        f"summary row -> {target_sheet}!{col_letter}{row_no} "
                        f"= R{val:,.2f}")

    # 2. any remaining TOTAL CARRIED FORWARD rows get a live SUM over the data
    #    rows in their own block, so the workbook still adds up on its own
    for name, lay in layouts.items():
        ws = wb[name]
        c_amt = lay.col("amount")
        if not c_amt or not lay.total_rows:
            continue
        letter_amt = get_column_letter(c_amt)
        for tr in lay.total_rows:
            if ws.cell(tr, c_amt).value is not None:
                continue          # already written by the summary pass
            prev = lay.header_row
            for r2 in lay.total_rows:
                if r2 >= tr:
                    break
                prev = r2
            rows = [r for r in lay.data_rows if prev < r < tr]
            cur = ws.cell(tr, c_amt).value
            if rows and not (isinstance(cur, str) and cur.startswith("=")):
                ws.cell(tr, c_amt,
                        f"=SUM({letter_amt}{min(rows)}:{letter_amt}{max(rows)})"
                        ).number_format = "#,##0.00"
        res.add("total", name, lay.total_rows[0] if lay.total_rows else 0,
                f"{len(lay.total_rows)} TOTAL row(s)",
                "remaining TOTAL rows given a live SUM over their own block")