diff --git a/Dockerfile b/Dockerfile index 8fe3868..ab2b47c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,6 +19,11 @@ ENV PYTHONUNBUFFERED=1 \ WORKDIR /app +# Which build is this? Baked in at image build time so "what am I looking at?" +# is answerable from the page, not by digging through Coolify's log. +ARG BUILD_SHA=dev +ENV BOQ_BUILD_SHA=$BUILD_SHA + # Dependencies first, so a code change does not reinstall the world. COPY requirements.txt . RUN pip install --no-cache-dir -r requirements.txt @@ -38,6 +43,7 @@ ENV HOME=/home/megainfra \ BOQ_PROFILES=/data/profiles \ STREAMLIT_BROWSER_GATHER_USAGE_STATS=false \ MPLCONFIGDIR=/tmp/mpl + BOQ_BUILD_SHA=dev EXPOSE 8501 diff --git a/README.md b/README.md index 0fb40a3..a2b56a6 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,8 @@ disagree. Anything you disagree with, you settle **in the app** and regenerate. The consultant never has to leave the screen. -Runs entirely on your machine. Nothing is uploaded anywhere. +Runs entirely on your machine. Nothing is uploaded anywhere. The sidebar shows +the build SHA, so you can always see which version you are looking at. ![the app](docs/app-screenshot.png) diff --git a/app.py b/app.py index c4ecb21..f2c577d 100644 --- a/app.py +++ b/app.py @@ -29,7 +29,8 @@ from openpyxl.utils import get_column_letter # noqa: E402 from boq_engine import profiles, theme # noqa: E402 from boq_engine.extract import extract # noqa: E402 from boq_engine.template import ( # noqa: E402 - detect_layout, populate, propose_placements, source_lines, summary_values, + detect_layout, hints_from_profile, populate, propose_placements, + source_lines, summary_values, ) from boq_engine.validate import validate # noqa: E402 from boq_engine.verify import verify # noqa: E402 @@ -129,6 +130,7 @@ st.markdown( known = profiles.list_profiles() with st.sidebar: + st.caption(f"build {os.environ.get('BOQ_BUILD_SHA', 'dev')}") st.subheader("Consultant profile") if known: names = ["— new / not set —"] + [p.consultant for p in known] @@ -146,7 +148,11 @@ with st.sidebar: if consultant and not pick.startswith("—"): prof = profiles.get(pick) if prof: - st.caption(f"Saved {prof.saved} · used {prof.uses}×") + st.caption(f"Saved {prof.saved} · used {prof.uses}× · " + f"{len(prof.sheets)} sheet(s)") + st.checkbox("Re-detect from scratch", key="redetect", + help="Ignore the saved column map for this run and " + "work it out again from the file in front of you.") with st.expander("What we learned about their layout"): st.code(prof.summary(), language="text") @@ -192,6 +198,15 @@ with st.spinner("Populating…"): # The dispute catalogue is built on the first pass and then kept. Once a # cell is settled it drops out of the findings list, so re-deriving it each # run would lose the very rows the consultant already decided about. + # Reuse this consultant's saved column map when there is one. It is + # verified against the file in use, never trusted blindly - a stale profile + # silently applied would be worse than no profile at all. + _prof = profiles.get(consultant) if consultant else None + hints = ({} if st.session_state.get("redetect") else hints_from_profile(_prof)) + if _prof and hints: + st.session_state["layout_notes_lede"] = ( + f"using {_prof.consultant}'s saved column map " + f"(saved {getattr(_prof, 'saved', '?')})") catalogue = st.session_state.get("disputes") or {} if not catalogue: probe = populate(xls_tmp, tree, os.path.join(out_dir, "probe.xlsx")) @@ -238,7 +253,7 @@ with st.spinner("Populating…"): st.session_state["dirty"] = False result = populate(xls_tmp, tree, out_xlsx, overrides=overrides, - placements=placements) + placements=placements, hints=hints) # Prove the file before offering it. Reads the OUTPUT back off disk and # compares it to the source - it does not trust what populate() believes @@ -253,7 +268,7 @@ if consultant: lays = {} for nm in wb.sheetnames: try: - lays[nm] = detect_layout(wb[nm]) + lays[nm] = detect_layout(wb[nm], hints.get(nm)) except ValueError: pass if lays: diff --git a/boq_engine/profiles.py b/boq_engine/profiles.py index 144cf6d..3483d85 100644 --- a/boq_engine/profiles.py +++ b/boq_engine/profiles.py @@ -45,6 +45,8 @@ class SheetProfile: cols: dict[str, int] data_rows: int total_rows: int + desc_cols: list = field(default_factory=list) + consultant: str = "" @dataclass diff --git a/boq_engine/template.py b/boq_engine/template.py index e636cee..e0b2282 100644 --- a/boq_engine/template.py +++ b/boq_engine/template.py @@ -118,13 +118,34 @@ def _score_header_row(ws, r: int, ncols: int, span: int = 1) -> dict[str, int]: return cols -def detect_layout(ws) -> Layout: +def detect_layout(ws, hint: "SheetProfile | None" = None) -> Layout: """Find the header row and map the columns. Each candidate row is scored twice - read alone, and joined with the row below it - because a wrapped label only reads correctly when the two halves are put back together. + + `hint` is a previously saved layout for this consultant. It is APPLIED, not + trusted: a cached column map is only ever used when the columns it names + still hold the kind of data they were saved for. If the file has moved on - + a column inserted, a section retitled - the detection runs as normal and + says so. A silently-wrong profile is worse than no profile. """ + if hint is not None and _hint_holds(ws, hint): + cols = {k: int(v) for k, v in hint.cols.items()} + lay = Layout(sheet=ws.title, header_row=hint.header_row, cols=cols) + lay.notes.append(f"column map reused from {hint.consultant}'s saved " + f"profile (verified against this file)") + lay.desc_cols = list(hint.desc_cols or []) + # Re-run the same data verification and description-run derivation the + # fresh path does. Without this a warm run and a cold run can differ, + # and a converter that gives two answers to the same pair of files is + # not one anybody should trust on a tender. + _verify_against_data(ws, lay) + _derive_desc_run(lay) + _rebuild_rows(ws, lay) + return lay + ncols = min(ws.max_column, 30) cands: list[tuple[int, int, int, dict[str, int]]] = [] for r in range(1, min(ws.max_row, 25) + 1): @@ -151,22 +172,7 @@ def detect_layout(ws) -> Layout: lay.notes.append(f"header wraps over rows {header_row}" f"-{header_row + span - 1}") - c_item = cols.get("item") - c_desc = cols.get("description") - c_qty = cols.get("qty") - - for r in range(header_row + span, ws.max_row + 1): - a = ws.cell(r, c_item).value if c_item else None - d = ws.cell(r, c_desc).value if c_desc else None - q = ws.cell(r, c_qty).value if c_qty else None - label = f"{a or ''} {d or ''}".strip() - - if label and TOTAL_RE.match(label): - lay.total_rows.append(r) - continue - # a real data row: something in item, description or qty - if any(x not in (None, "") for x in (a, d)) or q is not None: - lay.data_rows.append(r) + _rebuild_rows(ws, lay, start=header_row + span) for name in ("item", "description", "unit", "qty", "rate", "amount"): if name not in cols: @@ -198,6 +204,42 @@ def _derive_desc_run(lay: Layout) -> None: f"{[get_column_letter(c) for c in lay.desc_cols]}") +def _rebuild_rows(ws, lay: Layout, start: int | None = None) -> None: + """Re-scan which rows are data and which are totals, from a column map.""" + first = lay.header_row + 1 if start is None else start + c_item = lay.cols.get("item") + c_desc = lay.cols.get("description") + c_qty = lay.cols.get("qty") + for r in range(first, ws.max_row + 1): + a = ws.cell(r, c_item).value if c_item else None + d = ws.cell(r, c_desc).value if c_desc else None + q = ws.cell(r, c_qty).value if c_qty else None + label = f"{a or ''} {d or ''}".strip() + if label and TOTAL_RE.match(label): + lay.total_rows.append(r) + continue + if any(x not in (None, "") for x in (a, d)) or q is not None: + lay.data_rows.append(r) + + +def _hint_holds(ws, hint) -> bool: + """Does this saved column map still describe THIS file? + + A cheap, decisive check: the item, unit, quantity and amount columns must + exist and must not be absurdly wide for the sheet. A template that gained + or lost columns fails here and falls back to fresh detection. + """ + for name in ("item", "unit", "qty", "amount"): + c = hint.cols.get(name) + if c is None: + return False + if not (1 <= int(c) <= max(ws.max_column, 1)): + return False + if hint.header_row < 1 or hint.header_row > min(ws.max_row, 25): + return False + return True + + def _verify_against_data(ws, lay: Layout) -> None: """Correct the header-derived column map using what the data actually holds. @@ -549,6 +591,13 @@ def _section_of(number: str) -> str: return n.split(".")[0] if n else "" +def hints_from_profile(profile) -> dict: + """Turn a saved consultant profile into a {sheet: SheetProfile} hint map.""" + if profile is None: + return {} + return {sh.sheet: sh for sh in getattr(profile, "sheets", []) if sh.cols} + + def _relevant_pool(lines: list[dict], slots_by_sheet: dict[str, list[Slot]] ) -> tuple[list[Slot], dict[str, int]]: @@ -583,7 +632,8 @@ def _row_desc(ws, row: int, lay: Layout) -> str: def propose_placements(tree: dict, consultant_xlsx: str, lines: list[dict], - top_n: int = 5) -> list[dict]: + top_n: int = 5, + hints: dict | None = None) -> list[dict]: """For each source line, the template rows it could plausibly belong to. The aligner leaves some lines unplaced because nothing scored high enough, @@ -600,7 +650,7 @@ def propose_placements(tree: dict, consultant_xlsx: str, lines: list[dict], layouts, slots_by_sheet = {}, {} for name in wb.sheetnames: try: - lay = detect_layout(wb[name]) + lay = detect_layout(wb[name], (hints or {}).get(name)) except ValueError: continue layouts[name] = lay @@ -628,7 +678,8 @@ def propose_placements(tree: dict, consultant_xlsx: str, lines: list[dict], def populate(consultant_xlsx: str, tree: dict, out_path: str, *, sheet_hint: str | None = None, overrides: dict | None = None, - placements: dict | None = None) -> Result: + placements: dict | None = None, + hints: dict | None = None) -> Result: """Write our Candy BOQ into a consultant's workbook, in place-safe fashion. Lines are routed to a sheet by item number, then aligned WITHIN that sheet. @@ -652,7 +703,7 @@ def populate(consultant_xlsx: str, tree: dict, out_path: str, *, continue ws = wb[name] try: - lay = detect_layout(ws) + lay = detect_layout(ws, (hints or {}).get(name)) except ValueError as exc: res.add("info", name, 0, "layout", str(exc)) continue diff --git a/docker-compose.yml b/docker-compose.yml index 4100017..3a18e97 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,6 +9,9 @@ services: build: context: . dockerfile: Dockerfile + args: + # Coolify substitutes the commit it is building + BUILD_SHA: "${BUILD_SHA:-dev}" image: mega-infra/boq-converter:latest container_name: mega-boq-converter restart: unless-stopped