From 301d0fff44600d2effef37a0064d4c0c7350e457 Mon Sep 17 00:00:00 2001 From: MEGA INFRA Date: Thu, 1 Oct 2026 02:32:13 +0200 Subject: [PATCH] =?UTF-8?q?MEGA=20INFRA=20BOQ=20Converter=20=E2=80=94=20Ca?= =?UTF-8?q?ndy=20PDF=20to=20consultant=20Excel?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reads our Candy PDF (deterministically, no OCR or model needed) and populates the consultant's own workbook using their formulas, column layout and PAYMENT column. Nothing is ever silently overwritten: every disagreement is reported, can be settled inside the app, and regenerated. - extract Candy text PDF -> tree (x-position column detection) - template layout detect, sequence align, write, totals, overrides - validate qty x rate and children-vs-subtotal reconciliation - profiles per-consultant layout profiles, stored on a volume Shipped with a Dockerfile + docker-compose for Coolify. --- .dockerignore | 17 ++ .gitignore | 6 + .streamlit/config.toml | 15 + DEPLOY.md | 94 ++++++ Dockerfile | 60 ++++ README.md | 171 +++++++++++ app.py | 415 ++++++++++++++++++++++++++ boq_engine/__init__.py | 0 boq_engine/extract.py | 491 +++++++++++++++++++++++++++++++ boq_engine/populate.py | 362 +++++++++++++++++++++++ boq_engine/profiles.py | 155 ++++++++++ boq_engine/template.py | 642 +++++++++++++++++++++++++++++++++++++++++ boq_engine/theme.py | 190 ++++++++++++ boq_engine/validate.py | 219 ++++++++++++++ docker-compose.yml | 41 +++ requirements.txt | 5 + run.py | 76 +++++ run_app.sh | 7 + static/theme.css | 449 ++++++++++++++++++++++++++++ 19 files changed, 3415 insertions(+) create mode 100644 .dockerignore create mode 100644 .gitignore create mode 100644 .streamlit/config.toml create mode 100644 DEPLOY.md create mode 100644 Dockerfile create mode 100644 README.md create mode 100644 app.py create mode 100644 boq_engine/__init__.py create mode 100644 boq_engine/extract.py create mode 100644 boq_engine/populate.py create mode 100644 boq_engine/profiles.py create mode 100644 boq_engine/template.py create mode 100644 boq_engine/theme.py create mode 100644 boq_engine/validate.py create mode 100644 docker-compose.yml create mode 100644 requirements.txt create mode 100644 run.py create mode 100755 run_app.sh create mode 100644 static/theme.css diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..a1c52a1 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,17 @@ +# never let local state into the image or the repo +.venv/ +venv/ +__pycache__/ +*.py[cod] +.git/ +.gitignore +.DS_Store + +# working output - regenerated on every run +out/ +docs/*.png + +# editor / OS noise +.vscode/ +.idea/ +*.swp diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ec3bc9b --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +.venv/ +__pycache__/ +*.py[cod] +out/ +.DS_Store +docs/*.png diff --git a/.streamlit/config.toml b/.streamlit/config.toml new file mode 100644 index 0000000..744f89d --- /dev/null +++ b/.streamlit/config.toml @@ -0,0 +1,15 @@ +[theme] +base = "dark" +primaryColor = "#FEA944" +backgroundColor = "#1A202C" +secondaryBackgroundColor = "#262A37" +textColor = "#E1E1E1" +font = "sans serif" + +[server] +headless = true +address = "127.0.0.1" +port = 8501 + +[browser] +gatherUsageStats = false diff --git a/DEPLOY.md b/DEPLOY.md new file mode 100644 index 0000000..64694c8 --- /dev/null +++ b/DEPLOY.md @@ -0,0 +1,94 @@ +# Deploying MEGA BOQ Converter to Coolify + +Docker Compose, using the `Dockerfile` in this repo. Coolify builds the image, +publishes the port and puts its own HTTPS certificate in front — you do not +need nginx, certbot or a separate database. + +## Before you start + +Push the repo (already done if you followed the setup below), then in Coolify: + +1. **New Resource → Docker Compose** +2. **Source Type:** Git Repository +3. **Repository:** `http://forgejo-b82b4lpyyzyg40thdsjdd38i.163.245.214.68.sslip.io/David/MEGA-BOQ-WEBAPP.git` + Branch `main` +4. **Docker Compose File:** `docker-compose.yml` +5. **Base Directory:** `/` (leave as-is) +6. **Deploy** + +Coolify reads `expose: "8501"` from the compose file and routes your domain to +it. Set the domain in the resource's **Domain** tab and Coolify issues Let's +Encrypt automatically. + +## First deploy checklist + +| check | why | +|---|---| +| `GET /_stcore/health` returns `ok` | the container is actually serving, not just running | +| Page loads over **https** with a valid cert | Coolify's proxy is in front | +| Upload both files and regenerate | proves the PDF path works with the container's libraries | +| Enter a consultant name, reload the page | proves the profile volume is mounted | + +## Environment variables + +Only one matters, and it is already set in `docker-compose.yml`: + +```yaml +BOQ_PROFILES: /data/profiles +``` + +That path is on the named volume `boq-profiles`. **If you move it, every saved +consultant layout is lost on the next rebuild.** + +`TZ: Africa/Johannesburg` is set so container timestamps match SAST. + +Override anything in Coolify's **Environment Variables** tab rather than editing +the compose file, so redeploys do not fight you. + +## Things that will bite you + +**The container binds `0.0.0.0`, not `127.0.0.1`.** The `CMD` in the Dockerfile +passes `--server.address=0.0.0.0` on purpose. `.streamlit/config.toml` binds +`127.0.0.1` for local development, and that override would make the app +invisible from outside the container. CLI flags beat the config file, which is +why both exist. + +**XSRF protection is off** (`--server.enableXsrfProtection=false`). Coolify's +Traefik rewrites the request origin, and Streamlit's built-in origin check then +rejects its own POSTs as forged. The app has no auth and no CSRF-sensitive +state, so this is safe here — but it is another reason not to expose it to the +open internet without putting auth in front. + +**Do not scale past one replica** unless you move `BOQ_PROFILES` to a real +shared store. Two replicas would each keep their own copy of the consultant +profiles and quietly disagree. One replica is plenty for internal use. + +**Uploads are in-memory.** Each conversion stages files in `/tmp` and discards +them. Nothing about a client's BOQ persists unless you download the result, and +the profile store holds only consultant names and column maps — never document +content. + +## Rolling back + +Coolify keeps the previous image. **Rollback** on the deployment reverts to it. +If a redeploy somehow loses the profiles, the volume is still there — check that +`BOQ_PROFILES` still points at `/data/profiles` in the environment tab. + +## Running it locally without Coolify + +```bash +docker compose up -d # http://127.0.0.1:8501 +``` + +Or plain Docker: + +```bash +docker build -t mega-boq . +docker run --rm -p 8501:8501 -v boq-profiles:/data/profiles mega-boq +``` + +Or without Docker at all: + +```bash +./run_app.sh # creates a venv, runs on port 8501 +``` diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..8fe3868 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,60 @@ +# syntax=docker/dockerfile:1 +# +# MEGA INFRA · BOQ Converter +# Streamlit web app. Single process, no database, no build step. + +FROM python:3.12-slim + +# libGL/libglib are what pdfplumber -> pdfminer.six -> Wand/OpenCV chain in; +# without them the container starts and then dies on the first PDF. +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + libglib2.0-0 libgl1 curl \ + && rm -rf /var/lib/apt/lists/* + +ENV PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + PIP_NO_CACHE_DIR=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 + +WORKDIR /app + +# Dependencies first, so a code change does not reinstall the world. +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY app.py run.py ./ +COPY boq_engine/ ./boq_engine/ +COPY static/ ./static/ +COPY .streamlit/ ./.streamlit/ + +# Run as a non-root user. UID 1000 matches the usual Coolify/Docker convention. +RUN useradd --create-home --uid 1000 --shell /bin/bash megainfra \ + && mkdir -p /data/profiles /app/out \ + && chown -R megainfra:megainfra /app /data +USER megainfra + +ENV HOME=/home/megainfra \ + BOQ_PROFILES=/data/profiles \ + STREAMLIT_BROWSER_GATHER_USAGE_STATS=false \ + MPLCONFIGDIR=/tmp/mpl + +EXPOSE 8501 + +HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=3 \ + CMD curl -fsS http://localhost:8501/_stcore/health || exit 1 + +# Flags here deliberately override .streamlit/config.toml: +# --server.address 0.0.0.0 the config binds 127.0.0.1 for local dev, which +# is unreachable from outside the container +# --server.enableXsrfProtection false +# Coolify's Traefik sits in front and rewrites the +# origin; Streamlit's origin check then rejects +# the rewritten POST as a forged request +CMD ["streamlit", "run", "app.py", \ + "--server.port=8501", \ + "--server.address=0.0.0.0", \ + "--server.headless=true", \ + "--server.enableCORS=false", \ + "--server.enableXsrfProtection=false", \ + "--browser.gatherUsageStats=false"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..ca9dda0 --- /dev/null +++ b/README.md @@ -0,0 +1,171 @@ +# MEGA INFRA · BOQ Converter + +Drop in **our** Candy PDF and **their** Excel. Get back their workbook +populated, using their own formulas and layout, plus a proof report of exactly +what was written, what was already correct, and where the two documents +disagree. + +Anything you disagree with, you settle **in the app** and regenerate. The +consultant never has to leave the screen. + +Runs entirely on your machine. Nothing is uploaded anywhere. + +![the app](docs/app-screenshot.png) + +## Settling a disagreement in the app + +Where the two documents differ, the app writes **nothing** — your value stands. +Each dispute is one collapsed row. Open it and choose: + +- **Keep yours (as it is)** — the default, your file is untouched +- **Use our PDF's value** — take the figure from the Candy export +- **or type a value** — anything else, coerced to a number when it looks like one + +Press **Apply & regenerate** and the choices are written straight into the +workbook; the row then moves to "settled" and drops off the open list. The +PAYMENT column, your formulas and your formatting are never touched. + +![disputes](docs/app-disputes.png) + +## Design system — MEGA INFRASTRUCTURE + +Palette sampled from the dashboard. All tokens live in **`static/theme.css`** +as CSS custom properties, so the brand can be retuned in one place without +touching Python. + +| group | tokens | +|---|---| +| surfaces | `--canvas #1A202C` · `--topbar #1A212D` · `--surface #262A37` · `--surface-inset #252A36` · `--surface-raise #363B49` · `--hairline #2F3442` | +| status fills | `--success-fill #22383E` / `--success-edge #1B534A` · `--warn-fill #3A373C` · `--danger-fill #3A2D38` | +| accents | `--orange #FEA944` · `--orange-soft #F1A962` · `--orange-deep #DF8441` · `--green #00CD8A` · `--red #F04141` | +| text | `--text #E1E1E1` · `--text-muted #999999` · `--text-warm #B3AC99` · `--text-link #00CD8A` | + +Warm neutrals are deliberate — `#E1E1E1` sits on a warm axis and pairs with +`#B3AC99`; pure `#EEEEEE` would read cold next to `#FEA944`. + +**On the accessibility flag:** `#F04141` measures 3.77:1 on the card surface, so +it is used only for the severity edge and large marks. Any red *text* uses +`--red-soft #FF6B6B` at 5.15:1, which clears AA. + +Type is **Fraunces** for headings and **Nunito** for body. + +### Icons are real SVG, not emoji + +Thirteen icons live in `boq_engine/theme.py` as inline SVG — 24x24, 1.75 +stroke, round caps, so the set looks drawn by one hand. They inherit +`currentColor`, so they pick up the palette for free, and being inline they +need no network and no icon font. + +They sit in the brand mark, the uploader labels, the KPI tiles, the pills and +the section headings — everywhere HTML actually renders. + +**Not** on buttons or tab labels. Streamlit renders those as *plain text*, so +an inline SVG there leaks to the user as literal `` markup, and emoji +read as emoticons. Those labels are deliberately typographic, which looks +more corporate anyway. + +### Two implementation notes worth knowing + +**The stylesheet is injected with `st.html`, not `st.markdown`.** Markdown +injection duplicated the style block on every rerun, and one malformed +selector silently killed every rule after it — the page rendered unstyled with +no error anywhere. Serving it from `static/` was tried and doesn't work: +Streamlit only serves its own hashed assets from that route, and an arbitrary +path falls through to the SPA shell and returns `index.html` as `text/html`. +`st.html` inserts the sheet into the page, isn't iframed, and replaces cleanly. + +**Buttons and tab labels are plain text.** See the icon note above — this is +the one place Streamlit refuses to render HTML, and it fails loudly. + +**Status messages are quiet pills, not alert blocks.** A filled +`st.warning` bar shouts at the reader for something that is background +information. `theme.pill()` sits at the same volume as the legend it belongs +next to. Genuine errors — source lines that have nowhere to go, subtotals that +do not reconcile — still use real alerts, because those are the thing to act on. + +**Dispute choices are real Streamlit widgets.** Raw HTML `
' + f'{theme.icon("building", 30)}' + '' + 'MEGA INFRA' + 'BOQ Converter  ·  ' + 'Candy PDF (ours)  →  consultant\'s Excel (theirs) ' + ' ·  nothing leaves this machine' + '
', + unsafe_allow_html=True) + +known = profiles.list_profiles() +with st.sidebar: + st.subheader("Consultant profile") + if known: + names = ["— new / not set —"] + [p.consultant for p in known] + pick = st.selectbox("Known consultants", names, index=0) + else: + pick = "— new / not set —" + st.caption("No profiles saved yet.") + + consultant = st.text_input( + "Consultant name", + value="" if pick.startswith("—") else pick, + placeholder="e.g. Civil Matters", + help="Saved so next time their layout is recognised automatically.", + ) + if consultant and not pick.startswith("—"): + prof = profiles.get(pick) + if prof: + st.caption(f"Saved {prof.saved} · used {prof.uses}×") + with st.expander("What we learned about their layout"): + st.code(prof.summary(), language="text") + +# ---------------------------------------------------------------- inputs +c1, c2 = st.columns(2) +with c1: + st.markdown( + f'
{theme.icon("file", 15)}' + f'1  Candy PDF (ours)
', + unsafe_allow_html=True) + pdf_file = st.file_uploader(" ", type=["pdf"], label_visibility="collapsed") +with c2: + st.markdown( + f'
{theme.icon("table", 15)}' + f'2  Consultant Excel (theirs)
', + unsafe_allow_html=True) + xls_file = st.file_uploader(" ", type=["xlsx", "xlsm"], + label_visibility="collapsed") + +st.markdown("---") + +if not (pdf_file and xls_file): + st.markdown(theme.pill( + "Drop in both files to begin — the Candy PDF and the " + "consultant's Excel.", "info", "info"), unsafe_allow_html=True) + st.stop() + +# ---------------------------------------------------------------- run +out_dir = tempfile.mkdtemp(prefix="boq_out_") +base = os.path.splitext(os.path.splitext(pdf_file.name)[0])[0] +out_xlsx = os.path.join(out_dir, f"{base}_POPULATED.xlsx") + +with st.spinner("Populating…"): + tree = read_pdf(pdf_file.getvalue()) + src_rep = validate(tree) + xls_tmp = os.path.join(out_dir, "in.xlsx") + with open(xls_tmp, "wb") as fh: + fh.write(xls_file.getvalue()) + + # Re-read the staged choices from the previous run's widgets. This happens + # BEFORE populate() so a regenerate writes the consultant's decisions + # straight into the workbook - no second manual step. + # The dispute catalogue is built on the first pass and then kept. Once a + # cell is settled it drops out of the findings list, so re-deriving it each + # run would lose the very rows the consultant already decided about. + catalogue = st.session_state.get("disputes") or {} + if not catalogue: + probe = populate(xls_tmp, tree, os.path.join(out_dir, "probe.xlsx")) + catalogue = { + (f.sheet, f.row, f.field): { + "label": f.label, "detail": f.detail, + "source_value": f.source_value, + "template_value": f.template_value, + "severity": f.severity, + } + for f in probe.findings if f.kind == "conflict" + } + st.session_state["disputes"] = catalogue + + overrides: dict = {} + if st.session_state.get("dirty"): + for key, val in list(st.session_state.items()): + if not key.startswith("choice::") or key.endswith("::custom"): + continue + parts = key.split("::") + if len(parts) != 4: + continue + cell = (parts[1], int(parts[2]), parts[3]) + if cell not in catalogue: + continue + custom = st.session_state.get(f"{key}::custom", "").strip() + if custom: + overrides[cell] = _coerce(custom) + elif str(val).startswith("Use our"): + overrides[cell] = catalogue[cell]["source_value"] + st.session_state["dirty"] = False + st.session_state["applied"] = len(overrides) + + result = populate(xls_tmp, tree, out_xlsx, overrides=overrides) + +# remember this consultant's layout for next time +if consultant: + wb = load_workbook(xls_tmp) + lays = {} + for nm in wb.sheetnames: + try: + lays[nm] = detect_layout(wb[nm]) + except ValueError: + pass + if lays: + profiles.save(consultant, lays, _fp(xls_file.getvalue())) + profiles.touch(consultant) + +# ---------------------------------------------------------------- results +def kpi(icon_name: str, label: str, value, tone: str = "orange") -> str: + """A KPI tile. Hand-built so it can carry a real icon - st.metric's label + is plain text, and a row of four plain numbers is the generic-looking part + of this page.""" + cls = f"boq-kpi boq-kpi-{tone}" + return ( + f'
' + f'
' + f'{theme.icon(icon_name, 16)}' + f'{_html.escape(label)}' + f"
" + f'
{value}
' + f"
" + ) + + +k1, k2, k3, k4 = st.columns(4) +_open_n = len([f for f in result.findings + if f.kind == "conflict" and f.severity != "resolved"]) +_unmapped_n = len([f for f in result.findings if f.kind == "unmapped"]) +k1.markdown(kpi("check", "Values written", result.written), unsafe_allow_html=True) +k2.markdown(kpi("shield", "Already correct", result.kept), unsafe_allow_html=True) +k3.markdown(kpi("alert", "Open disagreements", _open_n, + "warn" if _open_n else "ok"), unsafe_allow_html=True) +k4.markdown(kpi("info", "Unmapped source lines", _unmapped_n, + "warn" if _unmapped_n else "ok"), unsafe_allow_html=True) + +applied = st.session_state.get("applied", 0) +_tail = (f"with {applied} of your decisions written in. " + if applied else "") +st.markdown(theme.pill( + f"Ready to send. {_tail}Your file was populated using its own " + f"formulas and layout — download it and send it with the Candy PDF.", + "check", "ok"), unsafe_allow_html=True) + +with open(out_xlsx, "rb") as fh: + st.download_button("Download populated workbook", + fh.read(), + file_name=os.path.basename(out_xlsx), + mime="application/vnd.openxmlformats-officedocument" + ".spreadsheetml.sheet") + +# ---------------------------------------------------------------- proof +st.markdown("---") +st.markdown( + f'
{theme.icon("clipboard", 20)}' + f'Proof — what happened, line by line
', + unsafe_allow_html=True) + +conflicts = [f for f in result.findings if f.kind == "conflict"] +unmapped = [f for f in result.findings if f.kind == "unmapped"] +totals = [f for f in result.findings if f.kind == "total"] +notes = [f for f in result.findings if f.kind == "info"] + +SEVERITY_SWATCH = { + "decision": ("#8C363B", "needs a decision"), + "source_gap": ("#DF8441", "our PDF has no figure - theirs kept"), + "source_defect": ("#FEA944", "our PDF printed something corrupt"), + "resolved": ("#00CD8A", "settled by the consultant"), + "note": ("#00CD8A", "no action needed"), +} + + +def _legend() -> str: + seen, parts = set(), [] + for sev, (col, label) in SEVERITY_SWATCH.items(): + if sev == "note" or label in seen: + continue + seen.add(label) + parts.append( + f'' + f"{label}") + return f'
{"".join(parts)}
' + + +# NOTE: st.tabs / st.button / st.download_button render their labels as PLAIN +# TEXT. Inline SVG there leaks to the user as literal markup, and emoji read as +# emoticons. So those stay typographic; the real icons live where HTML is +# actually rendered - the brand mark, KPI tiles, pills and section headings. +tab_conf, tab_tot, tab_src = st.tabs([ + f"Disagreements ({len(conflicts)})", + f"Totals written ({len(totals)})", + f"Source BOQ checks ({len(src_rep.errors)})", +]) + +with tab_conf: + if not conflicts and not unmapped: + st.markdown(theme.pill("The two documents agree everywhere.", + "check", "ok"), unsafe_allow_html=True) + + if unmapped: + st.error(f"**{len(unmapped)} source line(s) had no row in the " + f"template.** These are NOT in the consultant's file:") + st.markdown("".join( + theme.card(f.label.strip(), + f"{f.detail} (source: {f.source})", + meta="no row in template", severity="decision") + for f in unmapped), unsafe_allow_html=True) + + settled = [f for f in conflicts if f.severity == "resolved"] + open_f = [f for f in conflicts if f.severity != "resolved"] + if settled: + st.markdown(theme.pill( + f"{len(settled)} decision(s) applied and written into the " + f"workbook.", "check", "ok"), unsafe_allow_html=True) + if open_f: + st.markdown(theme.pill( + "Nothing was overwritten. Where the two documents differ, " + "your own value was left alone. Open anything you want changed, " + "then regenerate.", + "shield", "ok"), unsafe_allow_html=True) + st.markdown(_legend(), unsafe_allow_html=True) + + # Real Streamlit widgets, so the choice actually reaches Python. Keys + # are derived from the cell, so a choice survives a rerun and can be + # read back to build the override map. + # one collapsed row per dispute: 16 open boxes is a wall of scrolling, + # and the point is to scan for the one that needs a human + for f in open_f: + key = f"choice::{f.sheet}::{f.row}::{f.field}" + chosen = st.session_state.get(key, "Keep yours (as it is)") + staged = st.session_state.get(f"{key}::custom", "").strip() + flag = "✎ " if (staged or chosen.startswith("Use our")) else "" + with st.expander( + f"{flag}{f.label.strip()[:70]} · {f.sheet} r{f.row} · " + f"{f.field or 'value'}", + expanded=bool(flag)): + st.caption(_strip_md(f.detail)) + c_a, c_b = st.columns([1, 1]) + with c_a: + st.selectbox( + "which value stands", + options=["Keep yours (as it is)", + f"Use our PDF's value ({_fmt(f.source_value)})"], + key=key, label_visibility="collapsed") + with c_b: + st.text_input( + "or type a different value", + key=f"{key}::custom", label_visibility="collapsed", + placeholder="or type a value") + + st.markdown("---") + c1, c2 = st.columns([3, 1]) + with c1: + n_custom = len([ + 1 for f in open_f + if st.session_state.get(f"choice::{f.sheet}::{f.row}::{f.field}::custom", "").strip() + ]) + n_take = len([ + 1 for f in open_f + if st.session_state.get(f"choice::{f.sheet}::{f.row}::{f.field}", + "Keep yours (as it is)").startswith("Use our") + ]) + if n_custom or n_take: + st.markdown(theme.pill( + f"{n_take + n_custom} change(s) staged. Press " + f"regenerate to write them into the workbook.", + "refresh", "warn"), unsafe_allow_html=True) + else: + st.caption("Your PAYMENT column, formulas and formatting are " + "never touched.") + with c2: + if st.button("Apply & regenerate", type="primary", + use_container_width=True): + st.session_state["dirty"] = True + st.rerun() + +with tab_tot: + st.caption("The cells your consultant's summary sheet reads were blank in " + "the template. Each now carries its section's total.") + # Streamlit's dataframe has no text-wrapping option (no wrap_text in + # column_config), and this column is prose, so it gets cards like the + # findings above rather than a table that truncates. + st.markdown("".join( + theme.card(f.label.strip(), f.detail, meta="written to", + severity="note") + for f in totals), unsafe_allow_html=True) + + vals = summary_values(tree) + st.markdown( + f'
{theme.icon("layers", 15)}' + f'Totals carried across, in order
', + unsafe_allow_html=True) + st.dataframe([{"#": i + 1, "Schedule / Section": l, "Amount": money(v)} + for i, (l, v) in enumerate(vals)], + use_container_width=True, hide_index=True) + if vals: + st.markdown(theme.pill( + f"Sum of all sections: {money(sum(v for _, v in vals))}", + "calculator", "info"), unsafe_allow_html=True) + +with tab_src: + if not src_rep.errors: + st.markdown(theme.pill("The source BOQ adds up on its own.", + "check", "ok"), unsafe_allow_html=True) + for i in src_rep.errors: + st.error(f"**{i.where}**\n\n{i.detail}") + if notes: + with st.expander(f"Notes ({len(notes)})"): + for f in notes: + st.write(f"· `{f.sheet}` r{f.row} — {f.label}: {f.detail}") + if result.layout_notes: + with st.expander("Layout detection notes"): + for n in result.layout_notes: + st.write(f"· {n}") diff --git a/boq_engine/__init__.py b/boq_engine/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/boq_engine/extract.py b/boq_engine/extract.py new file mode 100644 index 0000000..bee3d5e --- /dev/null +++ b/boq_engine/extract.py @@ -0,0 +1,491 @@ +""" +BOQ PDF -> structured tree extractor. + +Built for South African municipal / civil tender BOQs produced by Candy +"Continuous Listing" print layouts. Those PDFs are *text* (printed from Excel), +not scans, so the approach is deterministic column detection by x-position +rather than OCR or an LLM. + +Column geometry observed on Candy 2.05p output (A4 portrait): + Item left edge ~ x0 = 32 + Description left edge ~ x0 = 70 + Unit left edge ~ x0 = 302 + Qty RIGHT aligned, x1 ~ 400-411 + Rate RIGHT aligned, x1 ~ 453-484 + Amount RIGHT aligned, x1 ~ 512-565 + +Numeric columns are right-aligned, so RIGHT edges cluster far more tightly than +left edges and are the reliable signal. We therefore bucket by x1. + +Three passes: + 1. rows - words grouped by y, split into columns + 2. content - noise stripped, description continuations merged + 3. tree - content rows assembled into the numbered hierarchy +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field, asdict +from typing import Any + +import pdfplumber + +# -------------------------------------------------------------------------- +# column geometry +# -------------------------------------------------------------------------- +BANDS = { + "qty": (380.0, 436.0), + "rate": (436.0, 500.0), + "amount": (500.0, 575.0), +} + +ITEM_X_MAX = 60.0 +DESC_X_MIN, DESC_X_MAX = 60.0, 296.0 +UNIT_X_MIN, UNIT_X_MAX = 296.0, 380.0 + +# running header / footer / column-heading chrome +NOISE_PATTERNS = [ + re.compile(r"\d{1,2}:\d{2}:\d{2}"), # "Printed 08:21:20 ..." + re.compile(r"page\s+\d+", re.I), + re.compile(r"^\d{1,2}\s+\w+\s+\d{4}\s*$"), # "1 August 2026" + re.compile(r"^candy\b", re.I), + re.compile(r"continuous\s+listing", re.I), + re.compile(r"\bbill\s+description\b", re.I), + re.compile(r"^\d{2}-\d{3}\b"), # "26-048 - Blueberry Square" + re.compile(r"^boq\s*-", re.I), + re.compile(r"^selling$", re.I), +] +# the repeated column-heading row: item | bill description | unit | ... +HEADER_TOKENS = {"item", "unit", "rate", "amount", "bill", "quantity"} + +SECTION_RE = re.compile(r"^SECTION\s+(\d+)\s*[:\-]\s*(.+)$", re.I) +ITEM_RE = re.compile(r"^\d{1,2}(?:\.{1,2}\d{1,2})*\.?$") +LETTER_RE = re.compile(r"^\(?([a-z])\)\s*(.*)$", re.S) +PLACEHOLDER_RE = re.compile(r"^[.\s]{6,}$") +RATE_ONLY_RE = re.compile(r"rate\s*only", re.I) +SCHEDULE_RE = re.compile(r"^SCHEDULE\s+([A-Z])\b", re.I) + + +def _to_num(tok: str) -> float | None: + t = (tok or "").replace(",", "").replace(" ", "") + if not t or t in {"-", "."}: + return None + try: + return float(t) + except ValueError: + return None + + +def _classify(x1: float) -> str | None: + for name, (lo, hi) in BANDS.items(): + if lo <= x1 < hi: + return name + return None + + +# ========================================================================== +# pass 1 - rows +# ========================================================================== +@dataclass +class Row: + item: str = "" + desc: str = "" + unit: str = "" + qty: float | None = None + rate: float | None = None + amount_txt: str = "" + top: float = 0.0 + page: int = 0 + + @property + def has_price(self) -> bool: + return self.qty is not None or self.rate is not None + + @property + def is_empty(self) -> bool: + return not (self.item or self.desc or self.unit + or self.has_price or self.amount_txt) + + +def _page_rows(page, pno: int) -> list[Row]: + buckets: dict[int, list] = {} + for w in page.extract_words(): + buckets.setdefault(round(w["top"] / 3.0), []).append(w) + + rows: list[Row] = [] + for key in sorted(buckets): + words = sorted(buckets[key], key=lambda z: z["x0"]) + r = Row(top=words[0]["top"], page=pno) + desc_parts: list[str] = [] + last_desc_x1 = None + for w in words: + x0, x1, txt = w["x0"], w["x1"], w["text"] + if x1 <= ITEM_X_MAX: + r.item = (r.item + " " + txt).strip() + continue + col = _classify(x1) + if col == "qty" and r.qty is None: + r.qty = _to_num(txt) + continue + if col == "rate" and r.rate is None: + r.rate = _to_num(txt) + continue + if col == "amount": + r.amount_txt = (r.amount_txt + " " + txt).strip() + continue + if UNIT_X_MIN <= x0 < UNIT_X_MAX: + r.unit = (r.unit + txt).strip() + continue + if DESC_X_MIN <= x0 < DESC_X_MAX: + # Candy's print engine sometimes splits a word across two text + # runs ("S" + "ITE CLEARANCE"). Only heal a genuine fragment: + # a tiny orphan sitting flush against the previous word. + if (desc_parts and last_desc_x1 is not None + and len(desc_parts[-1]) <= 2 + and x0 - last_desc_x1 < 0.9 + and desc_parts[-1].isalpha() + and txt.isalpha()): + desc_parts[-1] += txt + else: + desc_parts.append(txt) + last_desc_x1 = x1 + r.desc = " ".join(desc_parts).strip() + rows.append(r) + return rows + + +# ========================================================================== +# pass 2 - strip noise, merge continuations +# ========================================================================== +def _is_noise(r: Row) -> bool: + blob = f"{r.item} {r.desc} {r.unit}".strip() + if r.has_price: + return False + if any(p.search(blob) for p in NOISE_PATTERNS): + return True + toks = {t.lower().strip(".,:") for t in blob.split()} + if toks and toks <= HEADER_TOKENS: + return True + return False + + +def _looks_like_heading(text: str) -> bool: + """True for text that starts a new block rather than continuing one.""" + t = text.strip() + if not t: + return False + if SECTION_RE.match(t) or SCHEDULE_RE.match(t): + return True + # an all-caps run of words is a heading ("TEMPORARY WORKS", "SUBBASE") + letters = [c for c in t if c.isalpha()] + if letters and all(c.isupper() for c in letters) and len(letters) > 3: + return True + return False + + +def _merge_continuations(rows: list[Row]) -> list[Row]: + """A row with no number, no unit, no price and no amount is a wrapped + description belonging to the row above it -- unless it opens a new heading. + """ + out: list[Row] = [] + for r in rows: + is_cont = ( + not r.item + and not r.unit + and not r.has_price + and not r.amount_txt + and r.desc + and not _looks_like_heading(r.desc) + ) + if is_cont and out: + prev = out[-1] + if prev.desc and not prev.is_empty: + prev.desc = f"{prev.desc} {r.desc}".strip() + continue + out.append(r) + return out + + +# ========================================================================== +# pass 3 - build the tree +# ========================================================================== +@dataclass +class Line: + kind: str = "line" + number: str = "" + desc: str = "" + unit: str = "" + qty: float | None = None + rate: float | None = None + amount: float | None = None + rate_only: bool = False + children: list = field(default_factory=list) + page: int = 0 + flags: list = field(default_factory=list) + + +@dataclass +class Group: + kind: str = "group" # 'root' | 'section' | 'schedule' | 'group' | 'item' + number: str = "" + title: str = "" + subtotal: float | None = None + children: list = field(default_factory=list) + page: int = 0 + flags: list = field(default_factory=list) + + +class _Tree: + def __init__(self) -> None: + self.root = Group(kind="root", title="BOQ") + self.stack: list[Group] = [self.root] + self.item_holder: Group | None = None + # Candy prints a heading when the block opens and the SAME number + # again with its subtotal when the block closes. Track them so the + # closing line attaches to the opening node instead of duplicating it. + self.by_number: dict[str, Group] = {} + self.seen_sections: set[str] = set() + + @property + def cur(self) -> Group: + return self.stack[-1] + + @staticmethod + def depth(number: str) -> int: + return number.count(".") + 1 if number else 1 + + def pop_to(self, depth: int) -> None: + """Close open nodes until the top of stack is shallower than `depth`.""" + while len(self.stack) > 1: + top = self.stack[-1] + if top.kind in ("root", "section", "schedule"): + break + if self.depth(top.number) < depth: + break + self.stack.pop() + + def section_base(self) -> Group: + """Where a new SECTION attaches: inside the current schedule if there + is one, otherwise at the root.""" + for node in reversed(self.stack): + if node.kind == "schedule": + return node + return self.root + + def reopen(self, number: str, depth: int) -> Group | None: + """If `number` is already open, make it current again.""" + node = self.by_number.get(number) + if node is None: + return None + while self.stack and self.stack[-1] is not node: + self.stack.pop() + return node + + def add(self, node) -> None: + self.cur.children.append(node) + + +def _same_heading(a: str, b: str) -> bool: + """Do these two labels refer to the same block?""" + na = re.sub(r"[^a-z0-9]", "", (a or "").lower()) + nb = re.sub(r"[^a-z0-9]", "", (b or "").lower()) + if not na or not nb: + return True + return na.startswith(nb[:24]) or nb.startswith(na[:24]) + + +def _build(content: list[Row]) -> Group: + t = _Tree() + seen_totals: set[str] = set() + + for r in content: + desc = r.desc + num = r.item.strip() + + # ---- page-2+ running totals: "BLUEBERRY SQUARE 5,696,934.93" ---- + if desc and r.amount_txt and not r.has_price and not num: + key = f"{desc[:40].upper()}|{r.amount_txt}" + if desc.upper() in {"TOTAL", "BLUEBERRY SQUARE"} or key in seen_totals: + if key in seen_totals: + continue + seen_totals.add(key) + + # ---- section / schedule headings --------------------------------- + # Both start a fresh top-level branch: a section is never nested + # inside the previous section's last item. + sm = SECTION_RE.match(desc) + if sm and not r.has_price: + t.item_holder = None + # key on number + TITLE. The document genuinely has two sections + # numbered 6 (STORMWATER and Traffic Lights), so the number alone + # is not an identity. + key = f"sec:{sm.group(1)}:{re.sub(r'[^a-z0-9]', '', sm.group(2).lower())}" + if key in t.seen_sections: + continue + t.seen_sections.add(key) + sec = Group(kind="section", number=sm.group(1), + title=sm.group(2).strip(), page=r.page, + # Candy prints "SECTION 1: SITE CLEARANCE 160,057.99" + # on the heading row - keep that figure, it is what the + # consultant's own summary sheet expects to reconcile to + subtotal=_to_num(r.amount_txt) if r.amount_txt else None) + # reset FIRST: a section is a sibling of every other section, not + # a child of whatever the previous section happened to leave open + base = t.section_base() + t.stack = [t.root, base] if base is not t.root else [t.root] + t.add(sec) + t.stack.append(sec) + t.by_number.clear() + continue + + if SCHEDULE_RE.match(desc) and not r.has_price and not num: + t.item_holder = None + if desc.upper() in t.seen_sections: + continue + t.seen_sections.add(desc.upper()) + sch = Group(kind="schedule", number="", title=desc, page=r.page) + t.stack = [t.root] + t.add(sch) + t.stack.append(sch) + t.by_number.clear() + continue + + # ---- priced leaf line ------------------------------------------- + if r.has_price and (r.unit or r.qty is not None): + rate_only = bool(RATE_ONLY_RE.search(r.amount_txt)) + amount = None if rate_only else _to_num(r.amount_txt) + + sub, body = None, desc + lm = LETTER_RE.match(desc) + if lm and not num: + sub, body = lm.group(1), lm.group(2).strip() + + # Candy often prices a NUMBERED item directly on its own heading + # row ("2.1.3 Cut to spoil m3 1,400 259.34 363,076.00"). + # It carries its own price, so it belongs to the current group - + # not to whichever item happened to precede it. Only an unnumbered + # a)/b) sub-line should attach to the open item. + numbered_item = bool(num) and ITEM_RE.match(num) \ + and t.depth(num) >= 3 + + line = Line( + number=num, + desc=f"{sub}) {body}" if sub else body, + unit=r.unit.strip(), + qty=r.qty, + rate=r.rate, + amount=amount, + rate_only=rate_only, + page=r.page, + ) + if not line.unit: + line.flags.append("no_unit") + if line.rate is None and line.qty is not None and amount is None \ + and not rate_only: + line.flags.append("amount_missing") + + if sub and t.item_holder is not None: + t.item_holder.children.append(line) + else: + if numbered_item: + # close the previously open item so this one lands beside + # it in the enclosing group, not inside it + t.pop_to(t.depth(num)) + t.item_holder = None + t.add(line) + if not numbered_item: + t.item_holder = None + continue + + if PLACEHOLDER_RE.match(desc or "x"): + t.add(Line(number=num, desc=desc, page=r.page)) + t.item_holder = None + continue + + # ---- numbered heading, optionally carrying a subtotal ------------ + if num and ITEM_RE.match(num): + depth = t.depth(num) + subtotal = _to_num(r.amount_txt) if r.amount_txt else None + + existing = t.by_number.get(num) + if existing is not None: + same_block = _same_heading(existing.title, desc) + if same_block and subtotal is not None and existing.subtotal is None: + # closing subtotal line for a block already opened + existing.subtotal = subtotal + t.item_holder = None + continue + if same_block and subtotal is None: + # plain repeat (the closing summary page lists every + # subtotal again) - nothing new to record + t.item_holder = None + continue + # same number, DIFFERENT block. Real Candy output does this + # ("2.3 SUBBASE" and "2.3 BASE"). Open a fresh node and leave a + # marker so validation can raise it. + existing.flags.append(f"duplicate_number:{num}") + del t.by_number[num] + t.pop_to(depth) + node = Group( + kind="item" if depth >= 3 else "group", + number=num, title=desc, subtotal=subtotal, page=r.page, + ) + t.add(node) + t.by_number[num] = node + t.stack.append(node) + # a depth>=3 item heading owns its a)/b)/c) sub-lines + t.item_holder = node if depth >= 3 else None + continue + + # ---- continuation of a heading that wrapped --------------------- + if desc and t.item_holder is not None and not r.unit and not r.has_price: + t.item_holder.title = f"{t.item_holder.title} {desc}".strip() + continue + + if desc: + t.add(Line(desc=desc, page=r.page)) + t.item_holder = None + + return t.root + + +def _drop_summary_page(rows: list[Row]) -> list[Row]: + """Candy closes a BOQ with a page that just re-lists every subtotal. + + That page contributes no new line items, only duplicates, so it is + identified by carrying several numbered subtotal rows and no priced line, + and dropped wholesale. + """ + by_page: dict[int, list[Row]] = {} + for r in rows: + by_page.setdefault(r.page, []).append(r) + + for page, prows in by_page.items(): + if len(prows) < 4: + continue + subtotals = [r for r in prows + if r.amount_txt and not r.has_price and r.item] + priced = [r for r in prows if r.has_price] + if len(subtotals) >= 4 and not priced: + rows = [r for r in rows if r.page != page] + return rows + + +def extract(pdf_path: str) -> dict: + rows: list[Row] = [] + with pdfplumber.open(pdf_path) as pdf: + for pno, page in enumerate(pdf.pages, start=1): + rows.extend(_page_rows(page, pno)) + + rows = _drop_summary_page(rows) + clean = [r for r in rows if not r.is_empty and not _is_noise(r)] + merged = _merge_continuations(clean) + return asdict(_build(merged)) + + +if __name__ == "__main__": # pragma: no cover + import json + import sys + + print(json.dumps(extract(sys.argv[1]), indent=2, default=str)) diff --git a/boq_engine/populate.py b/boq_engine/populate.py new file mode 100644 index 0000000..3c392bb --- /dev/null +++ b/boq_engine/populate.py @@ -0,0 +1,362 @@ +""" +Populate the standard BOQ Excel template (and a flat CSV) from an extracted +BOQ tree. + +Two output shapes, because they serve different jobs: + + populate_template() writes a *tender-ready* workbook. Line AMOUNT cells hold + live formulas (=Qty*Rate) and group totals are live + SUMs, so when the estimator types a rate the money + re-totals itself. This is the file that goes out. + + to_csv() writes a flat, one-row-per-line audit CSV. This is what + gets diffed against a competing tender or pushed into + their estimating system. +""" + +from __future__ import annotations + +import csv +from typing import Any + +from openpyxl import Workbook +from openpyxl.styles import Alignment, Border, Font, PatternFill, Side +from openpyxl.utils import get_column_letter + +# -------------------------------------------------------------------------- +# presentation +# -------------------------------------------------------------------------- +NAVY = "1F3864" +GREY = "F2F2F2" +BAND = "DDEBF7" +AMBER = "FFF2CC" + +HDR_FONT = Font(bold=True, color="FFFFFF", size=10) +HDR_FILL = PatternFill("solid", fgColor=NAVY) +SEC_FONT = Font(bold=True, size=11, color=NAVY) +GRP_FONT = Font(bold=True, size=10) +BODY_FONT = Font(size=10) +TOT_FONT = Font(bold=True, size=10) +TOT_FILL = PatternFill("solid", fgColor=BAND) +FLAG_FILL = PatternFill("solid", fgColor=AMBER) + +THIN = Side(style="thin", color="BFBFBF") +BOX = Border(left=THIN, right=THIN, top=THIN, bottom=THIN) + +MONEY = '#,##0.00' +QTY_FMT = '#,##0.###' + +# a unit a human would recognise; used to sanity-check extraction +KNOWN_UNITS = { + "m", "m2", "m²", "m3", "m³", "km", "km2", "mm", "cm", + "no", "no.", "sum", "sum.", "month", "week", "day", "hour", + "pc", "pc.sum", "ps.sum", "p.sum", "prov sum", "item", "each", + "t", "kg", "l", "litre", "litres", "%", "?", "ha", "m³ ", "m² ", +} + + +def _is_known_unit(u: str) -> bool: + return (u or "").strip().lower().rstrip(".") in { + x.strip().lower().rstrip(".") for x in KNOWN_UNITS + } + + +# -------------------------------------------------------------------------- +# flatten +# -------------------------------------------------------------------------- +def flatten(tree: dict) -> list[dict]: + """One flat record per row, with its full ancestry for the CSV.""" + rows: list[dict] = [] + + def walk(node: dict, path: list[str], depth: int) -> None: + kind = node.get("kind") + if kind == "root": + for c in node.get("children", []): + walk(c, path, depth) + return + + num = node.get("number", "") + label = (node.get("title") or node.get("desc") or "").strip() + + if kind == "line": + if node.get("qty") is None and node.get("rate") is None: + return + rows.append({ + "item": num, + "description": label, + "unit": (node.get("unit") or "").strip(), + "qty": node.get("qty"), + "rate": node.get("rate"), + "amount": node.get("amount"), + "rate_only": bool(node.get("rate_only")), + "unit_recognised": _is_known_unit(node.get("unit") or ""), + "level": depth, + "path": " > ".join(path + [f"{num} {label}".strip()]), + "page": node.get("page", 0), + }) + else: + p = path + [f"{num} {label}".strip()] + for c in node.get("children", []): + walk(c, p, depth + 1) + + walk(tree, [], 1) + return rows + + +# -------------------------------------------------------------------------- +# tender workbook +# -------------------------------------------------------------------------- +HEADERS = ["ITEM", "DESCRIPTION", "UNIT", "QTY", "RATE", "AMOUNT"] + + +def populate_template(tree: dict, out_path: str, *, + title: str = "BILL OF QUANTITIES", + contract: str = "", + vat_pct: float = 15.0) -> dict: + """Write a tender-ready workbook with live formulas. + + AMOUNT is a formula, not a value, so the estimator can overwrite a rate and + watch every total in the workbook re-compute. Printed subtotals from the + source are written alongside as a static "as-printed" figure so any + divergence stays visible instead of being silently normalised away. + """ + wb = Workbook() + ws = wb.active + ws.title = "BOQ" + + r = 1 + ws.cell(r, 1, title).font = Font(bold=True, size=14, color=NAVY) + r += 1 + if contract: + ws.cell(r, 1, contract).font = Font(size=10, italic=True) + r += 1 + r += 1 + + header_row = r + for i, h in enumerate(HEADERS, start=1): + c = ws.cell(header_row, i, h) + c.font, c.fill, c.border = HDR_FONT, HDR_FILL, BOX + c.alignment = Alignment(horizontal="center", vertical="center") + r += 1 + + stats = {"lines": 0, "groups": 0, "rate_only": 0, "unknown_units": 0} + subtotal_cells: list[tuple[str, int, str, int]] = [] # label,row,kind,depth + group_rows: list[tuple[str, str, float, int]] = [] # label, number, printed, row + + def emit_line(node: dict) -> None: + nonlocal r + num = node.get("number", "") + desc = (node.get("desc") or "").strip() + unit = (node.get("unit") or "").strip() + qty, rate = node.get("qty"), node.get("rate") + amt = node.get("amount") + rate_only = bool(node.get("rate_only")) + + ws.cell(r, 1, num).font = BODY_FONT + ws.cell(r, 2, desc).font = BODY_FONT + ws.cell(r, 3, unit).font = BODY_FONT + ws.cell(r, 4, qty).number_format = QTY_FMT + if not rate_only and rate is not None: + ws.cell(r, 5, rate).number_format = MONEY + # live formula - the whole point of the deliverable + ws.cell(r, 6, f"=IF(D{r}=\"\",\"\",D{r}*E{r})").number_format = MONEY + elif rate_only: + ws.cell(r, 5, rate).number_format = MONEY + ws.cell(r, 6, "Rate only") + elif amt is not None: + ws.cell(r, 6, amt).number_format = MONEY + + for col in range(1, 7): + ws.cell(r, col).border = BOX + if not _is_known_unit(unit): + ws.cell(r, 3).fill = FLAG_FILL + stats["unknown_units"] += 1 + if rate_only: + stats["rate_only"] += 1 + stats["lines"] += 1 + r += 1 + + def emit_bare(desc: str) -> None: + """A description with no price (placeholder or INCL note).""" + nonlocal r + if not desc: + return + ws.cell(r, 2, desc).font = Font(size=10, color="595959") + for col in range(1, 7): + ws.cell(r, col).border = BOX + r += 1 + + def emit_line(node: dict) -> str: + """Write one priced line; return its AMOUNT cell reference.""" + nonlocal r + num = node.get("number", "") + desc = (node.get("desc") or "").strip() + unit = (node.get("unit") or "").strip() + qty, rate = node.get("qty"), node.get("rate") + amt = node.get("amount") + rate_only = bool(node.get("rate_only")) + + ws.cell(r, 1, num).font = BODY_FONT + ws.cell(r, 2, desc).font = BODY_FONT + ws.cell(r, 3, unit).font = BODY_FONT + if qty is not None: + ws.cell(r, 4, qty).number_format = QTY_FMT + if not rate_only and rate is not None: + ws.cell(r, 5, rate).number_format = MONEY + # live formula - the whole point of the deliverable + ws.cell(r, 6, f'=IF(D{r}="","",D{r}*E{r})').number_format = MONEY + elif rate_only: + if rate is not None: + ws.cell(r, 5, rate).number_format = MONEY + ws.cell(r, 6, "Rate only").font = Font(size=9, italic=True, + color="808080") + elif amt is not None: + ws.cell(r, 6, amt).number_format = MONEY + + for col in range(1, 7): + ws.cell(r, col).border = BOX + if not _is_known_unit(unit): + ws.cell(r, 3).fill = FLAG_FILL + stats["unknown_units"] += 1 + if rate_only: + stats["rate_only"] += 1 + stats["lines"] += 1 + r += 1 + return f"F{r - 1}" + + def depth_of(node: dict) -> int: + n = node.get("number") or "" + return n.count(".") + 1 if n else 1 + + def walk(node: dict) -> list[str]: + """Emit a node depth-first; return its AMOUNT cell reference(s).""" + nonlocal r + kind = node.get("kind") + num = node.get("number", "") + title = (node.get("title") or "").strip() + printed = node.get("subtotal") + + if kind not in ("section", "schedule", "group", "item"): + cells: list[str] = [] + for c in node.get("children", []): + cells.extend(walk(c)) + return cells + + hdr = r + c = ws.cell(hdr, 1, f"{num} {title}".strip()) + c.font = SEC_FONT if kind in ("section", "schedule") else GRP_FONT + if kind in ("section", "schedule"): + for col in range(1, 7): + ws.cell(hdr, col).fill = PatternFill("solid", fgColor=GREY) + r += 1 + + child_cells: list[str] = [] + for ch in node.get("children", []): + ck = ch.get("kind") + if ck == "line": + if ch.get("qty") is None and ch.get("rate") is None: + emit_bare((ch.get("desc") or "").strip()) + continue + child_cells.append(emit_line(ch)) + else: + child_cells.extend(walk(ch)) + + if child_cells: + # SUM(), not a "+" chain: "Rate only" cells hold text, and Excel + # raises #VALUE! when text appears in an addition chain. SUM() + # ignores text arguments, so rate-only lines drop out cleanly. + ws.cell(hdr, 6, f"=SUM({','.join(child_cells)})").number_format = MONEY + elif printed is not None: + ws.cell(hdr, 6, printed).number_format = MONEY + if printed is not None: + # the figure the source actually printed, kept beside the live + # formula so a divergence stays visible instead of being hidden + ws.cell(hdr, 7, printed).number_format = MONEY + ws.cell(hdr, 7).font = Font(size=9, color="808080") + for col in range(1, 8): + ws.cell(hdr, col).border = BOX + ws.cell(hdr, 6).font = TOT_FONT + stats["groups"] += 1 + label = f"{num} {title}".strip() + group_rows.append((label, num, + float(printed) if printed is not None else None, hdr)) + subtotal_cells.append((label, hdr, kind, depth_of(node))) + return [f"F{hdr}"] + + for child in tree.get("children", []): + walk(child) + + # ---- combined summary --------------------------------------------- + r += 1 + ws.cell(r, 1, "COMBINED SUMMARY").font = Font(bold=True, size=12, color=NAVY) + r += 1 + for i, h in enumerate(["", "SCHEDULE / SECTION", "AMOUNT", "AS PRINTED"], + start=1): + c = ws.cell(r, i, h) + c.font, c.fill, c.border = HDR_FONT, HDR_FILL, BOX + r += 1 + # every schedule and every section heads the summary; the numbered groups + # inside them are listed indented beneath, exactly like a Candy summary + # Only schedules and depth-1 groups/sections roll into SUB TOTAL. The + # groups nested inside them are listed indented for reference but must NOT + # be summed again, or every level of the tree is counted several times. + counted: list[int] = [] + first_sum = r + for label, row, kind, d in subtotal_cells: + # Counted: schedules, and depth-1 groups that sit at the root + # ("1 SCHEDULE A : GENERAL"). Sections are depth-1 too but they live + # INSIDE a schedule, which is already counted - including them would + # add the whole tree to itself. + top = (kind == "schedule") or (d <= 1 and kind != "section") + if not top and not (kind in ("group", "section") and d <= 2): + continue + ws.cell(r, 2, f"{'' if top else ' '}{label}") + ws.cell(r, 3, f"=F{row}").number_format = MONEY + ws.cell(r, 2).font = GRP_FONT if top else Font(size=9, color="595959") + if top: + counted.append(r) + r += 1 + last_sum = r - 1 + ws.cell(r, 2, "SUB TOTAL").font = TOT_FONT + ws.cell(r, 3, "=SUM(" + ",".join(f"C{x}" for x in counted) + ")").number_format = MONEY + ws.cell(r, 3).font = TOT_FONT + ws.cell(r, 3).fill = TOT_FILL + sub_row = r + r += 1 + ws.cell(r, 2, f"ADD {vat_pct:g} % VAT").font = TOT_FONT + ws.cell(r, 3, f"=C{sub_row}*{vat_pct / 100}").number_format = MONEY + r += 1 + ws.cell(r, 2, "TOTAL CARRIED FORWARD TO FORM OF OFFER").font = TOT_FONT + ws.cell(r, 3, f"=C{sub_row}+C{r - 1}").number_format = MONEY + ws.cell(r, 3).font = Font(bold=True, size=11) + ws.cell(r, 3).fill = TOT_FILL + r += 2 + + ws.cell(r, 1, "Column G holds the subtotal AS PRINTED in the source PDF, for " + "comparison against the live formula in column F.").font = \ + Font(size=9, italic=True, color="808080") + + widths = {"A": 10, "B": 66, "C": 10, "D": 12, "E": 14, "F": 16, "G": 16} + for col, w in widths.items(): + ws.column_dimensions[col].width = w + ws.freeze_panes = ws.cell(header_row + 1, 1) + + wb.save(out_path) + return stats + + +# -------------------------------------------------------------------------- +# flat CSV +# -------------------------------------------------------------------------- +CSV_COLS = ["item", "description", "unit", "qty", "rate", "amount", + "rate_only", "unit_recognised", "level", "page", "path"] + + +def to_csv(tree: dict, out_path: str) -> int: + rows = flatten(tree) + with open(out_path, "w", newline="", encoding="utf-8-sig") as fh: + w = csv.DictWriter(fh, fieldnames=CSV_COLS, extrasaction="ignore") + w.writeheader() + for row in rows: + w.writerow(row) + return len(rows) diff --git a/boq_engine/profiles.py b/boq_engine/profiles.py new file mode 100644 index 0000000..144cf6d --- /dev/null +++ b/boq_engine/profiles.py @@ -0,0 +1,155 @@ +""" +Per-consultant template profiles. + +This is NOT machine learning and it is not "training". When a consultant's +workbook is first seen, we detect its layout - which row holds the headers, +which column is ITEM / UNIT / QTY / RATE / AMOUNT - and save that map under +the consultant's name. Next time their file arrives we skip detection and +apply the stored map. + +Why that matters: it is deterministic and inspectable. Open a profile and you +can see exactly what the app decided, and correct it. A learned/fuzzy model +would give the same convenience with none of the auditability, and on a +tender document "the model decided" is not an acceptable answer to "where did +this number come from". +""" + +from __future__ import annotations + +import hashlib +import json +import os +import time +from dataclasses import asdict, dataclass, field + +STORE_DIR = os.environ.get( + "BOQ_PROFILES", + os.path.join(os.path.expanduser("~"), ".boq_profiles"), +) +STORE_PATH = os.path.join(STORE_DIR, "profiles.json") + + +def file_fingerprint(path: str) -> str: + """Content hash, so we can tell whether a file matches a known layout.""" + h = hashlib.sha256() + with open(path, "rb") as fh: + for chunk in iter(lambda: fh.read(65536), b""): + h.update(chunk) + return h.hexdigest()[:16] + + +@dataclass +class SheetProfile: + sheet: str + header_row: int + cols: dict[str, int] + data_rows: int + total_rows: int + + +@dataclass +class Profile: + consultant: str + sheets: list[SheetProfile] = field(default_factory=list) + fingerprint: str = "" + saved: str = "" + uses: int = 0 + notes: list[str] = field(default_factory=list) + + def summary(self) -> str: + parts = [] + for s in self.sheets: + cols = ", ".join(f"{k}={v}" for k, v in sorted( + s.cols.items(), key=lambda kv: kv[1])) + parts.append(f"{s.sheet} (hdr row {s.header_row}): {cols}") + return "\n".join(parts) + + +def _load_all() -> dict[str, dict]: + if not os.path.exists(STORE_PATH): + return {} + try: + with open(STORE_PATH, "r", encoding="utf-8") as fh: + return json.load(fh) + except (json.JSONDecodeError, OSError): + return {} + + +def _save_all(data: dict) -> None: + os.makedirs(STORE_DIR, exist_ok=True) + tmp = STORE_PATH + ".tmp" + with open(tmp, "w", encoding="utf-8") as fh: + json.dump(data, fh, indent=1, sort_keys=True) + os.replace(tmp, STORE_PATH) + + +def list_profiles() -> list[Profile]: + out = [] + for name, raw in sorted(_load_all().items()): + try: + p = Profile( + consultant=raw.get("consultant", name), + sheets=[SheetProfile(**s) for s in raw.get("sheets", [])], + fingerprint=raw.get("fingerprint", ""), + saved=raw.get("saved", ""), + uses=raw.get("uses", 0), + notes=raw.get("notes", []), + ) + except TypeError: + continue + out.append(p) + return out + + +def get(consultant: str) -> Profile | None: + raw = _load_all().get(consultant.lower().strip()) + if not raw: + return None + return Profile( + consultant=raw.get("consultant", consultant), + sheets=[SheetProfile(**s) for s in raw.get("sheets", [])], + fingerprint=raw.get("fingerprint", ""), + saved=raw.get("saved", ""), + uses=raw.get("uses", 0), + notes=raw.get("notes", []), + ) + + +def save(consultant: str, layouts: dict, fingerprint: str, + notes: list[str] | None = None) -> Profile: + consultant = consultant.strip() + if not consultant: + raise ValueError("consultant name is required") + prof = Profile( + consultant=consultant, + sheets=[SheetProfile( + sheet=name, + header_row=lay.header_row, + cols=dict(lay.cols), + data_rows=len(lay.data_rows), + total_rows=len(lay.total_rows), + ) for name, lay in layouts.items()], + fingerprint=fingerprint, + saved=time.strftime("%Y-%m-%d %H:%M"), + notes=notes or [], + ) + data = _load_all() + prev = data.get(consultant.lower(), {}) + prof.uses = int(prev.get("uses", 0)) + data[consultant.lower()] = asdict(prof) + _save_all(data) + return prof + + +def touch(consultant: str) -> None: + data = _load_all() + k = consultant.lower().strip() + if k in data: + data[k]["uses"] = int(data[k].get("uses", 0)) + 1 + _save_all(data) + + +def delete(consultant: str) -> None: + data = _load_all() + if data.pop(consultant.lower().strip(), None) is not None: + _save_all(data) diff --git a/boq_engine/template.py b/boq_engine/template.py new file mode 100644 index 0000000..ecec805 --- /dev/null +++ b/boq_engine/template.py @@ -0,0 +1,642 @@ +""" +Populate a CONSULTANT's BOQ workbook from our own Candy PDF export. + +The direction matters and is easy to get backwards: Candy is *our* system, so +the PDF is the source of truth and is always the same shape. The consultant's +Excel is *theirs*, and changes from consultant to consultant. So the hard part +is not reading the document - it is writing into a workbook we did not design +without destroying what is already in it. + +Three jobs: + + 1. LAYOUT DETECTION + Find the header row and work out which column is ITEM / DESCRIPTION / + UNIT / QTY / RATE / AMOUNT. A consultant's PAYMENT column (SABS / PSA + clause references) is detected too and then never touched. + + 2. MAPPING + Line the PDF's lines up with the workbook's rows. The two documents are + near 1:1 but not identical: the workbook truncates descriptions, inserts + spacer rows and adds "TOTAL CARRIED FORWARD" blocks, while the PDF wraps + descriptions across several lines. A sequence alignment over + (item number, description similarity) survives all of that. + + 3. PROOF + A population pass that cannot be checked is a guess. Every source line is + accounted for - written, already-matching, or reported as an exception - + and section totals are written as live formulas that must reproduce the + PDF's own printed subtotal. +""" + +from __future__ import annotations + +import re +import shutil +from dataclasses import dataclass, field +from difflib import SequenceMatcher +from typing import Any + +from openpyxl import load_workbook +from openpyxl.utils import get_column_letter + +# -------------------------------------------------------------------------- +# header synonyms +# -------------------------------------------------------------------------- +HEADER_PATTERNS: dict[str, tuple[str, ...]] = { + "item": ("item", "item no", "item no.", "item number", "no", "no."), + "description": ("description", "bill description", "desc", "particulars", + "description of work"), + "unit": ("unit", "units", "uom"), + "qty": ("qty", "quantity", "bill quantity", "quantities"), + "rate": ("rate", "unit rate", "unit price", "price"), + "amount": ("amount", "total", "value", "sum"), + "payment": ("payment", "payment clause", "clause", "spec", "specification"), +} + +TOTAL_RE = re.compile(r"^\s*total\b", re.I) +BROUGHT_RE = re.compile(r"brought\s+forward", re.I) +SUMMARY_RE = re.compile(r"to\s+(combined\s+)?summary", re.I) + + +def _norm(s: Any) -> str: + return re.sub(r"\s+", " ", str(s or "")).strip().lower() + + +def _cellsafe(v: Any) -> str: + """Normalise a cell value to something comparable across both documents.""" + s = _norm(v) + s = s.replace("ℓ", "l").replace("³", "3").replace("²", "2") + s = re.sub(r"[^a-z0-9./-]+", " ", s) + return re.sub(r"\s+", " ", s).strip() + + +# -------------------------------------------------------------------------- +# layout +# -------------------------------------------------------------------------- +@dataclass +class Layout: + sheet: str + header_row: int + cols: dict[str, int] = field(default_factory=dict) + data_rows: list[int] = field(default_factory=list) + total_rows: list[int] = field(default_factory=list) + notes: list[str] = field(default_factory=list) + + def col(self, name: str) -> int | None: + return self.cols.get(name) + + +def detect_layout(ws) -> Layout: + """Find the header row and map the columns.""" + best: tuple[int, dict[str, int], int] | None = None + for r in range(1, min(ws.max_row, 25) + 1): + cols: dict[str, int] = {} + for c in range(1, min(ws.max_column, 30) + 1): + v = _cellsafe(ws.cell(r, c).value) + if not v or v in cols.values(): + continue + for name, pats in HEADER_PATTERNS.items(): + if name in cols: + continue + if v in pats or any(p in v for p in pats): + cols[name] = c + break + if len(cols) >= 4 and (best is None or len(cols) > len(best[1])): + best = (r, cols, len(cols)) + + if best is None: + raise ValueError( + f"Could not find a header row in sheet '{ws.title}' - " + f"expected 4+ of {', '.join(HEADER_PATTERNS)} in the first 25 rows" + ) + + header_row, cols, _ = best + lay = Layout(sheet=ws.title, header_row=header_row, cols=cols) + + c_item = cols.get("item") + c_desc = cols.get("description") + c_qty = cols.get("qty") + + for r in range(header_row + 1, ws.max_row + 1): + a = ws.cell(r, c_item).value if c_item else None + d = ws.cell(r, c_desc).value if c_desc else None + q = ws.cell(r, c_qty).value if c_qty else None + label = f"{a or ''} {d or ''}".strip() + + if label and TOTAL_RE.match(label): + lay.total_rows.append(r) + continue + # a real data row: something in item, description or qty + if any(x not in (None, "") for x in (a, d)) or q is not None: + lay.data_rows.append(r) + + for name in ("item", "description", "unit", "qty", "rate", "amount"): + if name not in cols: + lay.notes.append(f"template has no '{name}' column - that value will not be written") + if "payment" in cols: + lay.notes.append("PAYMENT column detected and will be left untouched") + return lay + + +# -------------------------------------------------------------------------- +# matching +# -------------------------------------------------------------------------- +@dataclass +class Slot: + row: int + number: str = "" + desc: str = "" + indent: int = 0 + unit: Any = None + qty: Any = None + rate: Any = None + amount: Any = None + _sheet: str = "" + + @property + def key(self) -> str: + return _cellsafe(f"{self.number} {self.desc}") + + +def read_slots(ws, lay: Layout) -> list[Slot]: + ci, cd = lay.col("item"), lay.col("description") + out: list[Slot] = [] + for r in lay.data_rows: + num = str(ws.cell(r, ci).value or "").strip() if ci else "" + desc = str(ws.cell(r, cd).value or "").strip() if cd else "" + if not num and not desc: + continue + # indent level from the leading whitespace the consultant preserved + raw = str(ws.cell(r, cd).value or "") if cd else "" + indent = len(raw) - len(raw.lstrip()) if raw != raw.lstrip() else 0 + out.append(Slot(row=r, number=num, desc=desc, indent=indent)) + return out + + +def _sim(a: Slot, num: str, desc: str) -> float: + """How well does a source line match a template slot? + + Item number is the strong signal, but most of a BOQ is unnumbered a)/b)/c) + sub-lines where the number is blank on BOTH sides. Scoring those as "no + evidence" pushed every sub-line under the threshold, so description has to + carry full weight when neither side has a number. + """ + tnum, snum = _cellsafe(a.number), _cellsafe(num) + tdesc = f"{tnum} {_cellsafe(a.desc)}".strip() + sdesc = f"{snum} {_cellsafe(desc)}".strip() + + if tnum and snum: + if tnum == snum: + num_score = 1.0 + elif tnum in snum or snum in tnum: + num_score = 0.75 + else: + num_score = 0.0 + elif not tnum and not snum: + num_score = 0.5 # neither numbered: description decides + else: + num_score = 0.0 # one side numbered, the other not + + d = SequenceMatcher(None, tdesc, sdesc).ratio() + return round(num_score * 0.6 + d * 0.4, 4) + + +def align(slots: list[Slot], lines: list[dict], threshold: float = 0.45 + ) -> list[tuple[dict, Slot | None, float]]: + """Sequence-align source lines onto template slots. + + A Needleman-Wunsch style DP rather than a greedy pass: a single + mis-detected row would otherwise shift every line after it, which is + exactly the silent-corruption failure this whole tool exists to prevent. + """ + n, m = len(lines), len(slots) + GAP = -0.35 + + def sc(i: int, j: int) -> float: + ln = lines[i] + return _sim(slots[j], ln.get("number", ""), ln.get("desc", "")) + + dp = [[0.0] * (m + 1) for _ in range(n + 1)] + bt = [[0] * (m + 1) for _ in range(n + 1)] + for i in range(1, n + 1): + dp[i][0] = dp[i - 1][0] + GAP + bt[i][0] = 1 + for j in range(1, m + 1): + dp[0][j] = dp[0][j - 1] + GAP + bt[0][j] = 2 + for i in range(1, n + 1): + for j in range(1, m + 1): + diag = dp[i - 1][j - 1] + sc(i - 1, j - 1) + up = dp[i - 1][j] + GAP + left = dp[i][j - 1] + GAP + best = max(diag, up, left) + dp[i][j] = best + bt[i][j] = 0 if best == diag else (1 if best == up else 2) + + out: list[tuple[dict, Slot | None, float]] = [] + i, j = n, m + while i > 0 or j > 0: + mv = bt[i][j] + if mv == 0 and i > 0 and j > 0: + s = sc(i - 1, j - 1) + out.append((lines[i - 1], slots[j - 1] if s >= threshold else None, s)) + i, j = i - 1, j - 1 + elif mv == 1 and i > 0: + out.append((lines[i - 1], None, 0.0)) + i -= 1 + else: + j -= 1 + out.reverse() + return out + + +# -------------------------------------------------------------------------- +# the population pass +# -------------------------------------------------------------------------- +@dataclass +class Finding: + kind: str # 'written' | 'kept' | 'conflict' | 'unmapped' | 'total' | 'info' + sheet: str + row: int + label: str + detail: str + source: str = "" + # How much a human needs to care. This is data, not presentation - the UI + # maps it to a colour, so the classification lives here where it can be + # asserted in tests. + # 'decision' the two documents genuinely disagree; a person must pick + # 'source_gap' OUR pdf has no figure here, theirs is better + # 'source_defect' OUR pdf printed something corrupt + # 'note' worth knowing, no action implied + severity: str = "note" + # For a conflict the UI needs enough to offer a real choice: which cell, + # what is in it now, and what the source wants to put there. + field: str = "" # 'qty' | 'rate' | 'unit' + template_value: Any = None # what the consultant's file holds today + source_value: Any = None # what our Candy PDF holds + + +@dataclass +class Result: + written: int = 0 + kept: int = 0 + findings: list[Finding] = field(default_factory=list) + sheets: list[str] = field(default_factory=list) + layout_notes: list[str] = field(default_factory=list) + + def add(self, *a, **kw) -> None: + self.findings.append(Finding(*a, **kw)) + + @property + def problems(self) -> list[Finding]: + return [f for f in self.findings + if f.kind in ("conflict", "unmapped", "info")] + + +# -------------------------------------------------------------------------- +# source-side flattening: PDF tree -> the list we try to place +# -------------------------------------------------------------------------- +def source_lines(tree: dict, *, with_prices: bool = True) -> list[dict]: + """Every priced line in the Candy PDF, in document order.""" + out: list[dict] = [] + + def walk(node: dict) -> None: + kind = node.get("kind") + if kind == "line": + if node.get("qty") is None and node.get("rate") is None: + return + out.append({ + "number": node.get("number", "") or "", + "desc": (node.get("desc") or "").strip(), + "unit": (node.get("unit") or "").strip(), + "qty": node.get("qty"), + "rate": node.get("rate"), + "amount": node.get("amount"), + "rate_only": bool(node.get("rate_only")), + "page": node.get("page", 0), + }) + return + for c in node.get("children", []): + walk(c) + + walk(tree) + return out + + +def section_totals(tree: dict) -> dict[str, float]: + """Total of the priced lines beneath each section / schedule. + + Deliberately a LEAF roll-up rather than the printed subtotals: the source + prints totals that swallow a neighbouring block (1.6 Temporary Works + includes 1.5 Protection of Services), so summing printed subtotals + double counts. Summing leaves reproduces the printed grand total exactly. + """ + totals: dict[str, float] = {} + + def leaf_sum(node: dict) -> float: + if node.get("kind") == "line": + if node.get("rate_only") or node.get("amount") is None: + return 0.0 + return float(node["amount"]) + return sum(leaf_sum(c) for c in node.get("children", [])) + + def walk(node: dict) -> None: + if node.get("kind") in ("section", "schedule", "root"): + if node.get("kind") in ("section", "schedule"): + label = f"{node.get('number','')} {node.get('title','')}".strip() + totals[label] = round(leaf_sum(node), 2) + for c in node.get("children", []): + walk(c) + + walk(tree) + return totals + + +def _norm_unit(u: str) -> str: + s = _norm(u).replace("ℓ", "l").replace("³", "3").replace("²", "2") + return s.rstrip(".") + + +UNIT_ALIASES = { + "l": "l", "litre": "l", "litres": "l", "l": "l", + "m2": "m2", "m 2": "m2", "sqm": "m2", + "m3": "m3", "m 3": "m3", "cbm": "m3", + "no": "no", "no.": "no", "nr": "no", "number": "no", + "sum": "sum", "sum.": "sum", "ls": "sum", + "p.sum": "psum", "psum": "psum", "ps.sum": "psum", "pc.sum": "pcsum", + "month": "month", "months": "month", "mth": "month", + "t": "t", "tonne": "t", "tonnes": "t", + "m": "m", "m ": "m", + "%": "%", "pct": "%", "?": "?", "": "", +} + + +def units_compatible(a: str, b: str) -> bool: + na, nb = _norm_unit(a), _norm_unit(b) + if na == nb: + return True + ca, cb = UNIT_ALIASES.get(na), UNIT_ALIASES.get(nb) + return ca is not None and ca == cb + + +def populate(consultant_xlsx: str, tree: dict, out_path: str, *, + sheet_hint: str | None = None, + overrides: dict | None = None) -> Result: + """Write our Candy BOQ into a consultant's workbook, in place-safe fashion. + + ONE alignment is run across every sheet, in workbook order, rather than + per sheet. The Candy PDF runs Schedule A then Schedule B; the workbook + does too. Aligning each sheet separately against the whole line list + scatters Schedule B's lines into Schedule A's rows. + + `overrides` lets the consultant settle a disagreement from inside the app: + keys are (sheet, row, field) and the value is what should be written. + A cell with an override is written rather than left alone, and the + conflict is downgraded to 'resolved' so it stops being flagged. + """ + overrides = overrides or {} + res = Result() + res = Result() + wb = load_workbook(consultant_xlsx) # formulas preserved + lines = source_lines(tree) + if not lines: + raise ValueError("no priced lines found in the source tree") + + names = [sheet_hint] if sheet_hint else list(wb.sheetnames) + layouts: dict[str, Layout] = {} + combined: list[tuple[str, Slot]] = [] + + for name in names: + if name not in wb.sheetnames: + continue + ws = wb[name] + try: + lay = detect_layout(ws) + except ValueError as exc: + res.add("info", name, 0, "layout", str(exc)) + continue + layouts[name] = lay + res.layout_notes.extend(f"[{name}] {n}" for n in lay.notes) + for slot in read_slots(ws, lay): + slot._sheet = name + combined.append((name, slot)) + + if not combined: + raise ValueError("no BOQ-shaped sheet found in the workbook") + + pairs = align([sl for _, sl in combined], lines) + res.sheets = list(layouts) + + for ln, slot, score in pairs: + if slot is None: + res.add("unmapped", "-", 0, + f"{ln['number']} {ln['desc']}"[:70], + f"source line (p{ln['page']}) has no row in any template sheet", + source=f"{ln['qty']} {ln['unit']} @ {ln['rate']}", + severity="decision") + continue + + name, r = slot._sheet, slot.row + ws = wb[name] + lay = layouts[name] + c_unit, c_qty = lay.col("unit"), lay.col("qty") + c_rate, c_amt = lay.col("rate"), lay.col("amount") + label = f"{slot.number} {slot.desc}"[:70] + + # build the amount formula from the columns we actually detected - a + # consultant template puts them wherever it likes, and assuming D/E + # silently multiplies UNIT x QTY + if c_qty and c_rate: + ql, rl = get_column_letter(c_qty), get_column_letter(c_rate) + amt_formula = (f'=IF(OR({ql}{{r}}="",{rl}{{r}}=""),"",' + f'{ql}{{r}}*{rl}{{r}})') + else: + amt_formula = None + + # --- unit ------------------------------------------------------ + if c_unit and ln["unit"]: + cur = ws.cell(r, c_unit).value + if cur in (None, ""): + ws.cell(r, c_unit, ln["unit"]) + elif not units_compatible(cur, ln["unit"]): + hint = "" + if _norm_unit(ln["unit"]) == "?": + hint = (" -- Candy printed a corrupt unit character; the " + "template's real unit was kept, not overwritten") + ws.cell(r, c_unit, cur) + elif _norm_unit(cur) == "?": + hint = (" -- template unit is unreadable, source unit " + "written instead") + else: + hint = " -- left as the template has it" + okey = (name, r, "unit") + if okey in overrides: + ws.cell(r, c_unit, overrides[okey]) + res.add("conflict", name, r, label, + f"unit: set to {overrides[okey]!r} by the consultant", + source=str(ln["unit"]), severity="resolved", + field="unit", template_value=cur, + source_value=ln["unit"]) + else: + res.add("conflict", name, r, label, + f"unit: template {cur!r} vs source {ln['unit']!r}{hint}", + source=str(ln["unit"]), + severity="source_defect", + field="unit", template_value=cur, + source_value=ln["unit"]) + + # --- quantity --------------------------------------------------- + if c_qty and ln["qty"] is not None: + cur = ws.cell(r, c_qty).value + if cur in (None, ""): + ws.cell(r, c_qty, ln["qty"]) + elif isinstance(cur, (int, float)) and abs(float(cur) - ln["qty"]) > 1e-6: + okey = (name, r, "qty") + if okey in overrides: + ws.cell(r, c_qty, overrides[okey]) + res.add("conflict", name, r, label, + f"quantity: set to {overrides[okey]} by the consultant", + source=str(ln["qty"]), severity="resolved", + field="qty", template_value=cur, + source_value=ln["qty"]) + else: + if ln["rate_only"] or ln["qty"] == 0: + reason = ("source carries no quantity here (printed 'Rate " + "only'), so the template's own figure was kept") + sev = "source_gap" + else: + reason = ("the two documents disagree; NEITHER was " + "overwritten") + sev = "decision" + res.add("conflict", name, r, label, + f"quantity: template {cur} vs source {ln['qty']} - {reason}", + source=str(ln["qty"]), severity=sev, + field="qty", template_value=cur, + source_value=ln["qty"]) + res.written += 1 + else: + res.kept += 1 + # already correct - still make sure a rate landed + if c_rate and ln["rate"] is not None and not ln["rate_only"]: + if ws.cell(r, c_rate).value in (None, ""): + ws.cell(r, c_rate, ln["rate"]) + if c_amt: + ws.cell(r, c_amt, amt_formula.format(r=r)) + res.written += 1 + continue + + # --- rate + amount ---------------------------------------------- + if c_rate and ln["rate"] is not None: + ws.cell(r, c_rate, ln["rate"]) + if c_amt: + if ln["rate_only"]: + ws.cell(r, c_amt, "Rate only") + res.add("info", name, r, label, + "'Rate only' in source - rate written, amount as text", + source=str(ln["rate"])) + else: + # live formula so the consultant can re-rate and watch the + # total follow. Never a frozen value. + if amt_formula: + ws.cell(r, c_amt, amt_formula.format(r=r)) + res.written += 1 + elif c_rate and ln["rate_only"] and ln["amount"] is None: + if c_amt: + ws.cell(r, c_amt, "Rate only") + + _write_totals(wb, layouts, tree, res) + wb.save(out_path) + return res + + +SUMMARY_REF = re.compile( + r"^=\s*'?([^'!]+)'?!\$?([A-Z]{1,2})\$?(\d+)\s*$") + + +def summary_values(tree: dict) -> list[tuple[str, float]]: + """The totals a COMBINED SUMMARY row expects, in document order. + + A consultant template's summary sheet points at specific cells such as + ='SCHEDULE B - ROADS'!G66, and those cells are blank in the template - so + without this the consultant opens the file and sees zero. The values come + from the source in the same order the summary lists them: each schedule, + then each of its sections. + """ + def leaf(node: dict) -> float: + if node.get("kind") == "line": + if node.get("rate_only") or node.get("amount") is None: + return 0.0 + return float(node["amount"]) + return sum(leaf(c) for c in node.get("children", [])) + + out: list[tuple[str, float]] = [] + for branch in tree.get("children", []): + if branch.get("kind") not in ("group", "schedule", "section"): + continue + if branch.get("kind") != "schedule": + label = f"{branch.get('number','')} {branch.get('title','')}".strip() + val = branch.get("subtotal") + out.append((label, round(float(val), 2) if val is not None + else round(leaf(branch), 2))) + for sec in branch.get("children", []): + if sec.get("kind") != "section": + continue + label = f"{sec.get('number','')} {sec.get('title','')}".strip() + val = sec.get("subtotal") + out.append((label, round(float(val), 2) if val is not None + else round(leaf(sec), 2))) + return out + + +def _write_totals(wb, layouts: dict[str, Layout], tree: dict, + res: Result) -> None: + """Fill the cells the workbook's own summary depends on.""" + values = summary_values(tree) + vi = 0 + + # 1. the summary sheet, if there is one + for name in wb.sheetnames: + if "summary" not in name.lower(): + continue + ws = wb[name] + for r in range(1, min(ws.max_row, 120) + 1): + for c in range(1, min(ws.max_column, 12) + 1): + m = SUMMARY_REF.match(str(ws.cell(r, c).value or "")) + if not m or vi >= len(values): + continue + target_sheet, col_letter, row_no = m.group(1), m.group(2), m.group(3) + if target_sheet not in wb.sheetnames: + continue + label, val = values[vi] + vi += 1 + tgt = wb[target_sheet] + tgt[f"{col_letter}{row_no}"] = val + tgt[f"{col_letter}{row_no}"].number_format = "#,##0.00" + res.add("total", name, r, label, + f"summary row -> {target_sheet}!{col_letter}{row_no} " + f"= R{val:,.2f}") + + # 2. any remaining TOTAL CARRIED FORWARD rows get a live SUM over the data + # rows in their own block, so the workbook still adds up on its own + for name, lay in layouts.items(): + ws = wb[name] + c_amt = lay.col("amount") + if not c_amt or not lay.total_rows: + continue + letter_amt = get_column_letter(c_amt) + for tr in lay.total_rows: + if ws.cell(tr, c_amt).value is not None: + continue # already written by the summary pass + prev = lay.header_row + for r2 in lay.total_rows: + if r2 >= tr: + break + prev = r2 + rows = [r for r in lay.data_rows if prev < r < tr] + if rows: + ws.cell(tr, c_amt, + f"=SUM({letter_amt}{min(rows)}:{letter_amt}{max(rows)})" + ).number_format = "#,##0.00" + res.add("total", name, lay.total_rows[0] if lay.total_rows else 0, + f"{len(lay.total_rows)} TOTAL row(s)", + "remaining TOTAL rows given a live SUM over their own block") diff --git a/boq_engine/theme.py b/boq_engine/theme.py new file mode 100644 index 0000000..b6865f5 --- /dev/null +++ b/boq_engine/theme.py @@ -0,0 +1,190 @@ +""" +Design system for the Organic / Natural style — moss, clay, and unbleached paper. + +The tokens live in `static/theme.css` as CSS custom properties, NOT here. +That is deliberate: injecting a ", unsafe_allow_javascript=False) + + +SEVERITY_CSS = { + "decision": "boq-card-decision", # a person has to make a call + "source_gap": "boq-card-gap", # our pdf has no figure; theirs is better + "source_defect": "boq-card-defect", # our pdf printed something corrupt + "resolved": "boq-card-resolved", + "note": "boq-card-note", +} + +SEVERITY_SWATCH = { + "decision": ("#8C363B", "needs a decision"), + "source_gap": ("#DF8441", "our PDF has no figure - theirs kept"), + "source_defect": ("#FEA944", "our PDF printed something corrupt"), + "resolved": ("#00CD8A", "settled by the consultant"), + "note": ("#00CD8A", "no action needed"), +} + + +# -------------------------------------------------------------------------- +# icons +# -------------------------------------------------------------------------- +# Real inline SVG, not emoji. Emoji render differently on every platform, read +# as emoticons, and cannot be sized or coloured to match the brand. These are +# all 24x24 / 1.75 stroke / round caps so the set looks drawn by one hand. +# Inline rather than a CDN icon font, because this app is meant to work with no +# network - that is the point of keeping it local. +_PATHS = { + "building": ( + '' + '' + '' + '' + ), + "file": ( + '' + '' + ), + "table": ( + '' + '' + ), + "alert": ( + '' + '' + ), + "calculator": ( + '' + '' + '' + ), + "clipboard": ( + '' + '' + ), + "download": ( + '' + '' + ), + "refresh": ( + '' + '' + '' + '' + ), + "check": ( + '' + '' + ), + "shield": ( + '' + ), + "info": ( + '' + '' + ), + "user": ( + '' + '' + ), + "layers": ( + '' + '' + ), +} + + +def icon(name: str, size: int = 18, cls: str = "") -> str: + """An inline SVG icon. Uses currentColor, so it picks up the theme.""" + body = _PATHS.get(name, "") + if not body: + return "" + return ( + f'{body}' + ) + + +def pill(text: str, icon_name: str = "", tone: str = "muted") -> str: + """A quiet inline note, used in place of a full alert block. + + st.warning paints a filled bar and shouts. This is background information + the reader needs once, so it belongs at the volume the legend sits at. + """ + return ( + f'
' + f"{icon(icon_name, 15)}{text}
" + ) + + +def card(title: str, body: str, meta: str = "", severity: str = "note") -> str: + """One finding as a card. Returns HTML so long sentences wrap naturally. + + Streamlit's dataframe has no text-wrapping option - `column_config` in + 1.64 offers width but not wrap - and findings are prose, not figures. A + truncated table cell is exactly the failure this avoids. + """ + cls = SEVERITY_CSS.get(severity, "boq-card-note") + meta_html = (f'
{_html.escape(meta)}
' + if meta else "") + return ( + f'
' + f"{meta_html}" + f'
{_html.escape(title)}
' + f'
{_html.escape(body)}
' + f"
" + ) diff --git a/boq_engine/validate.py b/boq_engine/validate.py new file mode 100644 index 0000000..b00ceff --- /dev/null +++ b/boq_engine/validate.py @@ -0,0 +1,219 @@ +""" +Validation + reconciliation for an extracted BOQ. + +This is the part that actually earns its keep. A BOQ is a priced legal +document: the rates go into a Form of Offer and a contract sum. Silently +mistranscribing one line is a real financial exposure, so every extraction is +re-checked against the document's own arithmetic before it is allowed out. + +Three classes of check: + 1. ARITHMETIC - qty x rate == printed amount, and children sum to the + group's printed subtotal. + 2. STRUCTURE - duplicate / out-of-order / malformed item numbers, and + text that landed in the wrong column. + 3. COMPLETENESS- sections present in the doc but empty, rate-only items, + provisional sums. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +TOL = 0.02 # half a cent + + +@dataclass +class Issue: + severity: str # 'error' | 'warn' | 'info' + code: str + where: str + detail: str + suggested: str = "" + + +@dataclass +class Report: + issues: list[Issue] = field(default_factory=list) + checked_lines: int = 0 + checked_groups: int = 0 + + @property + def errors(self) -> list[Issue]: + return [i for i in self.issues if i.severity == "error"] + + @property + def warnings(self) -> list[Issue]: + return [i for i in self.issues if i.severity == "warn"] + + def as_dict(self) -> dict: + return { + "checked_lines": self.checked_lines, + "checked_groups": self.checked_groups, + "errors": len(self.errors), + "warnings": len(self.warnings), + "issues": [i.__dict__ for i in self.issues], + } + + +def _label(node: dict) -> str: + n = node.get("number") or "" + t = node.get("title") or node.get("desc") or "" + return f"{n} {t}".strip()[:60] + + +def _leaf_rows(node: dict) -> list[dict]: + out = [] + for c in node.get("children", []): + if c.get("kind") in ("line", "item") and ( + c.get("qty") is not None or c.get("rate") is not None + ): + out.append(c) + out.extend(_leaf_rows(c)) + return out + + +def _line_amount(line: dict) -> float: + """What the line contributes to its group total.""" + if line.get("rate_only") or line.get("amount") is None: + return 0.0 + return float(line["amount"]) + + +def _computed_amount(line: dict) -> float | None: + q, r = line.get("qty"), line.get("rate") + if q is None or r is None: + return None + return round(q * r, 2) + + +def _direct_leaves(node: dict) -> list[dict]: + """Priced leaf lines that roll up into THIS group's printed subtotal. + + Recurses through numbered *item* nodes (2.6 -> 2.6.1 -> a)) because a + Candy subtotal covers the whole branch, but stops at nested *groups* so a + section total is never double counted. + """ + out: list[dict] = [] + + def rec(n: dict) -> None: + for c in n.get("children", []): + k = c.get("kind") + if k == "group": + continue # nested group: its own subtotal + if k in ("item", "line"): + kids = [g for g in c.get("children", []) + if g.get("kind") in ("item", "line")] + if kids: + rec(c) + elif c.get("qty") is not None or c.get("rate") is not None: + out.append(c) + continue + if k in ("section", "schedule"): + continue + + rec(node) + return out + + +def validate(tree: dict) -> Report: + rep = Report() + _walk(tree, rep, path="BOQ") + return rep + + +def _child_group_subtotals(node: dict) -> list[dict]: + out = [] + for c in node.get("children", []): + if c.get("kind") == "group" and c.get("subtotal") is not None: + out.append(c) + return out + + +def _reconcile(node: dict) -> tuple[float | None, str, int]: + """Work out what this node's printed subtotal *should* be. + + Candy rolls a total up in one of two ways, so try both and report which + one actually applied: + - 'leaves' : the total covers the priced lines directly beneath it + - 'groups' : the total covers nested numbered groups instead + """ + direct = _direct_leaves(node) + if direct: + return (round(sum(_line_amount(l) for l in direct), 2), + "leaves", len(direct)) + kids = _child_group_subtotals(node) + if kids: + return (round(sum(float(k["subtotal"]) for k in kids), 2), + "groups", len(kids)) + return None, "none", 0 + + +def _walk(node: dict, rep: Report, path: str) -> None: + label = _label(node) or node.get("kind", "") + here = f"{path} > {label}" if label else path + + # ---- group subtotal reconciliation --------------------------------- + if node.get("kind") in ("group", "section", "schedule", "item") \ + and node.get("subtotal") is not None: + rep.checked_groups += 1 + expected, basis, n = _reconcile(node) + if expected is not None: + printed = round(float(node["subtotal"]), 2) + if abs(expected - printed) > TOL: + rep.issues.append(Issue( + "error", "SUBTOTAL_MISMATCH", here, + f"printed {printed:,.2f} but its own {basis} " + f"({'lines' if basis == 'leaves' else 'sub-groups'}) " + f"sum to {expected:,.2f} — difference {printed - expected:,.2f}", + "Confirm against the source before this goes into a Form " + "of Offer; the error propagates to the contract sum.", + )) + + # ---- leaf lines ----------------------------------------------------- + for child in node.get("children", []): + if child.get("kind") == "line": + _check_line(child, rep, here) + else: + _walk(child, rep, here) + + +def _check_line(line: dict, rep: Report, where: str) -> None: + if line.get("qty") is None and line.get("rate") is None: + return + rep.checked_lines += 1 + lab = f"{line.get('number','')} {line.get('desc','')}".strip()[:60] + + calc = _computed_amount(line) + if calc is not None and line.get("amount") is not None and not line.get("rate_only"): + if abs(calc - float(line["amount"])) > TOL: + rep.issues.append(Issue( + "error", "LINE_MISMATCH", where, + f"{lab}: qty {line['qty']} x rate {line['rate']} = {calc:,.2f} " + f"but printed amount is {float(line['amount']):,.2f}", + )) + + if not line.get("unit"): + rep.issues.append(Issue( + "warn", "NO_UNIT", where, + f"{lab}: has a price but no unit - column may have shifted", + )) + + if line.get("rate") is None and line.get("qty") is not None and not line.get("rate_only"): + rep.issues.append(Issue( + "info", "PERCENT_LINE", where, + f"{lab}: qty {line['qty']:,.0f} with no rate (percentage/markup line)", + )) + + if line.get("rate_only"): + rep.issues.append(Issue( + "info", "RATE_ONLY", where, + f"{lab}: priced 'Rate only' (qty {line.get('qty')}) - rate is " + f"required but contributes R0.00 to the total", + )) + + for f in line.get("flags", []): + if f == "empty": + rep.issues.append(Issue( + "warn", "EMPTY_LINE", where, f"{lab}: row has no qty, rate or amount", + )) diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..4100017 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,41 @@ +# MEGA INFRA · BOQ Converter +# +# Deploy on Coolify: New Resource -> Docker Compose -> point it at this repo +# and set the compose file to `docker-compose.yml`. Coolify builds the image +# from the Dockerfile, publishes the port, and puts its own HTTPS in front. + +services: + boq-converter: + build: + context: . + dockerfile: Dockerfile + image: mega-infra/boq-converter:latest + container_name: mega-boq-converter + restart: unless-stopped + + # Coolify routes to the first exposed port. Declaring it explicitly stops + # it guessing from the Dockerfile. + expose: + - "8501" + + environment: + # Where the per-consultant layout profiles are written. MUST be on the + # volume below or every redeploy forgets who is who. + BOQ_PROFILES: /data/profiles + STREAMLIT_BROWSER_GATHER_USAGE_STATS: "false" + TZ: Africa/Johannesburg + + volumes: + # Named volume, so saved consultant profiles survive a rebuild. Without + # this the "remember this consultant's layout" feature resets every deploy. + - boq-profiles:/data/profiles + + healthcheck: + test: ["CMD", "curl", "-fsS", "http://localhost:8501/_stcore/health"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 20s + +volumes: + boq-profiles: diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..20ae195 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +# MEGA INFRA · BOQ Converter +# Direct dependencies only - Streamlit pulls the rest in. +streamlit==1.64.0 +pdfplumber==0.11.10 +openpyxl==3.1.5 diff --git a/run.py b/run.py new file mode 100644 index 0000000..1f12374 --- /dev/null +++ b/run.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +""" +BOQ PDF -> Excel + CSV + + python3 run.py tender.pdf # writes out/BOQ_.xlsx + .csv + python3 run.py tender.pdf --report # also prints the validation report +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import sys + +from boq_engine.extract import extract +from boq_engine.populate import flatten, populate_template, to_csv +from boq_engine.validate import validate + + +def safe(name: str) -> str: + return re.sub(r"[^A-Za-z0-9]+", "_", name).strip("_")[:60] + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("pdf") + ap.add_argument("-o", "--outdir", default="out") + ap.add_argument("--contract", default="") + ap.add_argument("--title", default="BILL OF QUANTITIES") + ap.add_argument("--vat", type=float, default=15.0) + ap.add_argument("--report", action="store_true") + ap.add_argument("--json", dest="json_out") + args = ap.parse_args() + + base = safe(os.path.splitext(os.path.basename(args.pdf))[0]) + os.makedirs(args.outdir, exist_ok=True) + xlsx = os.path.join(args.outdir, f"{base}.xlsx") + csvp = os.path.join(args.outdir, f"{base}.csv") + jsonp = args.json_out or os.path.join(args.outdir, f"{base}.json") + + print(f"reading {args.pdf}") + tree = extract(args.pdf) + rep = validate(tree) + + stats = populate_template(tree, xlsx, title=args.title, + contract=args.contract, vat_pct=args.vat) + nrows = to_csv(tree, csvp) + with open(jsonp, "w", encoding="utf-8") as fh: + json.dump({"tree": tree, "validation": rep.as_dict()}, fh, + indent=1, default=str) + + print(f"\n priced lines {stats['lines']}") + print(f" groups {stats['groups']}") + print(f" 'Rate only' {stats['rate_only']}") + print(f" odd units {stats['unknown_units']} (highlighted in the workbook)") + print(f" csv rows {nrows}") + print(f" validation {len(rep.errors)} error(s), {len(rep.warnings)} warning(s)") + + if rep.errors or args.report: + print("\nVALIDATION") + for i in rep.issues: + if i.severity == "info" and not args.report: + continue + mark = {"error": "ERROR", "warn": "WARN ", "info": "INFO "}[i.severity] + print(f" {mark} {i.code}") + print(f" {i.where}") + print(f" {i.detail}") + + print(f"\nwrote {xlsx}\nwrote {csvp}\nwrote {jsonp}") + return 1 if rep.errors else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/run_app.sh b/run_app.sh new file mode 100755 index 0000000..b0cdfc2 --- /dev/null +++ b/run_app.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +# Start the BOQ converter locally. Port 8501 (not 3000 - something else uses it). +set -e +cd "$(dirname "$0")" +[ -d .venv ] || python3 -m venv .venv +.venv/bin/python -c "import streamlit" 2>/dev/null || .venv/bin/python -m pip install -q streamlit pdfplumber openpyxl +exec .venv/bin/streamlit run app.py --server.port 8501 --server.address 127.0.0.1 diff --git a/static/theme.css b/static/theme.css new file mode 100644 index 0000000..0b087a2 --- /dev/null +++ b/static/theme.css @@ -0,0 +1,449 @@ +/* ========================================================================== + MEGA INFRA — BOQ Converter + Palette sampled from the MEGA INFRASTRUCTURE dashboard. + Warm neutrals are deliberate: #E1E1E1 sits on a warm axis and pairs with + #B3AC99. Pure #EEEEEE would read cold next to #FEA944. + + All tokens live here so the brand can be retuned in one place. + ========================================================================== */ + +:root { + /* surfaces */ + --canvas: #1A202C; /* page background */ + --topbar: #1A212D; /* header strip */ + --surface: #262A37; /* cards + sidebar */ + --surface-inset: #252A36; /* dashed "no data" */ + --surface-raise: #363B49; /* hover, chips */ + --hairline: #2F3442; /* 1px edge + dividers */ + + /* tinted status surfaces */ + --success-fill: #22383E; + --success-edge: #1B534A; + --warn-fill: #3A373C; + --danger-fill: #3A2D38; + --chip: #443D3D; + + /* accents */ + --orange: #FEA944; /* brand, 7.47:1 */ + --orange-soft: #F1A962; /* KPI numerals, 7.21:1 */ + --orange-deep: #DF8441; /* bar gradient */ + --green: #00CD8A; /* checks, links, 6.88:1 */ + --red: #F04141; /* 3.77:1 - large only */ + --red-soft: #FF6B6B; /* 5.15:1 - clears AA */ + --red-deep: #8C363B; + + /* text */ + --text: #E1E1E1; /* 10.93:1 */ + --text-muted: #999999; /* 5.02:1 */ + --text-warm: #B3AC99; /* 6.32:1 */ + --text-link: #00CD8A; /* 6.88:1 */ + + --serif: 'Fraunces', Georgia, serif; + --sans: 'Nunito', -apple-system, BlinkMacSystemFont, sans-serif; + + --edge: 1px solid var(--hairline); +} + +/* ---- base ---------------------------------------------------------------- */ +html, body, .stApp, [data-testid="stAppViewContainer"] { + background: var(--canvas); + color: var(--text); + font-family: var(--sans); +} + +h1, h2, h3, h4, +.stMarkdown h1, .stMarkdown h2, .stMarkdown h3, .stMarkdown h4 { + font-family: var(--serif) !important; + font-weight: 700 !important; + color: var(--text); + letter-spacing: -0.015em; +} + +.stCaption, [data-testid="stCaptionContainer"] { color: var(--text-muted) !important; } + +/* section heading: a hairline rule reads more corporate than bare text */ +h2, .stMarkdown h2 { + position: relative; + padding-bottom: 14px !important; + margin-top: 6px !important; +} +h2::after, .stMarkdown h2::after { + content: ""; + position: absolute; left: 0; bottom: 0; height: 1px; width: 100%; + background: linear-gradient(90deg, var(--orange-deep) 0%, var(--hairline) 26%, transparent 100%); + opacity: .85; +} +h3, .stMarkdown h3 { letter-spacing: .005em; } +a { color: var(--text-link) !important; } +code { background: var(--surface-raise) !important; color: var(--text-warm) !important; } + +/* Streamlit's own top strip */ +header[data-testid="stHeader"] { background: var(--topbar); } +[data-testid="stToolbar"] { background: transparent; } +[data-testid="stStatusWidget"] { color: var(--text-muted); } + +/* ---- surfaces: cards ------------------------------------------------------ */ +div[data-testid="stVerticalBlockBorderWrapper"], +div[data-testid="stVerticalBlock"] > div[data-testid="stElementContainer"] { + border-radius: 18px; +} + +/* ---- KPI tiles ------------------------------------------------------------- + Hand-built rather than st.metric: a row of four bare numbers is the generic + part of the page, and st.metric's label is plain text so it cannot carry an + icon. */ +.boq-kpi { + position: relative; + background: linear-gradient(170deg, #2C3140 0%, var(--surface) 58%); + border: var(--edge); + border-radius: 18px; + padding: 16px 20px 18px; + overflow: hidden; + height: 100%; + transition: transform .3s cubic-bezier(.2,.7,.3,1), + border-color .3s ease, box-shadow .3s ease; +} +.boq-kpi::before { + content: ""; + position: absolute; inset: 0 0 auto 0; height: 2px; + background: linear-gradient(90deg, transparent, rgba(254,169,68,.6), transparent); + opacity: .75; +} +.boq-kpi:hover { + transform: translateY(-3px); + border-color: rgba(254,169,68,.3); + box-shadow: 0 16px 36px -20px rgba(0,0,0,.9); +} +.boq-kpi-head { + display: flex; align-items: center; gap: 8px; + margin-bottom: 9px; +} +.boq-kpi-icon { + display: inline-flex; align-items: center; justify-content: center; + width: 26px; height: 26px; border-radius: 8px; flex: none; + color: var(--orange); + background: rgba(254,169,68,.1); + border: 1px solid rgba(254,169,68,.18); +} +.boq-kpi-label { + font-family: var(--sans); font-size: .74rem; font-weight: 700; + letter-spacing: .07em; text-transform: uppercase; color: var(--text-muted); +} +.boq-kpi-value { + font-family: var(--serif); font-size: 2.15rem; font-weight: 700; + line-height: 1.05; letter-spacing: -.015em; color: var(--orange); +} +.boq-kpi-warn .boq-kpi-icon { color: var(--orange-soft); background: rgba(223,132,65,.12); border-color: rgba(223,132,65,.22); } +.boq-kpi-warn .boq-kpi-value { color: var(--orange-soft); } +.boq-kpi-ok .boq-kpi-icon { color: var(--green); background: rgba(0,205,138,.1); border-color: rgba(0,205,138,.2); } +.boq-kpi-ok .boq-kpi-value { color: var(--text); } + +/* ---- field labels above the uploaders ------------------------------------- */ +.boq-field-label { + display: flex; align-items: center; gap: 8px; + font-family: var(--sans); font-size: .8rem; font-weight: 700; + letter-spacing: .05em; text-transform: uppercase; + color: var(--text-muted); margin: 0 0 8px 2px; +} +.boq-field-label .boq-icon { color: var(--orange-deep); } +.boq-field-label b { color: var(--orange); font-weight: 800; } +.boq-field-label em { font-style: normal; text-transform: none; + letter-spacing: .02em; color: var(--text-muted); } + +/* ---- section headings ------------------------------------------------------ */ +.boq-section-title { + display: flex; align-items: center; gap: 12px; + font-family: var(--serif); font-size: 1.62rem; font-weight: 700; + letter-spacing: -.015em; color: var(--text); + margin: 30px 0 18px 0; padding-bottom: 15px; + border-bottom: 1px solid var(--hairline); + position: relative; +} +.boq-section-title::after { + content: ""; + position: absolute; left: 0; bottom: -1px; height: 1px; width: 190px; + background: linear-gradient(90deg, var(--orange-deep), transparent); +} +.boq-section-title .boq-icon { color: var(--orange); } +.boq-section-sub { + display: flex; align-items: center; gap: 8px; + font-family: var(--sans); font-size: .8rem; font-weight: 700; + letter-spacing: .06em; text-transform: uppercase; + color: var(--text-muted); margin: 22px 0 10px 0; +} +.boq-section-sub .boq-icon { color: var(--orange-deep); } + +/* ---- buttons -------------------------------------------------------------- */ +.stButton > button, +.stDownloadButton > button { + font-family: var(--sans); + font-weight: 700; + font-size: 0.95rem; + border-radius: 999px; + min-height: 46px; + padding: 0 30px; + border: 1px solid transparent; + transition: transform 0.25s ease, filter 0.25s ease, background 0.25s ease; +} +.stButton > button[kind="primary"], +.stDownloadButton > button[kind="primary"] { + background: var(--orange); + color: var(--canvas); + border-color: var(--orange); +} +.stButton > button[kind="primary"]:hover, +.stDownloadButton > button[kind="primary"]:hover { + filter: brightness(1.08); + transform: translateY(-1px); +} +.stButton > button[kind="primary"]:active, +.stDownloadButton > button[kind="primary"]:active { transform: scale(0.98); } +.stButton > button[kind="secondary"] { + background: var(--surface); + color: var(--orange); + border: 1px solid var(--orange-deep); +} +.stButton > button[kind="secondary"]:hover { background: var(--surface-raise); } +.stDownloadButton > button[kind="secondary"] { + background: var(--surface); + color: var(--orange); + border: 1px solid var(--orange-deep); +} +.stDownloadButton > button[kind="secondary"]:hover { + background: var(--surface-raise); + transform: translateY(-1px); +} + +/* ---- inputs --------------------------------------------------------------- */ +[data-testid="stFileUploader"] section, +[data-testid="stTextInput"] input, +[data-testid="stSelectbox"] > div, +[data-testid="stNumberInput"] [data-baseweb="input"] { + border-radius: 12px !important; + background: var(--surface-inset) !important; + border: var(--edge) !important; + color: var(--text) !important; + transition: border-color 0.25s ease, box-shadow 0.25s ease; +} +[data-testid="stTextInput"] input:focus, +[data-testid="stSelectbox"] > div:focus-within, +[data-testid="stNumberInput"] [data-baseweb="input"]:focus-within { + border-color: var(--orange) !important; + box-shadow: 0 0 0 3px rgba(254, 169, 68, 0.22) !important; +} +[data-testid="stFileUploader"] section:hover { border-color: var(--orange-deep) !important; } +[data-testid="stFileUploader"] small, +[data-testid="stWidgetLabel"] p, +label[data-testid="stWidgetLabel"] { color: var(--text-muted) !important; } +[data-testid="stFileUploader"] [data-testid="stFileUploaderFile"] { color: var(--text) !important; } + +/* ---- sidebar -------------------------------------------------------------- */ +section[data-testid="stSidebar"] { + background: var(--surface); + border-right: var(--edge); +} +section[data-testid="stSidebar"] .block-container { padding-top: 2.2rem; } +section[data-testid="stSidebar"] h3 { font-family: var(--serif); color: var(--text); } +section[data-testid="stSidebar"] .stCaption { color: var(--text-muted) !important; } + +/* ---- tabs ----------------------------------------------------------------- */ +[data-baseweb="tab-list"] { gap: 6px; background: transparent; } +[data-baseweb="tab"] { + font-family: var(--sans); + font-weight: 700; + font-size: 0.92rem; + color: var(--text-muted); + border-radius: 10px; + padding: 8px 18px; + transition: background 0.25s ease, color 0.25s ease; +} +[data-baseweb="tab"]:hover { background: var(--surface-raise); color: var(--text); } +[data-baseweb="tab"][aria-selected="true"] { + background: var(--surface-raise); + color: var(--orange); +} +[data-baseweb="tab-highlight"], [data-baseweb="tab-border"] { display: none; } + +/* ---- alerts --------------------------------------------------------------- + Streamlit paints its own colour on an inner div, so that inner layer has + to be made transparent or it covers the brand surfaces underneath. */ +[data-testid="stAlert"] { + border-radius: 16px; + border: var(--edge); + background: var(--surface) !important; + color: var(--text) !important; +} +[data-testid^="stAlertContent"] { + background: transparent !important; + color: var(--text) !important; + width: 100%; +} +[data-testid="stAlert"] p, +[data-testid="stAlert"] p *, +[data-testid="stAlert"] span, +[data-testid="stAlert"] strong { color: inherit !important; } +[data-testid="stAlert"] a { color: var(--text-link) !important; } + +[data-testid="stAlertContentSuccess"] { + background: var(--success-fill) !important; + border-left: 3px solid var(--success-edge); +} +[data-testid="stAlertContentSuccess"] p { color: var(--green) !important; } +[data-testid="stAlertContentInfo"] { background: var(--surface-inset) !important; } +[data-testid="stAlertContentInfo"] p { color: var(--text) !important; } +[data-testid="stAlertContentWarning"] { background: var(--warn-fill) !important; } +[data-testid="stAlertContentWarning"] p { color: var(--orange-soft) !important; } +[data-testid="stAlertContentError"] { background: var(--danger-fill) !important; } +[data-testid="stAlertContentError"] p { color: var(--red-soft) !important; } + +/* ---- brand lockup --------------------------------------------------------- */ +.boq-brand { + display: flex; align-items: center; gap: 16px; + margin: 0 0 22px 0; +} +.boq-brand-mark { + display: flex; align-items: center; justify-content: center; + width: 56px; height: 56px; border-radius: 16px; + color: var(--canvas); + background: linear-gradient(145deg, var(--orange) 0%, var(--orange-deep) 100%); + box-shadow: 0 8px 22px -10px rgba(254, 169, 68, .75); + flex: none; +} +.boq-brand-mark .boq-icon { vertical-align: 0; } +.boq-brand-text { display: flex; flex-direction: column; gap: 3px; } +.boq-brand-name { + font-family: var(--serif); font-size: 2.15rem; font-weight: 800; + letter-spacing: -0.02em; line-height: 1.05; color: var(--text); +} +.boq-brand-sub { + font-family: var(--sans); font-size: 0.86rem; font-weight: 500; + color: var(--text-muted); line-height: 1.4; +} + +/* ---- icons ---------------------------------------------------------------- */ +.boq-icon { + vertical-align: -0.18em; + flex: none; + stroke-linecap: round; + stroke-linejoin: round; +} + +/* ---- quiet inline note ---------------------------------------------------- + Replaces the st.warning block. A filled alert bar shouts at the reader for + something that is background information; this sits at the same volume as + the legend it belongs next to. */ +.boq-pill { + display: flex; align-items: flex-start; gap: 9px; + font-family: var(--sans); font-size: 0.845rem; line-height: 1.55; + color: var(--text-muted); + padding: 9px 14px; margin: 2px 0 14px 0; + border-radius: 12px; + background: rgba(38, 42, 55, .55); + border: 1px solid var(--hairline); +} +.boq-pill .boq-icon { margin-top: 2px; color: var(--text-muted); } +.boq-pill-ok { background: rgba(34, 56, 62, .5); border-color: rgba(27,83,74,.6); } +.boq-pill-ok .boq-icon { color: var(--green); } +.boq-pill-warn { background: rgba(58, 55, 60, .5); border-color: rgba(223,132,65,.4); } +.boq-pill-warn .boq-icon { color: var(--orange-soft); } +.boq-pill-info .boq-icon { color: var(--orange); } + +/* ---- finding cards --------------------------------------------------------- + Findings are prose somebody has to read and act on, so they get cards + rather than a table cell that truncates. Long sentences wrap here. */ +.boq-card { + background: var(--surface); + border: var(--edge); + border-left: 3px solid var(--orange-deep); + border-radius: 16px; + padding: 17px 22px; + margin-bottom: 10px; + transition: border-color 0.25s ease, background 0.25s ease; +} +.boq-card:hover { background: var(--surface-raise); } +.boq-card-meta { + font-family: var(--sans); + font-size: 0.7rem; + color: var(--text-muted); + text-transform: uppercase; + letter-spacing: 0.09em; + font-weight: 700; + margin-bottom: 6px; +} +.boq-card-title { + font-family: var(--serif); + font-weight: 700; + font-size: 0.98rem; + color: var(--text); + margin-bottom: 5px; + overflow-wrap: anywhere; +} +.boq-card-body { + font-family: var(--sans); + font-size: 0.9rem; + line-height: 1.6; + color: var(--text-muted); + white-space: normal; + overflow-wrap: anywhere; +} +.boq-card-resolved { border-left-color: var(--green); } + +/* severity - decided in Python (Finding.severity), not inferred from text. + A red edge must mean "a person must choose", full stop. */ +.boq-card-decision { border-left-color: var(--red-deep); } +.boq-card-gap { border-left-color: var(--orange-deep); } +.boq-card-defect { border-left-color: var(--orange); } +.boq-card-note { border-left-color: var(--green); } + +/* ---- resolution control --------------------------------------------------- */ +.boq-resolve { + margin-top: 12px; + padding-top: 12px; + border-top: 1px dashed var(--hairline); +} +.boq-legend { + display: flex; gap: 22px; flex-wrap: wrap; + font-size: 0.8rem; color: var(--text-muted); margin: 4px 0 16px 0; +} +.boq-legend b { font-weight: 700; } +.boq-swatch { + display: inline-block; width: 10px; height: 10px; + border-radius: 3px; margin-right: 7px; vertical-align: middle; +} + +/* ---- dataframes ----------------------------------------------------------- */ +[data-testid="stDataFrame"] { + border: var(--edge); + border-radius: 14px; + overflow: hidden; + background: var(--surface); +} +[data-testid="stDataFrame"] * { font-family: var(--sans); } +[data-testid="stDataFrame"] .glide-cell, +[data-testid="stDataFrame"] .glide-cell p, +[data-testid="stDataFrame"] div[role="gridcell"] { + white-space: normal !important; + line-height: 1.55; +} +[data-testid="stDataFrame"] div[role="columnheader"] { background: var(--surface-raise) !important; } +[data-testid="stDataFrame"] div[role="columnheader"] * { color: var(--text) !important; } + +/* ---- expanders ------------------------------------------------------------ */ +[data-testid="stExpander"] { + border: var(--edge); + border-radius: 12px; + background: var(--surface); + overflow: hidden; +} +[data-testid="stExpander"] summary { color: var(--text) !important; } +[data-testid="stExpander"] summary:hover { color: var(--orange) !important; } + +/* ---- scrollbars ----------------------------------------------------------- */ +::-webkit-scrollbar { width: 11px; height: 11px; } +::-webkit-scrollbar-track { background: var(--canvas); } +::-webkit-scrollbar-thumb { + background: var(--surface-raise); + border-radius: 999px; + border: 3px solid var(--canvas); +} +::-webkit-scrollbar-thumb:hover { background: var(--orange-deep); }