416 lines
16 KiB
Python
416 lines
16 KiB
Python
|
#!/usr/bin/env python3
|
|||
|
|
"""
|
|||
|
|
BOQ converter - local web app.
|
|||
|
|
|
|||
|
|
Drop in the Candy PDF (ours) and the consultant's Excel (theirs); get back the
|
|||
|
|
consultant's workbook populated, plus a proof report of exactly what was
|
|||
|
|
written, what was already correct, and what the two documents disagree about.
|
|||
|
|
|
|||
|
|
streamlit run app.py --server.port 8501
|
|||
|
|
|
|||
|
|
Runs entirely on this machine. Nothing is uploaded anywhere.
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
from __future__ import annotations
|
|||
|
|
|
|||
|
|
import os
|
|||
|
|
import sys
|
|||
|
|
import html as _html
|
|||
|
|
import re
|
|||
|
|
import tempfile
|
|||
|
|
|
|||
|
|
import streamlit as st
|
|||
|
|
|
|||
|
|
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|||
|
|
|
|||
|
|
from openpyxl import load_workbook # noqa: E402
|
|||
|
|
|
|||
|
|
from boq_engine import profiles, theme # noqa: E402
|
|||
|
|
from boq_engine.extract import extract # noqa: E402
|
|||
|
|
from boq_engine.template import ( # noqa: E402
|
|||
|
|
detect_layout, populate, summary_values,
|
|||
|
|
)
|
|||
|
|
from boq_engine.validate import validate # noqa: E402
|
|||
|
|
|
|||
|
|
def _fmt(v) -> str:
|
|||
|
|
if isinstance(v, float):
|
|||
|
|
return f"{v:,.6g}"
|
|||
|
|
return str(v)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _strip_md(t: str) -> str:
|
|||
|
|
"""Cell text can carry markdown bold from a wrapped description."""
|
|||
|
|
return re.sub(r"\*\*(.+?)\*\*", r"\1", t or "").strip()
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _coerce(text: str):
|
|||
|
|
"""A typed value should land as a number when it looks like one."""
|
|||
|
|
t = text.strip().replace(",", "")
|
|||
|
|
if not t:
|
|||
|
|
return text
|
|||
|
|
try:
|
|||
|
|
f = float(t)
|
|||
|
|
return int(f) if f.is_integer() else f
|
|||
|
|
except ValueError:
|
|||
|
|
return text
|
|||
|
|
|
|||
|
|
|
|||
|
|
st.set_page_config(page_title="MEGA INFRA · BOQ Converter",
|
|||
|
|
page_icon="M", layout="wide")
|
|||
|
|
|
|||
|
|
theme.inject()
|
|||
|
|
|
|||
|
|
@st.cache_data(show_spinner="Reading the Candy PDF…")
|
|||
|
|
def read_pdf(data: bytes):
|
|||
|
|
tmp = tempfile.NamedTemporaryFile(suffix=".pdf", delete=False)
|
|||
|
|
tmp.write(data)
|
|||
|
|
tmp.close()
|
|||
|
|
try:
|
|||
|
|
return extract(tmp.name)
|
|||
|
|
finally:
|
|||
|
|
os.unlink(tmp.name)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _fp(data: bytes) -> str:
|
|||
|
|
import hashlib
|
|||
|
|
return hashlib.sha256(data).hexdigest()[:16]
|
|||
|
|
|
|||
|
|
|
|||
|
|
def money(v) -> str:
|
|||
|
|
if isinstance(v, (int, float)):
|
|||
|
|
return f"R{v:,.2f}"
|
|||
|
|
return str(v) if v not in (None, "") else "—"
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ---------------------------------------------------------------- header
|
|||
|
|
st.markdown(
|
|||
|
|
'<div class="boq-brand">'
|
|||
|
|
f'<span class="boq-brand-mark">{theme.icon("building", 30)}</span>'
|
|||
|
|
'<span class="boq-brand-text">'
|
|||
|
|
'<span class="boq-brand-name">MEGA INFRA</span>'
|
|||
|
|
'<span class="boq-brand-sub">BOQ Converter · '
|
|||
|
|
'Candy PDF (ours) → consultant\'s Excel (theirs) '
|
|||
|
|
' · nothing leaves this machine</span>'
|
|||
|
|
'</span></div>',
|
|||
|
|
unsafe_allow_html=True)
|
|||
|
|
|
|||
|
|
known = profiles.list_profiles()
|
|||
|
|
with st.sidebar:
|
|||
|
|
st.subheader("Consultant profile")
|
|||
|
|
if known:
|
|||
|
|
names = ["— new / not set —"] + [p.consultant for p in known]
|
|||
|
|
pick = st.selectbox("Known consultants", names, index=0)
|
|||
|
|
else:
|
|||
|
|
pick = "— new / not set —"
|
|||
|
|
st.caption("No profiles saved yet.")
|
|||
|
|
|
|||
|
|
consultant = st.text_input(
|
|||
|
|
"Consultant name",
|
|||
|
|
value="" if pick.startswith("—") else pick,
|
|||
|
|
placeholder="e.g. Civil Matters",
|
|||
|
|
help="Saved so next time their layout is recognised automatically.",
|
|||
|
|
)
|
|||
|
|
if consultant and not pick.startswith("—"):
|
|||
|
|
prof = profiles.get(pick)
|
|||
|
|
if prof:
|
|||
|
|
st.caption(f"Saved {prof.saved} · used {prof.uses}×")
|
|||
|
|
with st.expander("What we learned about their layout"):
|
|||
|
|
st.code(prof.summary(), language="text")
|
|||
|
|
|
|||
|
|
# ---------------------------------------------------------------- inputs
|
|||
|
|
c1, c2 = st.columns(2)
|
|||
|
|
with c1:
|
|||
|
|
st.markdown(
|
|||
|
|
f'<div class="boq-field-label">{theme.icon("file", 15)}'
|
|||
|
|
f'<span><b>1</b> Candy PDF <em>(ours)</em></span></div>',
|
|||
|
|
unsafe_allow_html=True)
|
|||
|
|
pdf_file = st.file_uploader(" ", type=["pdf"], label_visibility="collapsed")
|
|||
|
|
with c2:
|
|||
|
|
st.markdown(
|
|||
|
|
f'<div class="boq-field-label">{theme.icon("table", 15)}'
|
|||
|
|
f'<span><b>2</b> Consultant Excel <em>(theirs)</em></span></div>',
|
|||
|
|
unsafe_allow_html=True)
|
|||
|
|
xls_file = st.file_uploader(" ", type=["xlsx", "xlsm"],
|
|||
|
|
label_visibility="collapsed")
|
|||
|
|
|
|||
|
|
st.markdown("---")
|
|||
|
|
|
|||
|
|
if not (pdf_file and xls_file):
|
|||
|
|
st.markdown(theme.pill(
|
|||
|
|
"Drop in <b>both</b> files to begin — the Candy PDF and the "
|
|||
|
|
"consultant's Excel.", "info", "info"), unsafe_allow_html=True)
|
|||
|
|
st.stop()
|
|||
|
|
|
|||
|
|
# ---------------------------------------------------------------- run
|
|||
|
|
out_dir = tempfile.mkdtemp(prefix="boq_out_")
|
|||
|
|
base = os.path.splitext(os.path.splitext(pdf_file.name)[0])[0]
|
|||
|
|
out_xlsx = os.path.join(out_dir, f"{base}_POPULATED.xlsx")
|
|||
|
|
|
|||
|
|
with st.spinner("Populating…"):
|
|||
|
|
tree = read_pdf(pdf_file.getvalue())
|
|||
|
|
src_rep = validate(tree)
|
|||
|
|
xls_tmp = os.path.join(out_dir, "in.xlsx")
|
|||
|
|
with open(xls_tmp, "wb") as fh:
|
|||
|
|
fh.write(xls_file.getvalue())
|
|||
|
|
|
|||
|
|
# Re-read the staged choices from the previous run's widgets. This happens
|
|||
|
|
# BEFORE populate() so a regenerate writes the consultant's decisions
|
|||
|
|
# straight into the workbook - no second manual step.
|
|||
|
|
# The dispute catalogue is built on the first pass and then kept. Once a
|
|||
|
|
# cell is settled it drops out of the findings list, so re-deriving it each
|
|||
|
|
# run would lose the very rows the consultant already decided about.
|
|||
|
|
catalogue = st.session_state.get("disputes") or {}
|
|||
|
|
if not catalogue:
|
|||
|
|
probe = populate(xls_tmp, tree, os.path.join(out_dir, "probe.xlsx"))
|
|||
|
|
catalogue = {
|
|||
|
|
(f.sheet, f.row, f.field): {
|
|||
|
|
"label": f.label, "detail": f.detail,
|
|||
|
|
"source_value": f.source_value,
|
|||
|
|
"template_value": f.template_value,
|
|||
|
|
"severity": f.severity,
|
|||
|
|
}
|
|||
|
|
for f in probe.findings if f.kind == "conflict"
|
|||
|
|
}
|
|||
|
|
st.session_state["disputes"] = catalogue
|
|||
|
|
|
|||
|
|
overrides: dict = {}
|
|||
|
|
if st.session_state.get("dirty"):
|
|||
|
|
for key, val in list(st.session_state.items()):
|
|||
|
|
if not key.startswith("choice::") or key.endswith("::custom"):
|
|||
|
|
continue
|
|||
|
|
parts = key.split("::")
|
|||
|
|
if len(parts) != 4:
|
|||
|
|
continue
|
|||
|
|
cell = (parts[1], int(parts[2]), parts[3])
|
|||
|
|
if cell not in catalogue:
|
|||
|
|
continue
|
|||
|
|
custom = st.session_state.get(f"{key}::custom", "").strip()
|
|||
|
|
if custom:
|
|||
|
|
overrides[cell] = _coerce(custom)
|
|||
|
|
elif str(val).startswith("Use our"):
|
|||
|
|
overrides[cell] = catalogue[cell]["source_value"]
|
|||
|
|
st.session_state["dirty"] = False
|
|||
|
|
st.session_state["applied"] = len(overrides)
|
|||
|
|
|
|||
|
|
result = populate(xls_tmp, tree, out_xlsx, overrides=overrides)
|
|||
|
|
|
|||
|
|
# remember this consultant's layout for next time
|
|||
|
|
if consultant:
|
|||
|
|
wb = load_workbook(xls_tmp)
|
|||
|
|
lays = {}
|
|||
|
|
for nm in wb.sheetnames:
|
|||
|
|
try:
|
|||
|
|
lays[nm] = detect_layout(wb[nm])
|
|||
|
|
except ValueError:
|
|||
|
|
pass
|
|||
|
|
if lays:
|
|||
|
|
profiles.save(consultant, lays, _fp(xls_file.getvalue()))
|
|||
|
|
profiles.touch(consultant)
|
|||
|
|
|
|||
|
|
# ---------------------------------------------------------------- results
|
|||
|
|
def kpi(icon_name: str, label: str, value, tone: str = "orange") -> str:
|
|||
|
|
"""A KPI tile. Hand-built so it can carry a real icon - st.metric's label
|
|||
|
|
is plain text, and a row of four plain numbers is the generic-looking part
|
|||
|
|
of this page."""
|
|||
|
|
cls = f"boq-kpi boq-kpi-{tone}"
|
|||
|
|
return (
|
|||
|
|
f'<div class="{cls}">'
|
|||
|
|
f'<div class="boq-kpi-head">'
|
|||
|
|
f'<span class="boq-kpi-icon">{theme.icon(icon_name, 16)}</span>'
|
|||
|
|
f'<span class="boq-kpi-label">{_html.escape(label)}</span>'
|
|||
|
|
f"</div>"
|
|||
|
|
f'<div class="boq-kpi-value">{value}</div>'
|
|||
|
|
f"</div>"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
|
|||
|
|
k1, k2, k3, k4 = st.columns(4)
|
|||
|
|
_open_n = len([f for f in result.findings
|
|||
|
|
if f.kind == "conflict" and f.severity != "resolved"])
|
|||
|
|
_unmapped_n = len([f for f in result.findings if f.kind == "unmapped"])
|
|||
|
|
k1.markdown(kpi("check", "Values written", result.written), unsafe_allow_html=True)
|
|||
|
|
k2.markdown(kpi("shield", "Already correct", result.kept), unsafe_allow_html=True)
|
|||
|
|
k3.markdown(kpi("alert", "Open disagreements", _open_n,
|
|||
|
|
"warn" if _open_n else "ok"), unsafe_allow_html=True)
|
|||
|
|
k4.markdown(kpi("info", "Unmapped source lines", _unmapped_n,
|
|||
|
|
"warn" if _unmapped_n else "ok"), unsafe_allow_html=True)
|
|||
|
|
|
|||
|
|
applied = st.session_state.get("applied", 0)
|
|||
|
|
_tail = (f"with <b>{applied}</b> of your decisions written in. "
|
|||
|
|
if applied else "")
|
|||
|
|
st.markdown(theme.pill(
|
|||
|
|
f"<b>Ready to send.</b> {_tail}Your file was populated using its own "
|
|||
|
|
f"formulas and layout — download it and send it with the Candy PDF.",
|
|||
|
|
"check", "ok"), unsafe_allow_html=True)
|
|||
|
|
|
|||
|
|
with open(out_xlsx, "rb") as fh:
|
|||
|
|
st.download_button("Download populated workbook",
|
|||
|
|
fh.read(),
|
|||
|
|
file_name=os.path.basename(out_xlsx),
|
|||
|
|
mime="application/vnd.openxmlformats-officedocument"
|
|||
|
|
".spreadsheetml.sheet")
|
|||
|
|
|
|||
|
|
# ---------------------------------------------------------------- proof
|
|||
|
|
st.markdown("---")
|
|||
|
|
st.markdown(
|
|||
|
|
f'<div class="boq-section-title">{theme.icon("clipboard", 20)}'
|
|||
|
|
f'<span>Proof — what happened, line by line</span></div>',
|
|||
|
|
unsafe_allow_html=True)
|
|||
|
|
|
|||
|
|
conflicts = [f for f in result.findings if f.kind == "conflict"]
|
|||
|
|
unmapped = [f for f in result.findings if f.kind == "unmapped"]
|
|||
|
|
totals = [f for f in result.findings if f.kind == "total"]
|
|||
|
|
notes = [f for f in result.findings if f.kind == "info"]
|
|||
|
|
|
|||
|
|
SEVERITY_SWATCH = {
|
|||
|
|
"decision": ("#8C363B", "needs a decision"),
|
|||
|
|
"source_gap": ("#DF8441", "our PDF has no figure - theirs kept"),
|
|||
|
|
"source_defect": ("#FEA944", "our PDF printed something corrupt"),
|
|||
|
|
"resolved": ("#00CD8A", "settled by the consultant"),
|
|||
|
|
"note": ("#00CD8A", "no action needed"),
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _legend() -> str:
|
|||
|
|
seen, parts = set(), []
|
|||
|
|
for sev, (col, label) in SEVERITY_SWATCH.items():
|
|||
|
|
if sev == "note" or label in seen:
|
|||
|
|
continue
|
|||
|
|
seen.add(label)
|
|||
|
|
parts.append(
|
|||
|
|
f'<span><span class="boq-swatch" style="background:{col}"></span>'
|
|||
|
|
f"{label}</span>")
|
|||
|
|
return f'<div class="boq-legend">{"".join(parts)}</div>'
|
|||
|
|
|
|||
|
|
|
|||
|
|
# NOTE: st.tabs / st.button / st.download_button render their labels as PLAIN
|
|||
|
|
# TEXT. Inline SVG there leaks to the user as literal markup, and emoji read as
|
|||
|
|
# emoticons. So those stay typographic; the real icons live where HTML is
|
|||
|
|
# actually rendered - the brand mark, KPI tiles, pills and section headings.
|
|||
|
|
tab_conf, tab_tot, tab_src = st.tabs([
|
|||
|
|
f"Disagreements ({len(conflicts)})",
|
|||
|
|
f"Totals written ({len(totals)})",
|
|||
|
|
f"Source BOQ checks ({len(src_rep.errors)})",
|
|||
|
|
])
|
|||
|
|
|
|||
|
|
with tab_conf:
|
|||
|
|
if not conflicts and not unmapped:
|
|||
|
|
st.markdown(theme.pill("The two documents agree everywhere.",
|
|||
|
|
"check", "ok"), unsafe_allow_html=True)
|
|||
|
|
|
|||
|
|
if unmapped:
|
|||
|
|
st.error(f"**{len(unmapped)} source line(s) had no row in the "
|
|||
|
|
f"template.** These are NOT in the consultant's file:")
|
|||
|
|
st.markdown("".join(
|
|||
|
|
theme.card(f.label.strip(),
|
|||
|
|
f"{f.detail} (source: {f.source})",
|
|||
|
|
meta="no row in template", severity="decision")
|
|||
|
|
for f in unmapped), unsafe_allow_html=True)
|
|||
|
|
|
|||
|
|
settled = [f for f in conflicts if f.severity == "resolved"]
|
|||
|
|
open_f = [f for f in conflicts if f.severity != "resolved"]
|
|||
|
|
if settled:
|
|||
|
|
st.markdown(theme.pill(
|
|||
|
|
f"<b>{len(settled)} decision(s) applied</b> and written into the "
|
|||
|
|
f"workbook.", "check", "ok"), unsafe_allow_html=True)
|
|||
|
|
if open_f:
|
|||
|
|
st.markdown(theme.pill(
|
|||
|
|
"<b>Nothing was overwritten.</b> Where the two documents differ, "
|
|||
|
|
"your own value was left alone. Open anything you want changed, "
|
|||
|
|
"then regenerate.",
|
|||
|
|
"shield", "ok"), unsafe_allow_html=True)
|
|||
|
|
st.markdown(_legend(), unsafe_allow_html=True)
|
|||
|
|
|
|||
|
|
# Real Streamlit widgets, so the choice actually reaches Python. Keys
|
|||
|
|
# are derived from the cell, so a choice survives a rerun and can be
|
|||
|
|
# read back to build the override map.
|
|||
|
|
# one collapsed row per dispute: 16 open boxes is a wall of scrolling,
|
|||
|
|
# and the point is to scan for the one that needs a human
|
|||
|
|
for f in open_f:
|
|||
|
|
key = f"choice::{f.sheet}::{f.row}::{f.field}"
|
|||
|
|
chosen = st.session_state.get(key, "Keep yours (as it is)")
|
|||
|
|
staged = st.session_state.get(f"{key}::custom", "").strip()
|
|||
|
|
flag = "✎ " if (staged or chosen.startswith("Use our")) else ""
|
|||
|
|
with st.expander(
|
|||
|
|
f"{flag}{f.label.strip()[:70]} · {f.sheet} r{f.row} · "
|
|||
|
|
f"{f.field or 'value'}",
|
|||
|
|
expanded=bool(flag)):
|
|||
|
|
st.caption(_strip_md(f.detail))
|
|||
|
|
c_a, c_b = st.columns([1, 1])
|
|||
|
|
with c_a:
|
|||
|
|
st.selectbox(
|
|||
|
|
"which value stands",
|
|||
|
|
options=["Keep yours (as it is)",
|
|||
|
|
f"Use our PDF's value ({_fmt(f.source_value)})"],
|
|||
|
|
key=key, label_visibility="collapsed")
|
|||
|
|
with c_b:
|
|||
|
|
st.text_input(
|
|||
|
|
"or type a different value",
|
|||
|
|
key=f"{key}::custom", label_visibility="collapsed",
|
|||
|
|
placeholder="or type a value")
|
|||
|
|
|
|||
|
|
st.markdown("---")
|
|||
|
|
c1, c2 = st.columns([3, 1])
|
|||
|
|
with c1:
|
|||
|
|
n_custom = len([
|
|||
|
|
1 for f in open_f
|
|||
|
|
if st.session_state.get(f"choice::{f.sheet}::{f.row}::{f.field}::custom", "").strip()
|
|||
|
|
])
|
|||
|
|
n_take = len([
|
|||
|
|
1 for f in open_f
|
|||
|
|
if st.session_state.get(f"choice::{f.sheet}::{f.row}::{f.field}",
|
|||
|
|
"Keep yours (as it is)").startswith("Use our")
|
|||
|
|
])
|
|||
|
|
if n_custom or n_take:
|
|||
|
|
st.markdown(theme.pill(
|
|||
|
|
f"<b>{n_take + n_custom} change(s) staged.</b> Press "
|
|||
|
|
f"regenerate to write them into the workbook.",
|
|||
|
|
"refresh", "warn"), unsafe_allow_html=True)
|
|||
|
|
else:
|
|||
|
|
st.caption("Your PAYMENT column, formulas and formatting are "
|
|||
|
|
"never touched.")
|
|||
|
|
with c2:
|
|||
|
|
if st.button("Apply & regenerate", type="primary",
|
|||
|
|
use_container_width=True):
|
|||
|
|
st.session_state["dirty"] = True
|
|||
|
|
st.rerun()
|
|||
|
|
|
|||
|
|
with tab_tot:
|
|||
|
|
st.caption("The cells your consultant's summary sheet reads were blank in "
|
|||
|
|
"the template. Each now carries its section's total.")
|
|||
|
|
# Streamlit's dataframe has no text-wrapping option (no wrap_text in
|
|||
|
|
# column_config), and this column is prose, so it gets cards like the
|
|||
|
|
# findings above rather than a table that truncates.
|
|||
|
|
st.markdown("".join(
|
|||
|
|
theme.card(f.label.strip(), f.detail, meta="written to",
|
|||
|
|
severity="note")
|
|||
|
|
for f in totals), unsafe_allow_html=True)
|
|||
|
|
|
|||
|
|
vals = summary_values(tree)
|
|||
|
|
st.markdown(
|
|||
|
|
f'<div class="boq-section-sub">{theme.icon("layers", 15)}'
|
|||
|
|
f'<span>Totals carried across, in order</span></div>',
|
|||
|
|
unsafe_allow_html=True)
|
|||
|
|
st.dataframe([{"#": i + 1, "Schedule / Section": l, "Amount": money(v)}
|
|||
|
|
for i, (l, v) in enumerate(vals)],
|
|||
|
|
use_container_width=True, hide_index=True)
|
|||
|
|
if vals:
|
|||
|
|
st.markdown(theme.pill(
|
|||
|
|
f"Sum of all sections: <b>{money(sum(v for _, v in vals))}</b>",
|
|||
|
|
"calculator", "info"), unsafe_allow_html=True)
|
|||
|
|
|
|||
|
|
with tab_src:
|
|||
|
|
if not src_rep.errors:
|
|||
|
|
st.markdown(theme.pill("The source BOQ adds up on its own.",
|
|||
|
|
"check", "ok"), unsafe_allow_html=True)
|
|||
|
|
for i in src_rep.errors:
|
|||
|
|
st.error(f"**{i.where}**\n\n{i.detail}")
|
|||
|
|
if notes:
|
|||
|
|
with st.expander(f"Notes ({len(notes)})"):
|
|||
|
|
for f in notes:
|
|||
|
|
st.write(f"· `{f.sheet}` r{f.row} — {f.label}: {f.detail}")
|
|||
|
|
if result.layout_notes:
|
|||
|
|
with st.expander("Layout detection notes"):
|
|||
|
|
for n in result.layout_notes:
|
|||
|
|
st.write(f"· {n}")
|