235 lines
12 KiB
Python
235 lines
12 KiB
Python
|
"""Add stormwater and sewer structures, pipes and levels to a Cedar Gate site model.
|
||
|
|
|
||
|
|
Reads the long-section tables of C3277-921-001 (stormwater, with the sewer sheets bound in) from a
|
||
|
|
LibreDWG DXF, and writes a copy of the base model folder with services.json extended. Plan routes and
|
||
|
|
water are kept as they were. Requires ezdxf. Coordinates: Lo Y/X in the tables are positive; local
|
||
|
|
east/north = (-Y, -X) minus the model origin, the same as extract_cedar_gate.py.
|
||
|
|
|
||
|
|
python3 extract_cedar_gate_services.py LONGSECTION.dxf BASE_MODEL_DIR OUT_DIR [ISSUED.pdf ...]
|
||
|
|
|
||
|
|
The bound DWG also holds superseded copies of some sewer long sections. When the issued PDFs are
|
||
|
|
given (needs poppler's pdftotext), each profile is scored by how many of its levels appear in them,
|
||
|
|
and for profiles drawn twice only the best match is kept.
|
||
|
|
"""
|
||
|
|
import hashlib
|
||
|
|
import io
|
||
|
|
import json
|
||
|
|
import math
|
||
|
|
import re
|
||
|
|
import shutil
|
||
|
|
import subprocess
|
||
|
|
import sys
|
||
|
|
from pathlib import Path
|
||
|
|
|
||
|
|
LAYER = "LS&DET$0$CC_LS_TEXT"
|
||
|
|
ROWS = ("GROUND LEVEL", "INVERT LEVEL", "DEPTH (m)", "GRADE (1:_)", "PEG DISTANCE (m)", "PIPE DESCRIPTION")
|
||
|
|
NON_STRUCTURE = re.compile(r"^(LH|Y-Junction)$", re.I)
|
||
|
|
COORD = re.compile(r"([A-Za-z][\w.\-()]*)\s+(\d{5}\.\d+)\s+(2\d{6}\.\d+)")
|
||
|
|
|
||
|
|
|
||
|
|
def read_texts(path):
|
||
|
|
"""TEXT/MTEXT on the long-section layer. LibreDWG wraps multiline text and writes payloads ezdxf rejects."""
|
||
|
|
import ezdxf
|
||
|
|
lines = Path(path).read_text(errors="replace").splitlines()
|
||
|
|
pairs, i = [], 0
|
||
|
|
while i < len(lines):
|
||
|
|
try:
|
||
|
|
code = int(lines[i])
|
||
|
|
except ValueError:
|
||
|
|
pairs[-1] = (pairs[-1][0], pairs[-1][1] + r"\P" + lines[i]); i += 1; continue
|
||
|
|
pairs.append((code, lines[i + 1])); i += 2
|
||
|
|
clean, skip = [], False
|
||
|
|
for code, value in pairs:
|
||
|
|
if code == 0:
|
||
|
|
skip = False
|
||
|
|
if code == 101 and value == "Embedded Object":
|
||
|
|
skip = True
|
||
|
|
if not skip:
|
||
|
|
clean.extend((str(code), value))
|
||
|
|
doc = ezdxf.read(io.StringIO("\n".join(clean) + "\n"))
|
||
|
|
out = []
|
||
|
|
def flatten(entities):
|
||
|
|
for e in entities:
|
||
|
|
if e.dxftype() == "INSERT":
|
||
|
|
if e.dxf.name in doc.blocks:
|
||
|
|
try:
|
||
|
|
yield from flatten(e.virtual_entities())
|
||
|
|
except Exception:
|
||
|
|
pass
|
||
|
|
else:
|
||
|
|
yield e
|
||
|
|
for e in flatten(doc.modelspace()):
|
||
|
|
if e.dxf.layer != LAYER or e.dxftype() not in ("TEXT", "MTEXT"):
|
||
|
|
continue
|
||
|
|
text = e.plain_text() if e.dxftype() == "MTEXT" else e.dxf.text
|
||
|
|
out.append((float(e.dxf.insert[0]), float(e.dxf.insert[1]), text))
|
||
|
|
return out
|
||
|
|
|
||
|
|
|
||
|
|
norm = lambda s: re.sub(r"\s+", "", s)
|
||
|
|
|
||
|
|
|
||
|
|
def num(s):
|
||
|
|
try:
|
||
|
|
return float(s)
|
||
|
|
except ValueError:
|
||
|
|
return None
|
||
|
|
|
||
|
|
|
||
|
|
def blocks(texts):
|
||
|
|
"""One block per 'MANHOLE' anchor: the row y positions and the x range of its columns."""
|
||
|
|
anchors = sorted((t for t in texts if t[2].strip() in ("MANHOLE", "MANHOLE / JUNCTION BOX")), key=lambda t: (-t[1], t[0]))
|
||
|
|
out = []
|
||
|
|
for ax, ay, label in anchors:
|
||
|
|
rows = {}
|
||
|
|
for name in ROWS:
|
||
|
|
near = [t for t in texts if t[2].strip() == name and abs(t[0] - ax) < 4 and 0 < ay - t[1] < 600]
|
||
|
|
if near:
|
||
|
|
rows[name] = max(near, key=lambda t: t[1])[1]
|
||
|
|
right = [a[0] for a in anchors if a[0] > ax + 20 and abs(a[1] - ay) < 60]
|
||
|
|
out.append({"ax": ax, "ay": ay, "kind": "stormwater" if "JUNCTION" in label else "sewer", "rows": rows,
|
||
|
|
"x0": ax + 60, "x1": min(right) - 5 if right else ax + 800})
|
||
|
|
return out
|
||
|
|
|
||
|
|
|
||
|
|
def read_block(texts, b):
|
||
|
|
band = lambda y, tol=5.5: [t for t in texts if b["x0"] <= t[0] < b["x1"] and abs(t[1] - y) <= tol]
|
||
|
|
names = [t for t in band(b["ay"], 8) if t[2].strip() and t[2].strip() not in ("MANHOLE", "MANHOLE / JUNCTION BOX") and not num(t[2])]
|
||
|
|
values = {r: band(b["rows"][r]) for r in ROWS if r in b["rows"]}
|
||
|
|
# every number belongs to the structure whose name is nearest across the table
|
||
|
|
xs = [t[0] for t in sorted(names)]
|
||
|
|
owned = {r: {i: [] for i in range(len(xs))} for r in values}
|
||
|
|
for r, vals in values.items():
|
||
|
|
for vx, _, text in vals:
|
||
|
|
v = num(text)
|
||
|
|
if v is None or not xs:
|
||
|
|
continue
|
||
|
|
k = min(range(len(xs)), key=lambda i: abs(xs[i] - vx))
|
||
|
|
if abs(xs[k] - vx) <= 12:
|
||
|
|
owned[r][k].append((vx, v))
|
||
|
|
items = []
|
||
|
|
for k, (x, y, text) in enumerate(sorted(names)):
|
||
|
|
gl, peg, depth = (sorted(owned.get(r, {}).get(k, [])) for r in ("GROUND LEVEL", "PEG DISTANCE (m)", "DEPTH (m)"))
|
||
|
|
inv = sorted(owned.get("INVERT LEVEL", {}).get(k, []))
|
||
|
|
if not gl or not inv or not peg:
|
||
|
|
continue
|
||
|
|
items.append({"name": norm(text.split("\n")[0]), "x": x, "gl": gl[0][1], "peg": peg[0][1],
|
||
|
|
"invL": inv[0][1], "invR": inv[-1][1], "depth": depth[0][1] if depth else None})
|
||
|
|
desc = []
|
||
|
|
for x, y, text in sorted(values.get("PIPE DESCRIPTION", [])):
|
||
|
|
m = re.search(r"(\d{2,4})\s*mm\s*(?:%%C|Ø)?\s*(.*)", text.replace("\n", " "))
|
||
|
|
if m:
|
||
|
|
desc.append({"x": x, "dia": int(m.group(1)), "text": re.sub(r"\s+", " ", text.replace("\n", " ")).strip()})
|
||
|
|
grades = sorted((t[0], num(t[2])) for t in values.get("GRADE (1:_)", []) if num(t[2]) is not None)
|
||
|
|
return items, desc, grades
|
||
|
|
|
||
|
|
|
||
|
|
def title(texts, b):
|
||
|
|
c = [t for t in texts if re.fullmatch(r"S[WE]-\S+", t[2].strip()) and abs(t[0] - b["ax"]) < 60 and 0 < t[1] - b["ay"] < 120]
|
||
|
|
return min(c, key=lambda t: t[1] - b["ay"])[2].strip() if c else None
|
||
|
|
|
||
|
|
|
||
|
|
def issued_text(pdfs):
|
||
|
|
return " ".join(subprocess.run(["pdftotext", "-raw", str(p), "-"], capture_output=True, text=True).stdout for p in pdfs)
|
||
|
|
|
||
|
|
|
||
|
|
def build(texts, origin, issued=""):
|
||
|
|
ox, oy = origin
|
||
|
|
coords = {}
|
||
|
|
for _, _, text in texts:
|
||
|
|
for m in COORD.finditer(text):
|
||
|
|
if not NON_STRUCTURE.match(m.group(1)):
|
||
|
|
coords.setdefault(norm(m.group(1)).lower(), (float(m.group(2)), float(m.group(3))))
|
||
|
|
local = lambda Y, X: (round(-Y - ox, 3), round(-X - oy, 3))
|
||
|
|
sw_struct, sw_pipes, sr_mh, sr_pipes, report = {}, [], {}, [], {"runs": [], "gradeChecks": [], "unlocated": []}
|
||
|
|
cands = []
|
||
|
|
for b in blocks(texts):
|
||
|
|
items, desc, grades = read_block(texts, b)
|
||
|
|
if len(items) < 2:
|
||
|
|
continue
|
||
|
|
levels = [f"{v:.3f}" for i in items for v in (i["invL"], i["invR"])]
|
||
|
|
score = sum(v in issued for v in levels) / len(levels) if issued else 1.0
|
||
|
|
cands.append({"b": b, "items": items, "desc": desc, "title": title(texts, b), "score": round(score, 3)})
|
||
|
|
keep = []
|
||
|
|
for c in cands: # a profile drawn twice: keep the copy that matches the issued sheets
|
||
|
|
same = [o for o in cands if o["b"]["kind"] == c["b"]["kind"] and o["title"] == c["title"] and c["title"]]
|
||
|
|
if c is max(same or [c], key=lambda o: o["score"]):
|
||
|
|
keep.append(c)
|
||
|
|
else:
|
||
|
|
report.setdefault("discarded", []).append({"kind": c["b"]["kind"], "title": c["title"], "score": c["score"], "structures": [i["name"] for i in c["items"]]})
|
||
|
|
run_no = {"stormwater": 0, "sewer": 0}
|
||
|
|
for c in keep:
|
||
|
|
b, items, desc, tname = c["b"], c["items"], c["desc"], c["title"]
|
||
|
|
run_no[b["kind"]] += 1
|
||
|
|
run = run_no[b["kind"]]
|
||
|
|
report["runs"].append({"kind": b["kind"], "run": run, "title": tname, "score": c["score"], "verifiedOnIssuedSheets": c["score"] >= 0.9, "structures": [i["name"] for i in items]})
|
||
|
|
real = [i for i in items if not NON_STRUCTURE.match(i["name"])]
|
||
|
|
for it in real:
|
||
|
|
c = coords.get(it["name"].lower())
|
||
|
|
e, n = local(*c) if c else (None, None)
|
||
|
|
if not c:
|
||
|
|
report["unlocated"].append(it["name"])
|
||
|
|
if b["kind"] == "stormwater":
|
||
|
|
s = sw_struct.setdefault(it["name"], {"name": it["name"], "e": e, "n": n, "Y": c[0] if c else None, "X": c[1] if c else None, "il": min(it["invL"], it["invR"]), "gl": it["gl"]})
|
||
|
|
s["il"] = min(s["il"], it["invL"], it["invR"])
|
||
|
|
else:
|
||
|
|
m = sr_mh.setdefault(f"S-{it['name']}", {"name": it["name"], "id": f"S-{it['name']}", "run": run, "il": min(it["invL"], it["invR"]), "located": bool(c), "gl": it["gl"]})
|
||
|
|
m["il"] = min(m["il"], it["invL"], it["invR"])
|
||
|
|
if c:
|
||
|
|
m.update({"Y": c[0], "X": c[1], "e": e, "n": n})
|
||
|
|
real.sort(key=lambda i: i["peg"])
|
||
|
|
for a, bb in zip(real, real[1:]):
|
||
|
|
length = round(bb["peg"] - a["peg"], 3)
|
||
|
|
if length <= 0:
|
||
|
|
continue
|
||
|
|
# flow runs downhill: from the higher pipe invert to the lower
|
||
|
|
a_to_b = a["invR"] > bb["invL"]
|
||
|
|
up, down = (a, bb) if a_to_b else (bb, a)
|
||
|
|
il_from = up["invR"] if a_to_b else up["invL"]
|
||
|
|
il_to = down["invL"] if a_to_b else down["invR"]
|
||
|
|
mid = (a["x"] + bb["x"]) / 2
|
||
|
|
d = min(desc, key=lambda t: abs(t["x"] - mid)) if desc else None
|
||
|
|
grade = round((il_from - il_to) / length * 100, 3)
|
||
|
|
pipe = {"from": up["name"], "to": down["name"], "run": run, "length": length, "dia": d["dia"] if d else 0,
|
||
|
|
"ilFrom": il_from, "ilTo": il_to, "grade": grade}
|
||
|
|
if b["kind"] == "stormwater":
|
||
|
|
pipe["diaSource"] = "drawing label"
|
||
|
|
sw_pipes.append(pipe)
|
||
|
|
else:
|
||
|
|
pipe.update({"from": f"S-{up['name']}", "to": f"S-{down['name']}", "hold": None})
|
||
|
|
pipe.pop("diaSource", None)
|
||
|
|
sr_pipes.append(pipe)
|
||
|
|
return sw_struct, sw_pipes, sr_mh, sr_pipes, report
|
||
|
|
|
||
|
|
|
||
|
|
def main(dxf, base_dir, out_dir, *pdfs):
|
||
|
|
base, out = Path(base_dir), Path(out_dir)
|
||
|
|
out.mkdir(parents=True, exist_ok=True)
|
||
|
|
road = json.loads((base / "bushveld.json").read_text())
|
||
|
|
origin = (road["origin"]["cadX"], road["origin"]["cadY"])
|
||
|
|
sw_struct, sw_pipes, sr_mh, sr_pipes, report = build(read_texts(dxf), origin, issued_text(pdfs) if pdfs else "")
|
||
|
|
services = json.loads((base / "services.json").read_text())
|
||
|
|
services["stormwater"] = {"structures": [s for s in sw_struct.values() if s["e"] is not None], "pipes": sw_pipes,
|
||
|
|
"notes": ["Pipes, sizes, inverts and ground levels read from the C3277-921-001 long sections; positions from their coordinate lists.",
|
||
|
|
"A pipe's size is the nearest PIPE DESCRIPTION on the long section; check against the plan labels before use.",
|
||
|
|
"Kerb inlets, grid inlets and the outlet are shown as structures with their invert levels.",
|
||
|
|
"The external SW-M1 line is on the DWG only; no issued PDF was supplied to check its levels against."]}
|
||
|
|
services["sewer"] = {"manholes": list(sr_mh.values()), "pipes": sr_pipes,
|
||
|
|
"notes": ["Manholes and pipes read from the C3277-931 sewer long sections; Y-junctions and lampholes on a run are not drawn as structures.",
|
||
|
|
"160 mm uPVC Class 400 kPa unless the long section shows otherwise."]}
|
||
|
|
for name in ("bushveld", "earthworks", "schedules", "surface"):
|
||
|
|
shutil.copyfile(base / f"{name}.json", out / f"{name}.json")
|
||
|
|
(out / "services.json").write_text(json.dumps(services, separators=(",", ":")))
|
||
|
|
sources = json.loads((base / "sources.json").read_text()) if (base / "sources.json").exists() else {}
|
||
|
|
sources.setdefault("sources", []).append({"file": Path(dxf).name, "role": "long sections (stormwater, sewer)"})
|
||
|
|
sources["servicesExtraction"] = {"stormwater": {"structures": len(services["stormwater"]["structures"]), "pipes": len(sw_pipes)},
|
||
|
|
"sewer": {"manholes": len(sr_mh), "pipes": len(sr_pipes)}, "unlocated": sorted(set(report["unlocated"]))}
|
||
|
|
(out / "sources.json").write_text(json.dumps(sources, indent=2))
|
||
|
|
(out / "services-extraction.json").write_text(json.dumps(report, indent=2))
|
||
|
|
print(json.dumps({"stormwater": sources["servicesExtraction"]["stormwater"], "sewer": sources["servicesExtraction"]["sewer"], "unlocated": sources["servicesExtraction"]["unlocated"]}))
|
||
|
|
return services
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
main(*sys.argv[1:])
|