Mega-Tracker/scripts/parse_pdf.py

#!/usr/bin/env python3
"""Parse the Mega Infrastructure Candy/Primavera barchart text dump into a WBS tree."""
import re, json, datetime
from collections import OrderedDict

SRC = "/tmp/mega.txt"

LINE_RE = re.compile(
    r"^(?P<indent>\s*)"
    r"(?P<code>[A-Z]\d{3})\s*-\s*"
    r"(?P<name>.*?)\s{2,}"
    r"(?P<dur>\d+)\s+"
    r"(?P<start>\d{2}\s+\w{3}\s+\d{2})\s+"
    r"(?P<finish>\d{2}\s+\w{3}\s+\d{2})"
)

MON = {"Oct":10,"Nov":11,"Dec":12,"Jan":1,"Feb":2,"Mar":3,"Apr":4,"May":5,"Jun":6,"Jul":7,"Aug":8,"Sep":9}

def parse_date(s, century):
    d, m, y = s.split()
    return datetime.date(2000 + int(y), MON[m], int(d))

acts = []
with open(SRC) as f:
    for raw in f:
        raw = raw.rstrip("\n")
        if not raw.strip():
            continue
        m = LINE_RE.match(raw)
        if not m:
            continue
        g = m.groupdict()
        # Candy prints dates as "06 Oct 26" -> 2026, "17 Mar 27" -> 2027
        start = parse_date(g["start"], None)
        finish = parse_date(g["finish"], None)
        acts.append({
            "code": g["code"],
            "name": re.sub(r"\s+", " ", g["name"]).strip(),
            "indent": len(g["indent"]),
            "duration": int(g["dur"]),
            "start": start.isoformat(),
            "finish": finish.isoformat(),
        })

# Build tree from leading-space indentation
roots, stack = [], []
for a in acts:
    while stack and stack[-1]["indent"] >= a["indent"]:
        stack.pop()
    a["parent"] = stack[-1]["code"] if stack else None
    a["children"] = []
    if stack:
        stack[-1]["children"].append(a["code"])
    else:
        roots.append(a["code"])
    stack.append(a)

by_code = {a["code"]: a for a in acts}

def span(a):
    return (datetime.date.fromisoformat(a["start"]), datetime.date.fromisoformat(a["finish"]))

print(f"TOTAL ACTIVITIES PARSED: {len(acts)}")
print(f"ROOT NODES: {roots}\n")

def wd(a, b):
    """working days inclusive"""
    d, n = a, 0
    while d <= b:
        if d.weekday() < 5:
            n += 1
        d += datetime.timedelta(days=1)
    return n

for r in roots:
    a = by_code[r]
    s, f = span(a)
    print("=" * 78)
    print(f"{a['code']} - {a['name']}   [{a['duration']}wd]  {a['start']} -> {a['finish']}")
    print(f"   actual calendar span: {(f-s).days+1} days, {wd(s,f)} working days")
    tot = 0
    for c in a["children"]:
        ca = by_code[c]
        s2, f2 = span(ca)
        tot += ca["duration"]
        flag = ""
        if ca["duration"] != wd(s2, f2):
            flag += f"  [!DUR {ca['duration']}d vs {wd(s2,f2)}wd actual]"
        if ca["finish"] > a["finish"] or ca["start"] < a["start"]:
            flag += "  [!OUTSIDE PARENT WINDOW]"
        print(f"   +- {ca['code']} - {ca['name']}  [{ca['duration']}wd] {ca['start']} -> {ca['finish']}{flag}")
        for g in ca["children"]:
            ga = by_code[g]
            s3, f3 = span(ga)
            gflag = ""
            if ga["duration"] != wd(s3, f3):
                gflag += f"  [!DUR {ga['duration']}d vs {wd(s3,f3)}wd]"
            print(f"      |  {ga['code']} - {ga['name']}  [{ga['duration']}wd] {ga['start']} -> {ga['finish']}{gflag}")
    print(f"   >> direct children duration sum: {tot} vs parent {a['duration']} (delta {tot - a['duration']:+d})")

# ---- global anomalies ----
print("\n" + "=" * 78)
print("ANOMALIES / DATA QUALITY FINDINGS")
print("=" * 78)

# 1. code order != tree order
order = [a["code"] for a in acts]
outoforder = [(a["code"], a["parent"]) for a in acts
              if a["parent"] and order.index(a["parent"]) > order.index(a["code"])]
print(f"\n1. Codes that sort AFTER their parent's children (code != hierarchy):")
for c, p in outoforder:
    print(f"   {c} (parent {p}) appears before sibling {c} numerically")

# 2. duration vs calendar mismatch
print(f"\n2. Duration field != actual working days between start/finish:")
n = 0
for a in acts:
    s, f = span(a)
    if a["duration"] != wd(s, f):
        n += 1
        print(f"   {a['code']} - {a['name']}: says {a['duration']}wd, span is {wd(s,f)}wd")
print(f"   total: {n}/{len(acts)}")

# 3. unconfirmed "?" text
print(f"\n3. Unresolved placeholders in spec text:")
for a in acts:
    if "?" in a["name"]:
        print(f"   {a['code']} - {a['name']}")

# 4. parents with no children / children with no dates
print(f"\n4. Leaf activities (bottom of WBS): {sum(1 for a in acts if not a['children'])}")
print(f"   Parents: {sum(1 for a in acts if a['children'])}")

# 5. typos
TYPOS = {"Intergration":"Integration","Opperator":"Operator","Cources":"Courses",
         "Expences":"Expenses","retetion":"retention","license":"licence",
         "expences":"expenses","VaT":"VAT"}
print(f"\n5. Likely typos in spec wording:")
for a in acts:
    for k, v in TYPOS.items():
        if k in a["name"]:
            print(f"   {a['code']}: {a['name']}  ->  {a['name'].replace(k, v)}")

# 6. identical windows (possible duplicates / parallel pairs)
print(f"\n6. Activities sharing an identical start/finish window:")
seen = {}
for a in acts:
    seen.setdefault((a["start"], a["finish"]), []).append(a["code"])
for k, v in seen.items():
    if len(v) > 1:
        print(f"   {k[0]} -> {k[1]}: {', '.join(v)}")

# 7. programme bounds
allstarts = [span(a)[0] for a in acts]
allfin = [span(a)[1] for a in acts]
print(f"\n7. Programme envelope: {min(allstarts)} -> {max(allfin)}")
print(f"   Today: {datetime.date(2026,9,30)}  ->  everything is FUTURE, nothing overdue at kickoff")

json.dump(acts, open("/tmp/mega_acts.json", "w"), indent=1)
print(f"\nWrote {len(acts)} activities -> /tmp/mega_acts.json")