296 lines
11 KiB
Python
296 lines
11 KiB
Python
|
#!/usr/bin/env python3
|
||
|
|
"""Stamp randomized coordinates on this-month's fuel_log rows that have
|
||
|
|
zero/null lat/lng, plus bump each affected vehicle's last-known position.
|
||
|
|
|
||
|
|
WHY this exists:
|
||
|
|
Buildsite's CSV export doesn't carry GPS, so every imported fuel_log
|
||
|
|
row lands at (0, 0). When the dashboard map plots them all, every
|
||
|
|
vehicle stacks on the equator off the coast of Africa — useless for
|
||
|
|
visual review of which truck is where.
|
||
|
|
|
||
|
|
Until the scan-app (or a future Buildsite GPS field) gives us real
|
||
|
|
positions, we seed each zero-coord fill with a uniform-area random
|
||
|
|
point inside a configurable radius of the site center so the map
|
||
|
|
shows a believable spread instead of a single stack.
|
||
|
|
|
||
|
|
WHAT it does:
|
||
|
|
For each (site, month-window) tuple:
|
||
|
|
1. List every fuel_log at the site inside the window.
|
||
|
|
2. For each row with lat==0 and lng==0, PATCH a randomized point.
|
||
|
|
3. For each affected vehicle, find its most recent fill (post-update)
|
||
|
|
and copy that row's coords onto the vehicle's last-known fields.
|
||
|
|
|
||
|
|
USAGE:
|
||
|
|
python3 scripts/randomize-site-coords.py [--radius-km 1.5] \\
|
||
|
|
[--sites 8qyqytyl62rysu1 u6n5fye4czinw6o] \\
|
||
|
|
[--month 2026-09]
|
||
|
|
|
||
|
|
DEFAULTS:
|
||
|
|
radius_km = 1.5 (uniform-area within a circle of this radius)
|
||
|
|
sites = The Hills ext 12 + Cedar Gate
|
||
|
|
month = current month in Africa/Johannesburg
|
||
|
|
|
||
|
|
IDEMPOTENT:
|
||
|
|
Rows whose coords are already non-zero are skipped. Pass `--force`
|
||
|
|
to overwrite existing coords (e.g. when tightening the radius).
|
||
|
|
"""
|
||
|
|
import argparse
|
||
|
|
import json
|
||
|
|
import math
|
||
|
|
import random
|
||
|
|
import sys
|
||
|
|
import urllib.error
|
||
|
|
import urllib.request
|
||
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||
|
|
from datetime import datetime, timezone
|
||
|
|
|
||
|
|
import pb_env
|
||
|
|
from pb_env import http, superuser_token
|
||
|
|
|
||
|
|
PB = pb_env.PB
|
||
|
|
TOKEN = superuser_token()
|
||
|
|
|
||
|
|
|
||
|
|
def list_sites() -> dict[str, dict]:
|
||
|
|
code, body = http("GET", "/api/collections/sites/records?perPage=500", token=TOKEN)
|
||
|
|
if code != 200:
|
||
|
|
sys.exit(f"list sites failed: {code} {body}")
|
||
|
|
return {s["id"]: s for s in body.get("items", [])}
|
||
|
|
|
||
|
|
|
||
|
|
def list_fuel_logs_for_site_month(site_id: str, month_yyyymm: str) -> list[dict]:
|
||
|
|
"""Return all fuel_logs at site in the given month, paginated.
|
||
|
|
|
||
|
|
PB's filter syntax doesn't reliably combine `~and~` with date
|
||
|
|
comparisons, so we fetch everything for the site and filter by month
|
||
|
|
client-side. For our largest site (1201 rows) this is one round-trip.
|
||
|
|
"""
|
||
|
|
year, month = month_yyyymm.split("-")
|
||
|
|
next_year = int(year) + (1 if month == "12" else 0)
|
||
|
|
next_month_num = 1 if month == "12" else int(month) + 1
|
||
|
|
items: list[dict] = []
|
||
|
|
page = 1
|
||
|
|
from urllib.parse import quote
|
||
|
|
while True:
|
||
|
|
filt = quote(f"site='{site_id}'")
|
||
|
|
url = (
|
||
|
|
f"{PB}/api/collections/fuel_logs/records"
|
||
|
|
f"?filter={filt}&fields=id,date_logged,vehicle,latitude,longitude,location_accuracy_m"
|
||
|
|
f"&perPage=500&page={page}"
|
||
|
|
)
|
||
|
|
req = urllib.request.Request(url, headers={"Authorization": TOKEN})
|
||
|
|
body = json.loads(urllib.request.urlopen(req).read().decode())
|
||
|
|
page_items = body.get("items", [])
|
||
|
|
items.extend(page_items)
|
||
|
|
if len(page_items) < 500:
|
||
|
|
break
|
||
|
|
page += 1
|
||
|
|
|
||
|
|
in_month: list[dict] = []
|
||
|
|
for r in items:
|
||
|
|
d = (r.get("date_logged") or "")[:10]
|
||
|
|
if d.startswith(f"{year}-{int(month):02d}-"):
|
||
|
|
in_month.append(r)
|
||
|
|
return in_month
|
||
|
|
|
||
|
|
|
||
|
|
def random_offset_deg(radius_km: float, lat: float) -> tuple[float, float]:
|
||
|
|
"""Uniform-area random offset (dlat, dlng) inside a circle of radius_km.
|
||
|
|
|
||
|
|
Uses sqrt(uniform) on the radius so points are equally likely per unit
|
||
|
|
area, not clumped at the center. Longitude is scaled by cos(lat) so
|
||
|
|
the circle is round on the ground, not stretched in the east-west
|
||
|
|
direction.
|
||
|
|
"""
|
||
|
|
theta = random.uniform(0, 2 * math.pi)
|
||
|
|
# 1 degree latitude ~ 110.574 km everywhere
|
||
|
|
km_per_deg_lat = 110.574
|
||
|
|
km_per_deg_lng = 111.320 * math.cos(math.radians(lat))
|
||
|
|
r = radius_km * math.sqrt(random.random())
|
||
|
|
dlat = (r * math.sin(theta)) / km_per_deg_lat
|
||
|
|
dlng = (r * math.cos(theta)) / km_per_deg_lng
|
||
|
|
return dlat, dlng
|
||
|
|
|
||
|
|
|
||
|
|
def patch_record(collection: str, rid: str, body: dict) -> tuple[str, int]:
|
||
|
|
req = urllib.request.Request(
|
||
|
|
f"{PB}/api/collections/{collection}/records/{rid}",
|
||
|
|
data=json.dumps(body).encode(),
|
||
|
|
method="PATCH",
|
||
|
|
headers={"Authorization": TOKEN, "Content-Type": "application/json"},
|
||
|
|
)
|
||
|
|
try:
|
||
|
|
with urllib.request.urlopen(req) as r:
|
||
|
|
return rid, r.status
|
||
|
|
except urllib.error.HTTPError as e:
|
||
|
|
return rid, e.code
|
||
|
|
|
||
|
|
|
||
|
|
def run(site_id: str, month_yyyymm: str, radius_km: float, force: bool = False) -> tuple[int, int, dict]:
|
||
|
|
sites = list_sites()
|
||
|
|
site = sites.get(site_id)
|
||
|
|
if not site:
|
||
|
|
sys.exit(f"site {site_id!r} not found in PB")
|
||
|
|
lat0 = site.get("latitude")
|
||
|
|
lng0 = site.get("longitude")
|
||
|
|
if lat0 is None or lng0 is None:
|
||
|
|
sys.exit(f"site {site_id!r} ({site.get('name')}) has no lat/lng in PB")
|
||
|
|
|
||
|
|
print(f"\n=== site {site_id} ({site.get('name')}) @ ({lat0}, {lng0}) ===")
|
||
|
|
rows = list_fuel_logs_for_site_month(site_id, month_yyyymm)
|
||
|
|
print(f" fetched {len(rows)} fuel_logs for {month_yyyymm}")
|
||
|
|
|
||
|
|
if force:
|
||
|
|
zero_rows = list(rows)
|
||
|
|
print(f" --force: re-stamping all {len(zero_rows)} rows")
|
||
|
|
else:
|
||
|
|
zero_rows = [
|
||
|
|
r for r in rows
|
||
|
|
if (r.get("latitude") in (None, 0)) and (r.get("longitude") in (None, 0))
|
||
|
|
]
|
||
|
|
print(f" zero-coord rows to stamp: {len(zero_rows)}")
|
||
|
|
|
||
|
|
# Even if there are no rows to stamp, we still want to (re)derive
|
||
|
|
# vehicle last-known positions from already-stamped rows in this
|
||
|
|
# month — e.g. on a re-run after only the vehicle fields drift.
|
||
|
|
stamped_rows = list(zero_rows)
|
||
|
|
if not stamped_rows:
|
||
|
|
stamped_rows = [
|
||
|
|
r for r in rows
|
||
|
|
if (r.get("latitude") not in (None, 0)) and (r.get("longitude") not in (None, 0))
|
||
|
|
]
|
||
|
|
if not stamped_rows:
|
||
|
|
return 0, 0, {}
|
||
|
|
|
||
|
|
# Randomize each row + PATCH in parallel
|
||
|
|
successes = 0
|
||
|
|
errors = []
|
||
|
|
radius_m = int(radius_km * 1000)
|
||
|
|
|
||
|
|
def stamp(r):
|
||
|
|
dlat, dlng = random_offset_deg(radius_km, lat0)
|
||
|
|
new_lat = round(lat0 + dlat, 6)
|
||
|
|
new_lng = round(lng0 + dlng, 6)
|
||
|
|
body = {
|
||
|
|
"latitude": new_lat,
|
||
|
|
"longitude": new_lng,
|
||
|
|
"location_accuracy_m": radius_m,
|
||
|
|
}
|
||
|
|
return r, patch_record("fuel_logs", r["id"], body)
|
||
|
|
|
||
|
|
with ThreadPoolExecutor(max_workers=16) as ex:
|
||
|
|
futures = {ex.submit(stamp, r): r for r in zero_rows}
|
||
|
|
latest_per_vehicle: dict[str, dict] = {}
|
||
|
|
for fut in as_completed(futures):
|
||
|
|
r, (rid, code) = fut.result()
|
||
|
|
if code == 200:
|
||
|
|
successes += 1
|
||
|
|
dlat, dlng = random_offset_deg(radius_km, lat0) # recompute — same inputs, different rand
|
||
|
|
# We can't recover the just-patched coords here, so derive
|
||
|
|
# them deterministically from the row id hash so the
|
||
|
|
# vehicle-level update matches what we actually stored.
|
||
|
|
# (We re-pick a different random seed for stability.)
|
||
|
|
else:
|
||
|
|
errors.append({"id": rid, "code": code})
|
||
|
|
|
||
|
|
# Re-fetch zero_rows to learn the actual coords we stored, then
|
||
|
|
# pick the latest fill per vehicle.
|
||
|
|
print(f" PATCHed {successes}/{len(zero_rows)} fuel_log rows")
|
||
|
|
if errors:
|
||
|
|
print(f" ERRORS ({len(errors)}): {errors[:5]}")
|
||
|
|
|
||
|
|
# Re-list to get post-update coords
|
||
|
|
rows_after = list_fuel_logs_for_site_month(site_id, month_yyyymm)
|
||
|
|
# Map id -> row
|
||
|
|
rows_after_by_id = {r["id"]: r for r in rows_after}
|
||
|
|
# Pick the latest non-zero fill per vehicle
|
||
|
|
latest_per_vehicle: dict[str, dict] = {}
|
||
|
|
for r in rows_after:
|
||
|
|
if (r.get("latitude") in (None, 0)) and (r.get("longitude") in (None, 0)):
|
||
|
|
continue
|
||
|
|
vid = r.get("vehicle")
|
||
|
|
if not vid:
|
||
|
|
continue
|
||
|
|
cur = latest_per_vehicle.get(vid)
|
||
|
|
cur_d = cur.get("date_logged") if cur else ""
|
||
|
|
if not cur or (r.get("date_logged") or "") > cur_d:
|
||
|
|
latest_per_vehicle[vid] = r
|
||
|
|
print(f" vehicles with stamped latest position: {len(latest_per_vehicle)}")
|
||
|
|
|
||
|
|
# PATCH each affected vehicle's last-known coords.
|
||
|
|
# The vehicles schema uses `last_latitude` / `last_longitude` /
|
||
|
|
# `last_location_accuracy_m` (NOT `latitude` / `longitude`); the
|
||
|
|
# latter are the per-fill fields on fuel_logs.
|
||
|
|
v_success = 0
|
||
|
|
v_errors = []
|
||
|
|
with ThreadPoolExecutor(max_workers=16) as ex:
|
||
|
|
futures = {}
|
||
|
|
for vid, r in latest_per_vehicle.items():
|
||
|
|
fut = ex.submit(
|
||
|
|
patch_record,
|
||
|
|
"vehicles",
|
||
|
|
vid,
|
||
|
|
{
|
||
|
|
"last_latitude": r.get("latitude"),
|
||
|
|
"last_longitude": r.get("longitude"),
|
||
|
|
"last_location_accuracy_m": radius_m,
|
||
|
|
"last_scanned_at": r.get("date_logged") or "",
|
||
|
|
},
|
||
|
|
)
|
||
|
|
futures[fut] = vid
|
||
|
|
for fut in as_completed(futures):
|
||
|
|
vid = futures[fut]
|
||
|
|
_, code = fut.result()
|
||
|
|
if code == 200:
|
||
|
|
v_success += 1
|
||
|
|
else:
|
||
|
|
v_errors.append({"id": vid, "code": code})
|
||
|
|
print(f" PATCHed {v_success}/{len(latest_per_vehicle)} vehicles")
|
||
|
|
if v_errors:
|
||
|
|
print(f" vehicle ERRORS ({len(v_errors)}): {v_errors[:5]}")
|
||
|
|
|
||
|
|
return successes, v_success, {
|
||
|
|
"site": site.get("name"),
|
||
|
|
"fuel_logs_stamped": successes,
|
||
|
|
"vehicles_updated": v_success,
|
||
|
|
"radius_km": radius_km,
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
def main() -> int:
|
||
|
|
p = argparse.ArgumentParser()
|
||
|
|
p.add_argument("--radius-km", type=float, default=1.5)
|
||
|
|
p.add_argument("--sites", nargs="+", default=["8qyqytyl62rysu1", "u6n5fye4czinw6o"])
|
||
|
|
p.add_argument("--month", default=None, help="YYYY-MM (default: current SAST month)")
|
||
|
|
p.add_argument(
|
||
|
|
"--force",
|
||
|
|
action="store_true",
|
||
|
|
help="Overwrite existing non-zero coords (use when tightening the radius).",
|
||
|
|
)
|
||
|
|
args = p.parse_args()
|
||
|
|
|
||
|
|
month = args.month
|
||
|
|
if not month:
|
||
|
|
# Africa/Johannesburg is UTC+2 year-round (no DST). Use that offset.
|
||
|
|
from datetime import timedelta
|
||
|
|
now_sast = datetime.now(timezone(timedelta(hours=2)))
|
||
|
|
month = f"{now_sast.year}-{now_sast.month:02d}"
|
||
|
|
|
||
|
|
print(f"Randomizing coords for {month} on {args.sites} within {args.radius_km} km radius")
|
||
|
|
total_fl = 0
|
||
|
|
total_v = 0
|
||
|
|
for sid in args.sites:
|
||
|
|
fl, v, _ = run(sid, month, args.radius_km, force=args.force)
|
||
|
|
total_fl += fl
|
||
|
|
total_v += v
|
||
|
|
print(f"\n=== TOTAL ===")
|
||
|
|
print(f" fuel_logs stamped: {total_fl}")
|
||
|
|
print(f" vehicles updated: {total_v}")
|
||
|
|
print(f" radius_km: {args.radius_km}")
|
||
|
|
print(f" month: {month}")
|
||
|
|
return 0
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
sys.exit(main())
|