David/MEGAINFRA-WEBAPP
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions
MEGAINFRA-WEBAPP/scripts/pb-bootstrap-schema.py
DEls cb463c0faa feat(workforce): add timesheet exports and duplicate protection 
Add per-site CSV and PDF matrix reports, editable sick-note date ranges, and weekend calendar styling. Preserve verified duplicate uploads for audit while excluding their entries from hours and costs, and prevent repeat files and overlapping daily imports.
2026-10-07 23:53:47 +02:00

589 lines
26 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/usr/bin/env python3
"""Bootstrap PocketBase schema for MEGA INFRASTRUCTURE.
PocketBase v0.39 JSON shape: type-specific options (values, maxSelect,
mimeTypes, collectionId, cascadeDelete, min, max) live at the field
top level — NOT nested under "options".
Reflects the post-ingestion-restructure schema (2026-09):
- `data_import_batches` (the shared queue pool) and the deprecated
`plant` alias for `vehicles` are gone.
- `timesheets`, `fuel_sheets` and `plant_hours_sheets` are the
uploaded-document collections; each carries its own
`processing_status` queue fields directly (no separate batch row).
- `timesheets` (old) was renamed `timesheet_entries` — the actual
per-employee-per-day time log.
- `received_items` was renamed `delivery_note_items`.
- `purchase_orders` absorbed the queue fields `data_import_batches`
used to carry, plus `file_hash` and `date_uploaded`.
- `project_code` was removed from every collection except `sites`
(the one place it is canonical); relations to `sites` carry site
attribution everywhere else.
- `qty_outstanding` was removed from `purchase_order_items` — it is
now computed at read time via src/lib/po-fulfilment.ts.
- `fuel_prices` and `income_claims` were always live collections but
were missing from this bootstrap script; both are included now.
"""
import json
import sys
import urllib.request
import urllib.error
PB = "https://pb.mega.infra.apphub.co.za"
EMAIL = "admin@mega.co.za"
PASSWORD = "12345678"
def http(method, path, body=None, token=None):
url = f"{PB}{path}"
data = json.dumps(body).encode() if body is not None else None
req = urllib.request.Request(url, data=data, method=method)
if body is not None:
req.add_header("Content-Type", "application/json")
if token:
req.add_header("Authorization", token)
try:
with urllib.request.urlopen(req) as r:
txt = r.read().decode()
return r.status, json.loads(txt) if txt else None
except urllib.error.HTTPError as e:
return e.code, json.loads(e.read().decode() or "{}")
code, auth = http("POST", "/api/collections/_superusers/auth-with-password",
{"identity": EMAIL, "password": PASSWORD})
if code != 200:
sys.exit(f"AUTH FAILED: {code} {auth}")
TOKEN = auth["token"]
print(f"auth ok, superuser id={auth['record']['id']}\n")
LOCKED = { # superuser-only access
"listRule": None, "viewRule": None,
"createRule": None, "updateRule": None, "deleteRule": None,
}
def field(name, type_, **kw):
"""Build a field. Extra kwargs (required, values, maxSelect, etc.)
go at the field top level."""
f = {"name": name, "type": type_}
f.update(kw)
return f
# Shared field set for the three "uploaded document" collections
# (timesheets, fuel_sheets, plant_hours_sheets) — each is a
# resumable/idempotent import queue item in its own right, not a row
# in a shared pool.
def sheet_fields(site_required):
return [
field("site", "relation", required=site_required,
collectionId="__SITES_ID__", cascadeDelete=False, maxSelect=1),
field("source_file", "file", required=False,
maxSelect=1, maxSize=20971520,
mimeTypes=["text/csv", "text/plain", "application/vnd.ms-excel",
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
"application/csv"]),
field("content_type", "text", required=False),
field("file_hash", "text", required=False),
field("date_uploaded", "date", required=False),
field("uploaded_by", "text", required=False),
field("processing_status", "select", required=False,
values=["pending", "processing", "processed", "failed", "dead"], maxSelect=1),
field("processed_at", "date", required=False),
field("attempts", "number", required=False),
field("last_error", "text", required=False),
field("rows_total", "number", required=False),
field("rows_created", "number", required=False),
field("rows_failed", "number", required=False),
field("row_errors", "json", required=False),
]
DEFS = {}
DEFS["sites"] = {
"name": "SITES", "type": "base",
"fields": [
field("project_code", "text", required=True, min=1, max=64),
field("name", "text", required=False),
field("duration", "text", required=False),
field("contract_value", "number", required=False),
field("work_hours", "number", required=False),
field("hours_per_week", "number", required=False),
field("wages_hours_per_day", "number", required=False),
field("latitude", "number", required=False),
field("longitude", "number", required=False),
field("job_start_date", "date", required=False),
# How the site counts its own "work months" — e.g. 21 means each
# work-month runs the 21st through the 20th of the next calendar
# month, not the 1st–30th. Unset (or 1) means ordinary calendar
# months. See src/lib/project-calendar.ts.
field("work_month_start_day", "number", required=False, min=1, max=31),
],
"indexes": [
"CREATE UNIQUE INDEX `idx_sites_project_code` ON `sites` (`project_code`)",
],
}
DEFS["employees"] = {
"name": "EMPLOYEES", "type": "base",
"fields": [
field("site", "relation", required=False,
collectionId="__SITES_ID__", cascadeDelete=False, maxSelect=1),
field("name", "text", required=False),
field("employee_no", "text", required=False),
field("position", "text", required=False),
field("rate_per_hour", "number", required=False),
field("hours_per_day_standard", "number", required=False),
field("worker_type", "select", required=False,
values=["permanent", "casual"], maxSelect=1),
field("employment_status", "select", required=False,
values=["active", "inactive"], maxSelect=1),
field("inactive_from", "date", required=False),
],
"indexes": [],
}
DEFS["employee_sick_notes"] = {
"name": "EMPLOYEE_SICK_NOTES", "type": "base",
"fields": [
field("employee", "relation", required=True, collectionId="__EMPLOYEES_ID__", cascadeDelete=False, maxSelect=1),
field("note_date", "date", required=True),
field("end_date", "date", required=False),
field("photo", "file", required=True, maxSelect=1, maxSize=10485760, mimeTypes=["image/jpeg", "image/png", "image/webp"], protected=True),
field("created", "autodate", onCreate=True, onUpdate=False),
],
"indexes": ["CREATE INDEX idx_employee_sick_notes_date ON EMPLOYEE_SICK_NOTES (employee, note_date)"],
}
DEFS["employee_rates"] = {
"name": "EMPLOYEE_RATES", "type": "base",
"fields": [
field("employee", "relation", required=True,
collectionId="__EMPLOYEES_ID__", cascadeDelete=False, maxSelect=1),
field("effective_from", "date", required=True),
field("rate_per_hour", "number", required=True, min=0),
field("notes", "text", required=False),
field("created", "autodate", onCreate=True, onUpdate=False),
field("updated", "autodate", onCreate=True, onUpdate=True),
],
"indexes": [
"CREATE UNIQUE INDEX `idx_employee_rates_employee_day` ON `EMPLOYEE_RATES` (`employee`, substr(`effective_from`, 1, 10))",
"CREATE INDEX `idx_employee_rates_employee_date` ON `EMPLOYEE_RATES` (`employee`, `effective_from`)",
],
}
DEFS["suppliers"] = {
"name": "SUPPLIERS", "type": "base",
"fields": [
field("name", "text", required=True, min=1, max=200),
field("reconciliation_rules", "json", required=False, maxSize=1048576),
],
"indexes": [],
}
DEFS["purchase_orders"] = {
"name": "PURCHASE_ORDERS", "type": "base",
"fields": [
field("purchase_order_number", "text", required=True, min=1, max=128),
field("site", "relation", required=True,
collectionId="__SITES_ID__", cascadeDelete=False, maxSelect=1),
field("supplier", "text", required=False),
field("supplier_ref", "relation", required=False,
collectionId="__SUPPLIERS_ID__", cascadeDelete=False, maxSelect=1),
field("required_by_date", "date", required=False),
field("ordered_by", "text", required=False),
field("items_count", "number", required=False),
field("total_value", "number", required=False),
field("purchase_order_file", "file", required=False,
maxSelect=1, maxSize=20971520,
mimeTypes=["application/pdf", "image/jpeg", "image/png", "image/webp"]),
field("raw_ocr_data", "json", required=False),
field("is_verified", "bool", required=False),
field("file_hash", "text", required=False),
field("processing_status", "select", required=False,
values=["pending", "processing", "processed", "failed", "dead"], maxSelect=1),
field("attempts", "number", required=False),
field("last_error", "text", required=False),
field("next_retry_at", "date", required=False),
field("last_attempt_at", "date", required=False),
field("date_uploaded", "date", required=False),
],
"indexes": [
"CREATE UNIQUE INDEX `idx_po_number` ON `purchase_orders` (`purchase_order_number`)",
"CREATE INDEX `idx_po_site_status` ON `purchase_orders` (`site`, `processing_status`)",
],
}
DEFS["purchase_order_items"] = {
"name": "PURCHASE_ORDER_ITEMS", "type": "base",
"fields": [
field("purchase_order", "relation", required=False,
collectionId="__PURCHASE_ORDERS_ID__", cascadeDelete=True, maxSelect=1),
field("material_code", "text", required=False),
field("cost_code", "text", required=False),
field("description", "text", required=False),
field("qty", "number", required=False),
field("unit", "text", required=False),
field("rate", "number", required=False),
field("disc_percent", "number", required=False),
field("vat_percent", "number", required=False),
field("amount", "number", required=False),
],
"indexes": [],
}
DEFS["fixed_costs"] = {
"name": "FIXED_COSTS", "type": "base",
"fields": [
field("site", "relation", required=False,
collectionId="__SITES_ID__", cascadeDelete=False, maxSelect=1),
field("date_logged", "date", required=False),
field("description", "text", required=False),
field("cost", "number", required=False),
field("notes", "text", required=False),
# bool, no default: PB applies the field default only when the
# value is absent. Defaulting to `true` would also be fine for
# new rows, but we leave the field "unset by default" so explicit
# writes of `false` cannot be silently overridden by a default.
# The partition helper treats undefined/null as monthly via a
# `=== false` check, and the backfill migration patches any
# pre-existing row to `true`.
field("is_recurring", "bool", required=False),
],
"indexes": [],
}
DEFS["vehicles"] = {
"name": "VEHICLES", "type": "base",
"fields": [
field("site", "relation", required=False,
collectionId="__SITES_ID__", cascadeDelete=False, maxSelect=1),
field("description", "text", required=False),
field("standard_work_hours", "text", required=False),
field("fuel_limits", "text", required=False),
field("qr_code_id", "text", required=False),
field("cost_per_hour", "number", required=False),
field("attention_dismissed_at", "date", required=False),
field("unit_type", "select", required=False,
values=["fuel", "plant"], maxSelect=1, default="fuel"),
# Latest GPS fix from a MEGA Clock Fleet scan. Denormalised off
# `fuel_logs` so the map page can render every vehicle in a
# single query. Stays null for vehicles never scanned from a
# kiosk phone (e.g. imported only from a pump sheet).
field("last_latitude", "number", required=False),
field("last_longitude", "number", required=False),
field("last_scanned_at", "date", required=False),
field("last_location_accuracy_m", "number", required=False),
],
"indexes": [
"CREATE INDEX `idx_vehicles_last_scanned` ON `vehicles` (`last_scanned_at`)",
],
}
DEFS["fuel_prices"] = {
"name": "FUEL_PRICES", "type": "base",
"fields": [
field("effective_from", "date", required=True),
field("price_per_litre", "number", required=True),
field("notes", "text", required=False),
],
"indexes": [
"CREATE UNIQUE INDEX `idx_fuel_prices_effective_from` ON `fuel_prices` (`effective_from`)",
],
}
DEFS["timesheets"] = {
"name": "TIMESHEETS", "type": "base",
"fields": sheet_fields(site_required=True) + [field("import_key", "text", required=False), field("duplicate_of", "text", required=False)],
"indexes": [
"CREATE UNIQUE INDEX `idx_timesheets_import_key` ON `timesheets` (`import_key`) WHERE `import_key` != ''",
"CREATE INDEX `idx_timesheets_hash` ON `timesheets` (`file_hash`)",
"CREATE INDEX `idx_timesheets_status` ON `timesheets` (`processing_status`, `date_uploaded`)",
],
}
DEFS["timesheet_entries"] = {
"name": "TIMESHEET_ENTRIES", "type": "base",
"fields": [
field("excluded_duplicate", "bool", required=False),
field("duplicate_of", "text", required=False),
field("import_key", "text", required=False),
field("employee", "relation", required=False,
collectionId="__EMPLOYEES_ID__", cascadeDelete=False, maxSelect=1),
field("date_logged", "date", required=False),
field("hours_worked", "number", required=False),
# Stamped from the employee's site at import time — NOT joined at
# read time. This ends historical drift when a worker moves sites.
field("site", "relation", required=True,
collectionId="__SITES_ID__", cascadeDelete=False, maxSelect=1),
# Optional forever: empty means captured directly by QR scan, not
# from an uploaded sheet.
field("timesheet", "relation", required=False,
collectionId="__TIMESHEETS_ID__", cascadeDelete=False, maxSelect=1),
],
"indexes": [
"CREATE UNIQUE INDEX `idx_timesheet_entries_import_key` ON `timesheet_entries` (`import_key`) WHERE `import_key` != ''",
"CREATE INDEX `idx_ts_entries_site_date` ON `timesheet_entries` (`site`, `date_logged`)",
"CREATE INDEX `idx_ts_entries_emp_date` ON `timesheet_entries` (`employee`, `date_logged`)",
],
}
DEFS["clock_events"] = {
"name": "CLOCK_EVENTS", "type": "base",
"fields": [
field("employee", "relation", required=True,
collectionId="__EMPLOYEES_ID__", cascadeDelete=False, maxSelect=1),
field("site", "relation", required=True,
collectionId="__SITES_ID__", cascadeDelete=False, maxSelect=1),
field("direction", "select", required=True,
values=["in", "out"], maxSelect=1),
field("scanned_at", "date", required=True),
# Optional: set only on "out" events, pointing at the
# timesheet_entries row that scan created/updated.
field("timesheet_entry", "relation", required=False,
collectionId="__TIMESHEET_ENTRIES_ID__", cascadeDelete=False, maxSelect=1),
],
"indexes": [
"CREATE INDEX `idx_clock_events_emp_scanned` ON `clock_events` (`employee`, `scanned_at`)",
],
}
DEFS["clock_missed_out_corrections"] = {
"name": "clock_missed_out_corrections", "type": "base",
"fields": [
field("clock_in_event", "relation", required=True,
collectionId="__CLOCK_EVENTS_ID__", cascadeDelete=False, maxSelect=1, minSelect=1),
field("employee", "relation", required=True,
collectionId="__EMPLOYEES_ID__", cascadeDelete=False, maxSelect=1, minSelect=1),
field("site", "relation", required=True,
collectionId="__SITES_ID__", cascadeDelete=False, maxSelect=1, minSelect=1),
field("actual_out_at", "date", required=True),
field("reason", "text", required=True, min=3, max=2000),
field("recorded_at", "date", required=True),
field("timesheet_entry", "relation", required=True,
collectionId="__TIMESHEET_ENTRIES_ID__", cascadeDelete=False, maxSelect=1, minSelect=1),
],
"indexes": [
"CREATE UNIQUE INDEX `idx_missed_out_in_event` ON `clock_missed_out_corrections` (`clock_in_event`)",
],
}
DEFS["clock_correction_locks"] = {
"name": "clock_correction_locks", "type": "base",
"fields": [
field("employee", "relation", required=True,
collectionId="__EMPLOYEES_ID__", cascadeDelete=False, maxSelect=1, minSelect=1),
field("owner", "text", required=True, min=1, max=128),
],
"indexes": [
"CREATE UNIQUE INDEX `idx_clock_correction_lock_employee` ON `clock_correction_locks` (`employee`)",
],
}
DEFS["fuel_sheets"] = {
"name": "FUEL_SHEETS", "type": "base",
# site is optional: a pump sheet can span sites, resolved per-row.
"fields": sheet_fields(site_required=False),
"indexes": [
"CREATE INDEX `idx_fuel_sheets_hash` ON `fuel_sheets` (`file_hash`)",
"CREATE INDEX `idx_fuel_sheets_status` ON `fuel_sheets` (`processing_status`, `date_uploaded`)",
],
}
DEFS["fuel_logs"] = {
"name": "FUEL_LOGS", "type": "base",
"fields": [
field("site", "relation", required=True,
collectionId="__SITES_ID__", cascadeDelete=False, maxSelect=1),
field("vehicle", "relation", required=False,
collectionId="__VEHICLES_ID__", cascadeDelete=False, maxSelect=1),
field("date_logged", "date", required=False),
field("litres", "number", required=False),
field("odo_reading", "number", required=False),
field("latest_consumption", "number", required=False),
# GPS fix captured at scan time on the kiosk phone (MEGA Clock
# Fleet tab). Optional — the spreadsheet import path has no GPS
# context, so desktop-imported rows stay null. Range validation
# is the route's job; PB stays permissive.
field("latitude", "number", required=False),
field("longitude", "number", required=False),
field("location_accuracy_m", "number", required=False),
field("fuel_sheet", "relation", required=False,
collectionId="__FUEL_SHEETS_ID__", cascadeDelete=False, maxSelect=1),
],
"indexes": [
"CREATE INDEX `idx_fuel_logs_site_date` ON `fuel_logs` (`site`, `date_logged`)",
"CREATE INDEX `idx_fuel_logs_vehicle_date` ON `fuel_logs` (`vehicle`, `date_logged`)",
],
}
DEFS["plant_hours_sheets"] = {
"name": "PLANT_HOURS_SHEETS", "type": "base",
# site is optional, same reasoning as fuel_sheets.
"fields": sheet_fields(site_required=False),
"indexes": [
"CREATE INDEX `idx_plant_hours_sheets_hash` ON `plant_hours_sheets` (`file_hash`)",
"CREATE INDEX `idx_plant_hours_sheets_status` ON `plant_hours_sheets` (`processing_status`, `date_uploaded`)",
],
}
DEFS["plant_hours_log"] = {
"name": "PLANT_HOURS_LOG", "type": "base",
"fields": [
field("date_logged", "date", required=True),
field("vehicle_qr", "text", required=True),
field("vehicle", "relation", required=True,
collectionId="__VEHICLES_ID__", cascadeDelete=False, maxSelect=1, minSelect=1),
field("site", "relation", required=True,
collectionId="__SITES_ID__", cascadeDelete=False, maxSelect=1, minSelect=1),
field("start_date", "date", required=True),
field("end_date", "date", required=True),
field("start_time", "text", required=False),
field("end_time", "text", required=False),
field("start_hours", "number", required=True),
field("end_hours", "number", required=True),
field("hours", "number", required=True),
field("notes", "text", required=False),
field("hour_meter_budget", "number", required=True),
field("cost_per_hour", "number", required=True),
field("value", "number", required=True),
field("over_budget", "bool", required=False),
field("hours_mismatch", "bool", required=False),
field("vehicle_unit_type_at_import", "select", required=True,
values=["fuel", "plant"], maxSelect=1),
field("plant_hours_sheet", "relation", required=False,
collectionId="__PLANT_HOURS_SHEETS_ID__", cascadeDelete=False, maxSelect=1, minSelect=0),
],
"indexes": [
"CREATE UNIQUE INDEX `idx_plant_hours_log_qr_start` ON `plant_hours_log` (`vehicle_qr`, `start_date`)",
"CREATE INDEX `idx_ph_log_vehicle_start` ON `plant_hours_log` (`vehicle`, `start_date`)",
],
}
DEFS["income_claims"] = {
"name": "INCOME_CLAIMS", "type": "base",
"fields": [
field("claim_number", "text", required=True),
field("site", "relation", required=False,
collectionId="__SITES_ID__", cascadeDelete=False, maxSelect=1),
field("period_start", "date", required=False),
field("period_end", "date", required=False),
field("gross_amount", "number", required=False),
field("certified_by", "text", required=False),
field("paid_on", "date", required=False),
field("status", "select", required=False,
values=["draft", "certified", "paid"], maxSelect=1),
field("is_verified", "bool", required=False),
field("notes", "text", required=False),
],
"indexes": [],
}
DEFS["delivery_notes"] = {
"name": "DELIVERY_NOTES", "type": "base",
"fields": [
field("receiving_identity_key", "text", required=False, max=100),
field("processing_status", "select", required=False,
values=["pending", "ocr-complete", "auto-verified",
"needs-review", "manual-verified", "failed"], maxSelect=1),
field("delivery_note_number", "text", required=False),
field("site", "relation", required=True,
collectionId="__SITES_ID__", cascadeDelete=False, maxSelect=1),
field("purchase_order", "relation", required=False,
collectionId="__PURCHASE_ORDERS_ID__", cascadeDelete=False, maxSelect=1),
field("date_received", "date", required=False),
field("received_by", "text", required=False),
field("slip_image", "file", required=False,
maxSelect=1, maxSize=20971520,
mimeTypes=["image/jpeg", "image/png", "image/webp", "application/pdf"]),
field("raw_ocr_data", "json", required=False),
field("is_verified", "bool", required=False),
field("ocr_confidence", "number", required=False),
field("ocr_completed_at", "date", required=False),
field("received_by_name", "text", required=False),
field("file_hash", "text", required=False),
],
"indexes": [
"CREATE INDEX `idx_dn_site_date` ON `delivery_notes` (`site`, `date_received`)",
"CREATE INDEX `idx_dn_status` ON `delivery_notes` (`processing_status`)",
],
}
DEFS["delivery_note_items"] = {
"name": "DELIVERY_NOTE_ITEMS", "type": "base",
"fields": [
field("delivery_note", "relation", required=False,
collectionId="__DELIVERY_NOTES_ID__", cascadeDelete=True, maxSelect=1),
field("purchase_order_item", "relation", required=False,
collectionId="__PURCHASE_ORDER_ITEMS_ID__", cascadeDelete=False, maxSelect=1),
field("qty_received", "number", required=False),
field("condition_notes", "text", required=False),
field("allocation_key", "text", required=False, max=100),
field("source_line_keys", "json", required=False, maxSize=10000),
field("booking", "json", required=False, maxSize=100000),
],
"indexes": ["CREATE UNIQUE INDEX idx_dn_allocation_key ON delivery_note_items (allocation_key) WHERE allocation_key != ''"],
}
# Dependency order: a collection referencing another via `relation` must be
# created after the collection it points to, so its `collectionId`
# placeholder can be resolved.
ORDER = [
"sites",
"employees",
"employee_rates",
"employee_sick_notes",
"suppliers",
"purchase_orders",
"purchase_order_items",
"fixed_costs",
"vehicles",
"fuel_prices",
"timesheets",
"timesheet_entries",
"clock_events",
"clock_missed_out_corrections",
"clock_correction_locks",
"fuel_sheets",
"fuel_logs",
"plant_hours_sheets",
"plant_hours_log",
"income_claims",
"delivery_notes",
"delivery_note_items",
]
ids = {}
for key in ORDER:
body = dict(DEFS[key])
body.update(LOCKED)
for f in body["fields"]:
if f.get("type") == "relation":
cid = f.get("collectionId", "")
# placeholders use collection table name in upper, e.g. __SITES_ID__
for other_key, other_id in ids.items():
collection_name_upper = DEFS[other_key]["name"] # e.g. "SITES"
cid = cid.replace(f"__{collection_name_upper}_ID__", other_id)
if "__" in cid:
sys.exit(f"unresolved placeholder in {key}.{f['name']}: {cid}")
f["collectionId"] = cid
code, resp = http("POST", "/api/collections", body, TOKEN)
if code == 200:
ids[key] = resp["id"]
print(f" ✓ {body['name']:25s} → {resp['id']}")
elif code == 400:
code2, existing = http("GET", f"/api/collections/{body['name']}", token=TOKEN)
if code2 == 200:
ids[key] = existing["id"]
print(f" • {body['name']:25s} → {existing['id']} (already existed)")
else:
print(f" ✗ {body['name']:25s} FAILED: {resp}", file=sys.stderr)
sys.exit(1)
else:
print(f" ✗ {body['name']:25s} FAILED: {code} {resp}", file=sys.stderr)
sys.exit(1)
print(f"\nDone. {len(ids)} collections on {PB}.")
Reference in a new issue View git blame Copy permalink