From 8eb256360b3bcf0c437bd01d89a2f20da03e2377 Mon Sep 17 00:00:00 2001 From: DEls Date: Fri, 11 Sep 2026 16:17:41 +0200 Subject: [PATCH] docs(schema): bring bootstrap and data-model docs back to truth pb-bootstrap-schema.py -- the script a fresh PocketBase instance would run against -- had fallen ten migrations behind and still described the pre-restructure world. Rewrote it field-by-field against a live production dump (17 collections, every field/required-flag/relation/ index verified). Found and fixed two pre-existing gaps unrelated to this restructure while verifying: fuel_prices and income_claims were entirely absent from the bootstrap script. pb-relax-rules.py's collection list updated to match. No local PocketBase binary is available in this environment, so the plan's "bootstrap fresh and diff against production" verification step was replaced with a manual field-by-field comparison against the live dump -- same goal, different mechanism, documented as a substitution rather than silently skipped. Added site scoping to the one agent tool that still lacked it (list_imports, backed by loadImportBatches) -- every other list_* tool already had it. Preparation for the future per-user site-allowlist work, not part of this restructure itself. Rewrote docs/data-model.md and docs/data-flows.md end to end: the site-attribution table, the collection reference, the five ingestion pipelines, and the schema-drift section that was describing the exact problem this task fixes. Every remaining data_import_batches reference in both docs is now historical context, not a current description. This is the last task of the ingestion restructure (docs/superpowers/plans/2026-09-11-ingestion-restructure.md). Co-Authored-By: Claude Sonnet 5 --- docs/data-flows.md | 332 +++++++++------- docs/data-model.md | 353 +++++++++++------- .../pb-bootstrap-schema.cpython-314.pyc | Bin 12099 -> 16582 bytes .../pb-relax-rules.cpython-314.pyc | Bin 0 -> 4409 bytes scripts/pb-bootstrap-schema.py | 222 ++++++++--- scripts/pb-relax-rules.py | 12 +- src/lib/agent/load.ts | 25 +- src/lib/agent/tools.ts | 12 + 8 files changed, 611 insertions(+), 345 deletions(-) create mode 100644 scripts/__pycache__/pb-relax-rules.cpython-314.pyc diff --git a/docs/data-flows.md b/docs/data-flows.md index a1b3668..59be033 100644 --- a/docs/data-flows.md +++ b/docs/data-flows.md @@ -2,21 +2,37 @@ > End-to-end ingestion pipelines with sequence diagrams and the exact site-attribution strategy each one uses. -Every importer writes into a `data_import_batches` row, which carries the -original file for the audit trail and the resulting `processed_record_id`. -Three of the pipelines are synchronous (the file is parsed and inserted in -the same request) and two are async (a separate worker, n8n, processes -them and POSTs results back). +Reflects the shape produced by the 2026-09-11 ingestion restructure +(`docs/superpowers/plans/2026-09-11-ingestion-restructure.md`). The shared +`data_import_batches` pool is gone. Two pipeline shapes replaced it: + +- **The three sheet imports** (timesheets, fuel logs, plant hours) are + still synchronous, but each now creates its own typed "uploaded document" + row (`timesheets`, `fuel_sheets`, `plant_hours_sheets`) instead of a + generic batch row, and computes its real outcome via + `resolveDocumentOutcome()` (`src/lib/document-status.ts`) once parsing is + done — never hardcoding `status: processed` before a single row exists. +- **Purchase orders** moved from synchronous-upload-then-async-batch to + fully async against `purchase_orders` itself: the row is born at upload + time with a placeholder number and `processing_status: pending`, and an + n8n cron claims, extracts, and fills it in. There is no batch row at any + point in this pipeline any more. +- **Delivery notes** keep their existing shape (site + file stamped at + upload, OCR result posted back by webhook) minus the batch row. --- -## 1. Timesheet import (synchronous) +## 1. Timesheet import (synchronous, document-first) **Route:** `POST /api/timesheets/import` (`src/app/api/timesheets/import/route.ts`) -**Input:** `multipart/form-data` with `file` (xlsx / xls / csv) plus optional `siteId` and `createMissingEmployees` form fields -**Output:** `{ batchId, totalRows, imported, skipped, errors, detectedFormat, createdEmployees }` +**Input:** `multipart/form-data` with `file` (xlsx / xls / csv), **required** `siteId`, and optional `createMissingEmployees` +**Output:** `{ documentId, totalRows, imported, skipped, errors, detectedFormat, createdEmployees }` **Triggers:** `ImportTimesheetsButton` on the Workforce page +Unlike the fuel/plant-hours sheets, a wage sheet always belongs to exactly +one site — `siteId` is required and the upload is rejected before anything +is parsed if it's missing. + **Format auto-detection** (the route walks every sheet, first match wins): 1. **Hills Casual Matrix** — `looksLikeCasualMatrix(rows)` (`src/lib/timesheet-casual-parse.ts`) → one long row per (employee, day). Hours are `HH:MM`, "Absent" cells are skipped, names with leading commas are cleaned. @@ -30,39 +46,48 @@ sequenceDiagram participant U as User (browser) participant R as /api/timesheets/import participant P as PocketBase - U->>R: multipart { file, siteId?, createMissingEmployees? } + U->>R: multipart { file, siteId (required), createMissingEmployees? } R->>R: parseFile() → uniform long rows - alt createMissingEmployees && siteId + R->>P: create(timesheets, site=siteId, source_file, file_hash, processing_status=pending) + alt createMissingEmployees R->>P: getFullList(employees, fields="id,employee_no") R->>P: create(missing employees, casual, site=siteId) end - R->>P: create(data_import_batches, batch_type=timesheets, original_file) - R->>P: getFullList(employees, fields="id,employee_no") + R->>P: getFullList(employees, fields="id,employee_no,site") loop for each parsed row - R->>P: create(timesheets, employee=lookup[row.employee_no]) + R->>P: create(timesheet_entries, employee=lookup[row.employee_no], site=employeeSite.get(employeeId) || siteId) end - R-->>U: { batchId, totalRows, imported, skipped, errors, ... } + R->>R: resolveDocumentOutcome({ rowsTotal, rowsCreated, rowErrors }, now) + R->>P: update(timesheets doc, processing_status, rows_total, rows_created, rows_failed, row_errors, last_error) + R-->>U: { documentId, totalRows, imported, skipped, errors, ... } ``` -**Site attribution.** Timesheets do not carry a `site` field. The route -writes the row against the employee record; site attribution happens at -*read time* in the metrics helpers by joining through -`employeeRates.get(timesheet.employee).site`. This is the only -collection in the system with no `site` or `project_code` on the row itself. +**Site attribution.** `timesheet_entries` (the collection that used to be +called `timesheets` before the rename) now carries a **required** `site` +relation, stamped from the employee's site at the moment of import — not +joined at read time. This is a deliberate change: the old at-read-time join +through `employees.site` meant a worker moving sites silently rewrote the +site attribution of every historical hour they'd ever logged. Stamping at +write time freezes each entry's site to what it actually was that day. **Failure behaviour.** - Unknown `employee_no` (and `createMissingEmployees` not set) → row errored, import continues. -- Per-row `pb.collection("timesheets").create` failure → row errored, import continues. +- Per-row `pb.collection("timesheet_entries").create` failure → row errored, import continues. - File parse failure → 400 with a top-level `error` and `details`. +- `resolveDocumentOutcome()` marks the document `failed` (not `processed`) + if the file had zero data rows, or if parsing produced rows but zero of + them landed — see `src/lib/document-status.ts`. A partial import (some + rows landed, some failed) is `processed` with `rows_failed > 0` and + `last_error` summarising the count; it is not silently reported clean. --- -## 2. Fuel log import (synchronous) +## 2. Fuel log import (synchronous, document-first) **Route:** `POST /api/fuel-logs/import` (`src/app/api/fuel-logs/import/route.ts`) -**Input:** `multipart/form-data` with `file` (xlsx or csv) -**Output:** `{ batchId, totalRows, imported, skipped, errors, qrMatched, descMatched }` +**Input:** `multipart/form-data` with `file` (xlsx or csv), optional `siteId` +**Output:** `{ documentId, totalRows, imported, skipped, errors, qrMatched, descMatched }` **Triggers:** `ImportFuelLogsButton` on the Fleet page **Expected columns** (case-insensitive, alias-resolved by `resolveColumn` in `src/lib/fuel-import-parse.ts`): @@ -84,51 +109,56 @@ sequenceDiagram participant U as User (browser) participant R as /api/fuel-logs/import participant P as PocketBase - U->>R: multipart { file } + U->>R: multipart { file, siteId? } R->>R: SheetJS parse → rowsRaw - R->>P: create(data_import_batches, batch_type=fuel_logs, original_file, status=processed) - R->>P: getFullList(vehicles, fields="id,description,project_code,qr_code_id") - R->>P: getFullList(sites, fields="id,name,project_code") + R->>P: create(fuel_sheets, site=siteId?, source_file, file_hash, processing_status=pending) + R->>P: getFullList(vehicles, fields="id,description,site,qr_code_id") + R->>P: getFullList(sites, fields="id,name") R-->>R: vehicleByQr, vehicleByDesc, siteByName maps loop for each row R->>R: parseDate, toNumber R->>R: vehicle = vehicleByQr[qr] ?? vehicleByDesc[desc] - R->>R: site = siteByName[row.site.toLowerCase()] - R->>R: projectCode = site?.projectCode ?? vehicle.projectCode - R->>P: create(fuel_logs, site, vehicle, project_code, ...) + R->>R: rowSiteId = siteByName[row.site.toLowerCase()] ?? vehicle.site + R->>P: create(fuel_logs, site=rowSiteId, vehicle, fuel_sheet=doc.id, ...) end - R-->>U: { batchId, totalRows, imported, skipped, errors, qrMatched, descMatched } + R->>R: resolveDocumentOutcome({ rowsTotal, rowsCreated, rowErrors }, now) + R->>P: update(fuel_sheets doc, processing_status, rows_total, rows_created, rows_failed, row_errors, last_error) + R-->>U: { documentId, totalRows, imported, skipped, errors, qrMatched, descMatched } ``` -**Site attribution strategy.** Two-tier match: +**Site attribution strategy.** Two-tier match, same shape as before the +restructure, but the destination is a real required `site` relation on +`fuel_logs` instead of a denormalised `project_code` fallback: 1. The row's `Current Site` cell is matched case-insensitive against - `sites.name`. If found, the site relation and its `project_code` are - stamped on the row. -2. If the row has no site cell (legacy CSVs), the row's `project_code` - is set to the matched vehicle's `project_code` and the `site` relation - is left **empty**. The fuel log exists but isn't tied to a site by - relation — the only link is the text `project_code`. + `sites.name`. If found, that site is stamped on the row. +2. If the row has no site cell (legacy CSVs) or no match, the row falls + back to the matched vehicle's current `site`. -This is the seam where **site attribution goes wrong** as the fleet grows — -see [multi-site-assessment §4](./multi-site-assessment.md#4-site-resolution-by-name-string). +This is still the seam where **site attribution goes wrong** as the fleet +grows — see [multi-site-assessment §4](./multi-site-assessment.md#4-site-resolution-by-name-string). Two sites with near-identical names, a renamed site, or a vehicle moved -between sites all produce silently wrong attribution. +between sites all still produce silently wrong attribution; what changed is +that there is no separate `project_code` fallback path to reason about any +more — one relation, one resolution order. **Failure behaviour.** - Unknown vehicle (no QR match, no description match) → row errored, import continues. - Invalid date / odo / litres → row errored, import continues. - PB insert failure → row errored, import continues; the importer extracts - the field-level validation message via `formatPbError` (lines 65-78). + the field-level validation message via `formatPbError`. +- `resolveDocumentOutcome()` governs the document's final status exactly as + in the timesheet pipeline (§1) — zero rows parsed or zero rows landed is + `failed`, never `processed`. --- -## 3. Plant hours import (synchronous) +## 3. Plant hours import (synchronous, document-first) **Route:** `POST /api/plant-hours/import` (`src/app/api/plant-hours/import/route.ts`) -**Input:** `multipart/form-data` with `file` (xlsx or csv, max 10 MB) -**Output:** `{ batchId, totalRows, imported, skipped, mismatched, warnings, errors }` +**Input:** `multipart/form-data` with `file` (xlsx or csv, max 10 MB), optional `siteId` +**Output:** `{ documentId, totalRows, imported, skipped, mismatched, warnings, errors }` **Orchestrator:** `importPlantHours` in `src/lib/plant-hours-import.ts` **Triggers:** `ImportPlantHoursButton` on the Plant page @@ -140,61 +170,70 @@ sequenceDiagram participant R as /api/plant-hours/import participant O as importPlantHours (pure) participant P as PocketBase - U->>R: multipart { file } + U->>R: multipart { file, siteId? } R->>R: SheetJS parse - R->>P: create(data_import_batches, batch_type=plant_hours, original_file, status=processed) - R->>P: getFullList(vehicles, fields="id,description,project_code,qr_code_id,unit_type,site,cost_per_hour", perPage=5000) - R->>P: getFullList(sites, fields="id,name,project_code", perPage=5000) - R->>O: importPlantHours(rowsRaw, batchId, { vehicles, sites }) + R->>P: create(plant_hours_sheets, site=siteId?, source_file, file_hash, processing_status=pending) + R->>P: getFullList(vehicles, fields="id,description,qr_code_id,unit_type,site,cost_per_hour", perPage=5000) + R->>P: getFullList(sites, fields="id,name", perPage=5000) + R->>O: importPlantHours(rowsRaw, { vehicles, sites }) O->>O: resolve each row by qr_code_id (or desc fallback) O->>O: compute hours, value, over_budget, hours_mismatch O->>O: snapshot vehicle_unit_type_at_import loop for each prepped row - R->>P: create(plant_hours_log, ...) + R->>P: create(plant_hours_log, site=vehicle.site, plant_hours_sheet=doc.id, ...) alt duplicate (vehicle_qr, start_date) P-->>R: 400 validation_not_unique R->>R: record as "duplicate: vehicle_qr=X start_date=Y" end end - R-->>U: { batchId, totalRows, imported, skipped, mismatched, warnings, errors } + R->>R: resolveDocumentOutcome({ rowsTotal, rowsCreated, rowErrors }, now) + R->>P: update(plant_hours_sheets doc, processing_status, rows_total, rows_created, rows_failed, row_errors, last_error) + R-->>U: { documentId, totalRows, imported, skipped, mismatched, warnings, errors } ``` **Site attribution.** The orchestrator pulls the site from the matched -vehicle (`vehicle.site`) and stamps both `site` and `project_code` on -the row. This is the **only collection where `site` is required -(`minSelect=1`)** — every plant_hours_log row has a non-null site. +vehicle (`vehicle.site`) and stamps it on the row. This remains the +**only collection where `site` is required (`minSelect=1`)** on the log +itself — every `plant_hours_log` row has a non-null site. **Composite unique index.** `idx_plant_hours_log_qr_start` on `(vehicle_qr, start_date)` blocks accidental double-imports of the same week for the same vehicle. The route surfaces the PB-level rejection as -a friendly `duplicate: ...` error per row, not a batch-level failure. +a friendly `duplicate: ...` error per row, not a document-level failure. See [multi-site-assessment §5](./multi-site-assessment.md#5-vehicles-plant-and-employees-moving-between-sites) for the cross-site implication of this index. **Failure behaviour.** Per-row. The orchestrator collects row-level -errors and warnings; the route appends PB insert errors. The batch -itself is never aborted by one bad row. +errors and warnings; the route appends PB insert errors. The document +itself is never aborted by one bad row — `resolveDocumentOutcome()` decides +the final status the same way as §1/§2. -**Exception report.** `GET /api/plant-hours/exception-report/[batchId]` +**Exception report.** `GET /api/plant-hours/exception-report/[sheetId]` reconstructs a per-row exception CSV by re-parsing the source file -from the batch's `original_file` and matching against the imported rows -in `plant_hours_log.filter('batch_reference = ""')`. Used to +from the document's `source_file` and matching against the imported rows +in `plant_hours_log.filter('plant_hours_sheet = ""')`. Used to re-export a corrected sheet when the operator needs to fix and re-import. --- -## 4. Purchase order upload → ingest (async via n8n) +## 4. Purchase order upload → ingest (async, no batch row) -Three routes cooperate. The browser never talks to n8n directly. +Two routes cooperate; the browser never talks to n8n directly. There is no +longer a poll-for-status route, a separate batch collection, or a "move the +file onto the new row" step — the `purchase_orders` row *is* the queue item +from the moment it's created. **Routes:** - `POST /api/purchase-orders/upload` — `src/app/api/purchase-orders/upload/route.ts` - `POST /api/purchase-orders/process-batch` — called by n8n — `src/app/api/purchase-orders/process-batch/route.ts` -- `GET /api/data-import-batches/[id]/status` — poll target for the browser -**Auth model.** All three are server-side; only `process-batch` requires -the shared `N8N_INGEST_SECRET` header. The other two run as the -PocketBase superuser. +Full implementation detail, including the n8n node map, hardcoded values, +and a manual smoke-test script, lives in +`docs/n8n/purchase-order-ingest-implementation.md` (rewritten for this +shape in Task 15 of the restructure) — this section is the short version. + +**Auth model.** Both routes are server-side; `process-batch` additionally +requires the shared `N8N_INGEST_SECRET` header. **Pipeline:** @@ -203,66 +242,76 @@ sequenceDiagram participant U as User participant W as /api/purchase-orders/upload participant P as PocketBase - participant N as n8n (PO ingest workflow) + participant N as n8n (PO ingest cron, every ~3 min) participant B as /api/purchase-orders/process-batch - participant B2 as /api/data-import-batches/:id/status - U->>W: multipart { file (PDF/image) } + U->>W: multipart { file (PDF/image), siteId } W->>W: SHA-256 the bytes - W->>P: getFullList(batches, file_hash=X && status=processed && date_uploaded >= 30d_ago) + W->>P: getFullList(purchase_orders, file_hash=X && date_uploaded >= 30d_ago) alt duplicate - W-->>U: 200 { duplicate: true, existing_batch_id, ... } + W-->>U: 200 { duplicate: true, existing_id, ... } else new - W->>P: create(data_import_batches, batch_type=purchase_orders, status=pending, file_hash) - W-->>U: 202 { batch_id, status: queued } + W->>P: create(purchase_orders, site=siteId, purchase_order_number="PENDING-...", processing_status=pending, file_hash, purchase_order_file) + W-->>U: 202 { purchase_order_id, status: queued } end - N->>P: poll /api/files/data_import_batches/:id/ (admin auth) - N->>N: AI Agent extracts PO fields + line items - N->>B: POST { batch_id, extracted } + x-ingest-secret - B->>P: getOne(batch) — must be in 'processing' - B->>P: getFullList(purchase_orders, filter=po_number=...) — dedup - B->>B: matchSite(extracted.project_code) → siteId - B->>P: download batch.original_file via /api/files - B->>P: create(purchase_orders, file=..., site=siteId, raw_ocr_data, is_verified=false) - loop for each line item - B->>P: create(purchase_order_items, purchase_order=poRecord.id) + N->>P: PATCH purchase_orders/{id}?filter=processing_status="pending" (atomic claim → processing, attempts+1) + N->>P: GET /files/purchase_orders/{id}/{purchase_order_file} + N->>N: AI Agent extracts PO header + line items + N->>B: POST { purchase_order_id, extracted } + x-ingest-secret + B->>P: getOne(purchase_orders, id) — must still be 'processing' + B->>P: getFullList(purchase_orders, filter=purchase_order_number=extracted.number) — dedup + alt fill in (no existing PO with that number) + B->>P: update(this row, purchase_order_number=extracted.number, processing_status=processed, raw_ocr_data, is_verified=false) + loop for each line item + B->>P: create(purchase_order_items, purchase_order=this row.id) + end + else merge (a real PO with that number already exists) + B->>P: update(existing PO with any extracted fields it was missing) + B->>P: delete(this placeholder row) + else invalid extraction + B->>P: update(this row, processing_status=failed, last_error) // never deleted — stays visible and recoverable end - alt dedup branch - B->>P: update(batch, status=processed, processed_record_id=existing.id) // keep original_file - else success branch - B->>P: update(batch, status=processed, processed_record_id=poRecord.id, original_file="") - end - U->>B2: GET /api/data-import-batches/:id/status - B2-->>U: { id, status, processed_record_id, processed_record_label, ... } + B-->>N: { ok: true } or 4xx/5xx ``` -**Three dedup layers, in order:** +**Three-outcome `process-batch` contract.** This is the core of the async +design — one route, three possible endings for the *same* placeholder row, +never a fourth row created: + +1. **Fill in.** No existing PO shares the extracted number — the + placeholder row is updated in place with the real number, OCR payload, + and line items. This is the common case. +2. **Merge / dedup / delete placeholder.** A real PO with that number + already exists (a genuine duplicate upload, or two near-simultaneous + uploads of the same document). The existing PO is patched with anything + useful the new extraction found that it was missing, and *this* + placeholder row is deleted. The caller-visible id becomes the winner's. +3. **Mark failed.** The extraction was invalid (no PO number, garbage + output) — the row moves to `processing_status=failed` with `last_error` + set. It is **never deleted** on this path, so it stays visible and + recoverable in the dashboard, and the n8n-side escalation logic can + retry or eventually mark it `dead` after repeated failures. + +**Dedup layers, in order:** 1. **Hash dedup** at upload time — SHA-256 of the file bytes, matched - against batches in the last 30 days with `status=processed`. Returns - `200 { duplicate: true }` before any new batch row is written. -2. **AI-extracted PO number dedup** at process time — - `getFullList(purchase_orders, filter=po_number="...")`. If found, - the existing PO id is recorded on the batch but **the duplicate file - is kept on the batch row** for audit (`clearFile: false`). + against `purchase_orders` rows created in the last 30 days. Returns + `200 { duplicate: true }` before any new placeholder row is written. +2. **Extracted-number dedup** at process time — outcome 2 above. 3. **PB unique index** `idx_po_number` on - `purchase_orders(purchase_order_number)` — the final backstop. If - the AI dedup misses, this rejects the insert. + `purchase_orders(purchase_order_number)` — the final backstop. If the + dedup logic misses, this rejects the insert. -**Site attribution strategy.** `matchSite(pb, payload.project_code)` in -`src/lib/purchase-order-schema.ts`: +**Site attribution strategy.** `matchSite(pb, siteId or project code hint)` +in `src/lib/purchase-order-schema.ts` resolves the site once, at upload +time, before OCR ever runs — `purchase_orders.site` is a required field, so +there is no unmatched/null-site PO state to reason about downstream the way +there used to be. -1. Exact match on `sites.project_code`. -2. Fuzzy match `project_code ~ "X"` (PB's `~` operator does case-insensitive substring). -3. If both miss, the PO is stored with `site=null` and a warning is - surfaced: `unmatched_project_code: !siteId` is written into - `raw_ocr_data`. The `unmatched_project_code` flag is the operator's - signal that a manual assignment is needed. - -**Failure behaviour.** Per the `process-batch` header comment, the route -distinguishes `retryable: true` (5xx with a transient signal — n8n -retries) from `retryable: false` (400 — n8n marks the batch `dead` and -surfaces it in the dashboard). Network blips and PB write conflicts are -retryable; validation failures are not. +**Failure behaviour.** Per the `process-batch` route's own contract, it +distinguishes `retryable: true` (5xx with a transient signal — n8n retries) +from `retryable: false` (400 — n8n marks the row `dead` after enough +attempts and surfaces it in the dashboard). Network blips and PB write +conflicts are retryable; validation failures are not. --- @@ -287,9 +336,7 @@ sequenceDiagram participant R as /api/delivery-notes/:id/ocr-result participant A as /api/delivery-notes/:id/assign-po U->>U2: multipart { file, site, delivery_note_number? } - U2->>P: getOne(sites, fields="project_code") - U2->>P: create(delivery_notes, slip_image=file, site, project_code, processing_status=pending) - U2->>P: create(data_import_batches, batch_type=delivery_notes, processed_record_id=dn.id) + U2->>P: create(delivery_notes, slip_image=file, site, processing_status=pending, file_hash) U2-->>U: 201 { id, siteId, processingStatus: pending } N->>S: GET (x-ingest-secret) S->>P: getOne(delivery_notes, fields="slip_image") @@ -304,15 +351,14 @@ sequenceDiagram alt tier = auto R->>P: update(delivery_notes, processing_status=auto-verified, is_verified=true, ocr_confidence, ocr_completed_at, purchase_order, raw_ocr_data, ...) loop for each item - R->>P: create(received_items, delivery_note, purchase_order_item=validatedId, qty_received, condition_notes) + R->>P: create(delivery_note_items, delivery_note, purchase_order_item=validatedId, qty_received, condition_notes) end else tier = needs-review R->>P: update(delivery_notes, processing_status=needs-review, is_verified=false, ...) - R->>P: delete + recreate received_items (idempotent) + R->>P: delete + recreate delivery_note_items (idempotent) else tier = manual - R->>P: update(delivery_notes, processing_status=needs-review, ...) — no received_items + R->>P: update(delivery_notes, processing_status=needs-review, ...) — no delivery_note_items end - R->>P: update(data_import_batches, status=processed, ...) opt operator fixes U->>A: POST { po: } (manual fallback) A->>P: getOne(purchase_orders, id) @@ -324,11 +370,11 @@ sequenceDiagram | Confidence | PO resolved | Items verifiable | Tier | Effect | | --- | --- | --- | --- | --- | -| ≥ 0.8 | ✅ | ✅ | `auto` | `is_verified=true`, `processing_status=auto-verified`, received_items created with PO links | +| ≥ 0.8 | ✅ | ✅ | `auto` | `is_verified=true`, `processing_status=auto-verified`, delivery_note_items created with PO links | | ≥ 0.8 | ❌ | n/a | `needs-review` (downgraded) | `is_verified=false`, no auto-link — orphan DNs are never auto-verified | | ≥ 0.8 | ✅ | ❌ (local fetch empty but n8n supplied ids) | `needs-review` (downgraded) | We don't claim auto when we can't cross-check the links | -| 0.5..0.79 | any | any | `needs-review` | `is_verified=false`, received_items created without PO links | -| < 0.5 | any | any | `manual` | `is_verified=false`, **no received_items** — the operator must capture manually | +| 0.5..0.79 | any | any | `needs-review` | `is_verified=false`, delivery_note_items created without PO links | +| < 0.5 | any | any | `manual` | `is_verified=false`, **no delivery_note_items** — the operator must capture manually | **PO match priority:** exact `purchase_order_number` → normalised (strip non-alnum, lowercase) → Levenshtein (single-char edit, threshold @@ -338,24 +384,23 @@ so the exception inbox can show "matched by Levenshtein, 0.9" rather than a silent "no match". **Site attribution strategy.** Site is stamped at upload time from the -form field (`src/app/api/delivery-notes/upload/route.ts:26`); the OCR -callback does not change it. The PO the OCR picks is what links the -delivery to a site indirectly (PO.site). For DNs uploaded without a -site, the route 400s. +form field (`src/app/api/delivery-notes/upload/route.ts`), a required +field on the row; the OCR callback does not change it. For DNs uploaded +without a site, the route 400s. **Idempotency.** If the OCR webhook is replayed, the route -**deletes** the existing `received_items` for this DN and recreates them -within the same call. This avoids stale matches if the OCR rerun finds +**deletes** the existing `delivery_note_items` for this DN and recreates +them within the same call. This avoids stale matches if the OCR rerun finds different items. The DN update is patch-style so missing fields (delivery_note_number, date_received) are never overwritten by missing data. -**Failure behaviour.** If the DN update itself fails, the route marks -the batch `failed` so the operator dashboard reflects the broken state -instead of leaving it pending forever. If a single `received_items` -create fails with a 400 (FK race — the PO item was deleted between -fetch and create), the route retries with the link dropped and -annotates the row as `link dropped: orphan id`. +**Failure behaviour.** If the DN update itself fails, the route marks the +DN's `processing_status` as `failed` so the operator dashboard reflects the +broken state instead of leaving it pending forever. If a single +`delivery_note_items` create fails with a 400 (FK race — the PO item was +deleted between fetch and create), the route retries with the link dropped +and annotates the row as `link dropped: orphan id`. --- @@ -370,7 +415,14 @@ annotates the row as `link dropped: orphan id`. - **PB ID validation.** All `[id]` path params use the PB ID pattern (`/^[a-z0-9]{15}$/i`) to close the filter-injection sink before the route does `filter: "=''"`. -- **Audit trail.** Every batch keeps the original file via - `data_import_batches.original_file`. The PO route clears this field - on success (the file is moved to the PO row); it keeps the bytes on - the dedup path. The fuel/plant/timesheet routes never clear the file. +- **Audit trail.** Every document/queue collection (`timesheets`, + `fuel_sheets`, `plant_hours_sheets`, `purchase_orders`) keeps its own + original file on its own row via `source_file` (or `purchase_order_file` + for POs) — there is no separate batch collection to clear a file from any + more. `file_hash` on each of those, plus `delivery_notes`, drives + upload-time dedup. +- **No `created`/`updated` autodate fields.** This PocketBase instance + never enabled them on any collection. Several routes have comments + noting that sorting or filtering by `created`/`updated` 400s here — use + the collection's own `date_uploaded` / `date_logged` / `date_received` + field instead. diff --git a/docs/data-model.md b/docs/data-model.md index c08a8ae..02c502d 100644 --- a/docs/data-model.md +++ b/docs/data-model.md @@ -3,10 +3,17 @@ > Every PocketBase collection that backs the app, with fields, relations, and > the site-attribution strategy each one uses. -Source of truth for types: `src/lib/pb-types.ts` (13 collection interfaces). +Source of truth for types: `src/lib/pb-types.ts` (16 collection interfaces). Source of truth for the live schema: `scripts/pb-bootstrap-schema.py`. Production instance: `https://pb.mega.infra.apphub.co.za`. +This reflects the shape produced by the 2026-09-11 ingestion restructure +(`docs/superpowers/plans/2026-09-11-ingestion-restructure.md`): the shared +`data_import_batches` pool is gone, replaced by typed document collections +(`timesheets`, `fuel_sheets`, `plant_hours_sheets`) and, for purchase orders, +queue fields carried directly on `purchase_orders` itself. `project_code` +denormalisation is gone everywhere except `sites`. + --- ## 1. Entity-relationship diagram @@ -21,96 +28,96 @@ erDiagram SITES ||--o{ DELIVERY_NOTES : "site (relation)" SITES ||--o{ FUEL_LOGS : "site (relation)" SITES ||--o{ PLANT_HOURS_LOG : "site (relation)" + SITES ||--o{ TIMESHEET_ENTRIES : "site (relation, stamped at import)" + SITES ||--o{ TIMESHEETS : "site (relation, required)" + SITES ||--o{ FUEL_SHEETS : "site (relation, optional)" + SITES ||--o{ PLANT_HOURS_SHEETS : "site (relation, optional)" - EMPLOYEES ||--o{ TIMESHEETS : "employee (relation)" + EMPLOYEES ||--o{ TIMESHEET_ENTRIES : "employee (relation)" VEHICLES ||--o{ FUEL_LOGS : "vehicle (relation)" VEHICLES ||--o{ PLANT_HOURS_LOG : "vehicle (relation)" - DATA_IMPORT_BATCHES ||--o{ FUEL_LOGS : "batch_reference (relation)" - DATA_IMPORT_BATCHES ||--o{ TIMESHEETS : "batch_reference (relation)" - DATA_IMPORT_BATCHES ||--o{ PLANT_HOURS_LOG : "batch_reference (loose)" + TIMESHEETS ||--o{ TIMESHEET_ENTRIES : "timesheet (relation, optional)" + FUEL_SHEETS ||--o{ FUEL_LOGS : "fuel_sheet (relation, optional)" + PLANT_HOURS_SHEETS ||--o{ PLANT_HOURS_LOG : "plant_hours_sheet (relation, optional)" PURCHASE_ORDERS ||--|{ PURCHASE_ORDER_ITEMS : "purchase_order (cascade)" PURCHASE_ORDERS ||--o{ DELIVERY_NOTES : "purchase_order" - DELIVERY_NOTES ||--|{ RECEIVED_ITEMS : "delivery_note (cascade)" - PURCHASE_ORDER_ITEMS ||--o{ RECEIVED_ITEMS : "purchase_order_item" + DELIVERY_NOTES ||--|{ DELIVERY_NOTE_ITEMS : "delivery_note (cascade)" + PURCHASE_ORDER_ITEMS ||--o{ DELIVERY_NOTE_ITEMS : "purchase_order_item" ``` -The PLANT collection is intentionally absent from the diagram — it is a -deprecated legacy alias for the unified VEHICLES collection -(`src/lib/pb-types.ts:58-60`). +`data_import_batches` and the deprecated `plant` alias (0 rows, 0 code +references) were deleted entirely as part of this restructure — neither +exists in production any more. --- ## 2. Site attribution strategy, per collection -This is the table to read first. The question "where does this row belong?" -has three different answers depending on which collection you're in. +This is the table to read first. Every non-joining collection now carries a +real `site` relation; `project_code` survives only on `sites` itself, as the +one canonical, human-readable label. -| Collection | `site` relation | `project_code` text | Attribution source | Drift risk | -| --- | --- | --- | --- | --- | -| `sites` | n/a | required, **unique** | The site row itself | None — the unique index is authoritative | -| `employees` | ✅ | ✅ | Both, but `site` is canonical for metrics; `project_code` is denormalised at create time | Medium — changing a worker's site later leaves `project_code` stale | -| `vehicles` | ✅ | ✅ | Both, but `site` is the relation; `project_code` is denormalised at create time | High — fuel logs are filtered by `log.site`, not by the current vehicle's site | -| `fixed_costs` | ✅ | ✅ | `site` is the relation; `project_code` is denormalised | Low — the import routes always stamp both from the same source | -| `income_claims` | ✅ | ✅ | Same as fixed_costs | Low | -| `purchase_orders` | ✅ | ✅ | Same as fixed_costs | Low | -| `delivery_notes` | ✅ | ✅ | Same as fixed_costs | Low | -| `fuel_logs` | ✅ | ✅ | `site` is filled when the source row has a site; `project_code` is the canonical fallback when the site is missing | Medium — see [multi-site-assessment.md §5](./multi-site-assessment.md#5-vehicles-plant-and-employees-moving-between-sites) | -| `plant_hours_log` | ✅ **required** | ✅ **required** | Always set from the vehicle at import time | Lower than fuel_logs because required | -| `timesheets` | ❌ | ❌ | **Joined via `employees.site`** at read time — `timesheets` has no site or project_code of its own | High — site attribution is a per-employee property that can move | -| `purchase_order_items` | ❌ | ❌ | Inherited through `purchase_order → site` | None — never queried directly by site | -| `received_items` | ❌ | ❌ | Inherited through `delivery_note → site` | None — same | -| `data_import_batches` | ❌ | ❌ | Inferred from `processed_record_id` + the processed record's collection | n/a | +| Collection | `site` relation | Attribution source | Drift risk | +| --- | --- | --- | --- | +| `sites` | n/a | The site row itself; `project_code` is required + unique | None — the unique index is authoritative | +| `employees` | ✅ optional | The worker's current site assignment | Low — read directly, nothing denormalised to go stale | +| `vehicles` | ✅ optional | The vehicle's current site assignment | Low — same | +| `fixed_costs` | ✅ optional | Stamped by the entry form | Low | +| `income_claims` | ✅ optional | Stamped by the entry form | Low | +| `purchase_orders` | ✅ **required** | Resolved by `matchSite` at upload time | Low — required field, no null-site POs | +| `delivery_notes` | ✅ **required** | Stamped at upload time from the form | Low | +| `fuel_logs` | ✅ **required** | Resolved per-row during sheet import (site cell, or vehicle's current site as fallback) | Low — required field | +| `plant_hours_log` | ✅ **required** | Always set from the matched vehicle's site at import time | Low — required field | +| `timesheets` (document) | ✅ **required** | A wage sheet belongs to one site, stamped at upload | None | +| `timesheet_entries` | ✅ **required** | **Stamped from the employee's site at import time** — not joined at read time | None — this ends the historical drift the old `timesheets` collection had when a worker moved sites after the fact | +| `fuel_sheets` (document) | ✅ optional | A pump sheet can span sites; left empty and resolved per-row against `fuel_logs.site` | Low | +| `plant_hours_sheets` (document) | ✅ optional | Same reasoning as `fuel_sheets` | Low | +| `purchase_order_items` | ❌ | Inherited through `purchase_order → site` | None — never queried directly by site | +| `delivery_note_items` | ❌ | Inherited through `delivery_note → site` | None — same | -**Reading the table.** Every collection except the two joining-tables and -`data_import_batches` carries both a `site` relation and a `project_code` -string. The only collection that carries **neither** is `timesheets` — and -that's the collection whose site attribution drifts the most, because it's -synthesised at read time from the employee record. +**Reading the table.** `timesheet_entries` (the renamed collection that used +to be called `timesheets`, and used to have no site of its own) is no longer +the worst drift risk in the system — it now carries a `site` relation that is +**stamped once, at import time**, from the employee's site as it was that +day. A worker moving sites later does not silently rewrite history the way +the old at-read-time join through `employees.site` did. The two join tables +(`purchase_order_items`, `delivery_note_items`) remain the only collections +with no site attribution of their own, by design — they're always reached +through their parent. --- -## 3. The `project_code` denormalisation +## 3. The `project_code` denormalisation — removed -`project_code` is a free-text identifier like `MEG-001` or `HILLS-2026` that -uniquely names a contract. It is the human-readable label that appears on -every report and filter dropdown. +Before this restructure, `project_code` (a free-text identifier like +`MEG-001`) was copied onto every child collection as a denormalised string +at insert time, alongside the `site` relation. That copy was never updated +when the parent site's `project_code` changed, which was the single biggest +drift risk in the schema (see the old §3 in git history for the full +writeup). -The schema carries it on every child collection as a *denormalised copy* of -the parent site's `project_code`. It is **not** a foreign key — it is just a -string, written at insert time, never updated when the parent site changes. +**Current state:** `project_code` now lives in exactly one place — +`sites.project_code`, required and unique +(`idx_sites_project_code`, enforced in `scripts/pb-bootstrap-schema.py`). +Every other collection that needs a human-readable site label resolves it by +expanding or joining the `site` relation at read time +(`src/lib/agent/load.ts`'s `loadSiteRefs()` is the canonical example). There +is no denormalised copy left anywhere to drift. -**Why it exists.** The chat agent and several older reports filter by -`project_code` (text) rather than expanding the `site` relation -(`expand: "site"` would force a join the dashboard pages don't want). It is -also what the import routes use as the canonical key when a row's `site` -relation is empty. - -**The drift problem.** If you change a site's `project_code` in PocketBase, -every existing `fuel_logs`, `fixed_costs`, `employees`, `vehicles`, -`purchase_orders`, `delivery_notes`, and `income_claims` row keeps the -*old* string forever. Code that joins through the `site` relation sees the -new value; code that filters on `project_code` text sees the old. - -`scripts/pb-relax-rules.py:24` and the `PATCH /api/sites/[id]` route -(`src/app/api/sites/[id]/route.ts:23-28`) deliberately **refuse to let you -change `project_code` on an existing site** — that is exactly the rule -designed to prevent drift. The `POST /api/sites` route also enforces a 64-char -ceiling and uniqueness via the `idx_sites_project_code` index -(`scripts/pb-bootstrap-schema.py:84`). - -What this means in practice: the denormalisation is **stable** as long as no -one re-creates a site with the same name and a different code. The biggest -remaining risk is bulk imports that resolve site by name and stamp the -wrong `project_code` onto historical rows — see [multi-site-assessment.md -§4](./multi-site-assessment.md#4-site-resolution-by-name-string). +`scripts/pb-relax-rules.py` and the `PATCH /api/sites/[id]` route +(`src/app/api/sites/[id]/route.ts`) still refuse to let you change +`project_code` on an existing site — that rule now protects the single +source of truth rather than a chain of denormalised copies, but it is +exactly as important as it always was: renaming a site is a new site, not an +edit. --- ## 4. Collection reference Each entry below lists every typed field, the relation direction, and any -notable constraints. Optional/legacy fields are marked. +notable constraints. ### 4.1 `sites` @@ -125,15 +132,15 @@ The site is the unit of business. One row per construction contract. | `work_hours` | number | optional | Total project work hours | | `hours_per_week` | number | optional | Standard workweek for the site | | `wages_hours_per_day` | number | optional | Standard wage day length | +| `latitude`, `longitude` | number | optional | Site coordinates | | `job_start_date` | date | optional | ISO date; the projection helper treats this as the authoritative start when present | ### 4.2 `employees` | Field | Type | Notes | | --- | --- | --- | -| `project_code` | text | Denormalised from `site.project_code` at create time | | `site` | relation → `sites` | The site the worker is currently assigned to | -| `name`, `employee_no`, `position` | text | `employee_no` is the cross-system payroll id; **uppercased** on import (`src/app/api/timesheets/import/route.ts:289`) | +| `name`, `employee_no`, `position` | text | `employee_no` is the cross-system payroll id; **uppercased** on import | | `rate_per_hour` | number | Drives payroll burn | | `hours_per_day_standard` | number | 9 by default for auto-created casuals | | `worker_type` | select | `permanent` or `casual` | @@ -142,7 +149,6 @@ The site is the unit of business. One row per construction contract. | Field | Type | Notes | | --- | --- | --- | -| `project_code` | text | Denormalised | | `site` | relation → `sites` | Current assignment | | `description` | text | e.g. "Toyota Hilux 2.4 GD-6" | | `standard_work_hours` | text/number | Free-text on schema; numeric in practice. Used by the plant utilisation calc | @@ -152,8 +158,9 @@ The site is the unit of business. One row per construction contract. | `unit_type` | select | `fuel` or `plant` (default `fuel`). Splits the bucket between Fleet and Plant pages | | `attention_dismissed_at` | date | When the user dismissed the vehicle's "over fuel limit" alert from the fuel page | -The legacy `plant` collection is a deprecated alias kept around for old -importers. New plant items go in `vehicles` with `unit_type="plant"`. +The `PLANT` collection (deprecated legacy alias) was deleted in this +restructure — it had 0 rows and 0 code references. All plant items live in +`vehicles` with `unit_type="plant"`. ### 4.4 `fixed_costs` @@ -161,7 +168,6 @@ Monthly or one-off costs. `is_recurring` is the partition key. | Field | Type | Notes | | --- | --- | --- | -| `project_code` | text | Denormalised | | `site` | relation → `sites` | | | `date_logged` | date | Drives the monthly tile and trend charts | | `description`, `notes` | text | | @@ -173,7 +179,6 @@ Monthly or one-off costs. `is_recurring` is the partition key. | Field | Type | Notes | | --- | --- | --- | | `claim_number` | text | The contractor's IPN / claim ref | -| `project_code` | text | Denormalised | | `site` | relation → `sites` | | | `period_start`, `period_end` | date | The period the claim covers | | `gross_amount` | number | Rand | @@ -183,13 +188,27 @@ Monthly or one-off costs. `is_recurring` is the partition key. | `is_verified` | bool | Verified by a human | | `notes` | text | | -### 4.6 `purchase_orders` +### 4.6 `fuel_prices` + +Unchanged by this restructure. Global (not per-site) pump price history. | Field | Type | Notes | | --- | --- | --- | -| `purchase_order_number` | text | Unique (`idx_po_number`). Normalised to uppercase with "PO" prefix stripped before insert (`src/lib/purchase-order-schema.ts`) | -| `project_code` | text | Denormalised | -| `site` | relation → `sites` | Resolved by `matchSite` at ingest time | +| `effective_from` | date | Required, unique (`idx_fuel_prices_effective_from`) | +| `price_per_litre` | number | Required | +| `notes` | text | optional | + +### 4.7 `purchase_orders` + +Now carries the queue state `data_import_batches` used to carry — see +`docs/n8n/purchase-order-ingest-implementation.md` for the full async +pipeline. The row is created at upload time with a placeholder number and +`processing_status: pending`; there is no longer a separate batch row. + +| Field | Type | Notes | +| --- | --- | --- | +| `purchase_order_number` | text | Unique (`idx_po_number`). Placeholder `PENDING-...` (`src/lib/po-placeholder.ts`) until OCR fills it in | +| `site` | relation → `sites`, **required** | Resolved by `matchSite` at upload time | | `supplier` | text | | | `required_by_date` | date | When the materials are needed | | `ordered_by` | text | Who placed it | @@ -197,11 +216,20 @@ Monthly or one-off costs. `is_recurring` is the partition key. | `purchase_order_file` | file | The original PDF/image; mime allowlist enforced by the upload route | | `raw_ocr_data` | json | Full OCR payload + ingest metadata | | `is_verified` | bool | Human-verified flag | +| `file_hash` | text | SHA-256 of the original file; drives the upload-time dedup window | +| `processing_status` | select | `pending` / `processing` / `processed` / `failed` / `dead` | +| `attempts` | number | Incremented on each claim | +| `last_error` | text | Last failure reason | +| `next_retry_at`, `last_attempt_at` | date | Backoff scheduling for the n8n cron | +| `date_uploaded` | date | | -### 4.7 `purchase_order_items` +Index: `idx_po_site_status` on `(site, processing_status)` — the shape the +n8n cron's "fetch pending" query and the dashboard's per-site pending count +both need. -Joins POs to line items. `cascadeDelete: true` on the parent relation -(`scripts/pb-bootstrap-schema.py:132`). +### 4.8 `purchase_order_items` + +Joins POs to line items. `cascadeDelete: true` on the parent relation. | Field | Type | Notes | | --- | --- | --- | @@ -209,35 +237,81 @@ Joins POs to line items. `cascadeDelete: true` on the parent relation | `material_code`, `cost_code` | text | | | `description` | text | The free-text the OCR matches against in the DN line | | `qty` | number | Ordered quantity | -| `qty_outstanding` | number | Decreased as received_items roll up — but the canonical fulfilment % is computed from `received_items` directly (see `/dashboard/purchasing` page and the orphan-id note in `src/app/dashboard/purchasing/page.tsx:52-72`) | | `unit`, `rate`, `disc_percent`, `vat_percent`, `amount` | | The amount field is the **computed** total, not stored separately | -### 4.8 `timesheets` +`qty_outstanding` no longer exists as a stored field. Outstanding quantity +is computed at read time from `delivery_note_items` via +`src/lib/po-fulfilment.ts` (`computeOutstanding`) — there is nothing left to +drift out of sync with the actual delivery history. -The collection with **no site** and **no project_code**. Site is resolved -through `employeeRates` at every read site. +### 4.9 `timesheets` (uploaded document) + +New collection. One row per uploaded wage sheet — the file itself, plus +queue/outcome fields shared with `fuel_sheets` and `plant_hours_sheets`. + +| Field | Type | Notes | +| --- | --- | --- | +| `site` | relation → `sites`, **required** | A wage sheet belongs to one site, unlike the fuel/plant sheets | +| `source_file` | file | The uploaded bytes | +| `content_type` | text | MIME of the original file | +| `file_hash` | text | SHA-256 | +| `date_uploaded`, `uploaded_by` | date/text | | +| `processing_status` | select | `pending` / `processing` / `processed` / `failed` / `dead` | +| `processed_at` | date | | +| `attempts`, `last_error` | number/text | | +| `rows_total`, `rows_created`, `rows_failed` | number | Set by `resolveDocumentOutcome()` (`src/lib/document-status.ts`) — never hardcoded to "processed" before rows are actually inserted | +| `row_errors` | json | Up to `MAX_ROW_ERRORS` (200) per-row failure reasons | + +### 4.10 `timesheet_entries` (renamed from `timesheets`) + +The actual time log: one row per (employee, date, hours). This is the +collection that used to be called `timesheets` before the restructure — +watch for stale references to the old name in anything not covered by this +rewrite. | Field | Type | Notes | | --- | --- | --- | | `employee` | relation → `employees` | | -| `batch_reference` | relation → `data_import_batches` | | | `date_logged` | date | | | `hours_worked` | number | Decimal | +| `site` | relation → `sites`, **required** | Stamped from the employee's site at import time — not joined at read time (see §2) | +| `timesheet` | relation → `timesheets`, optional | Empty means captured directly by QR scan, not from an uploaded sheet — this stays optional forever | -### 4.9 `fuel_logs` +Indexes: `idx_ts_entries_site_date` on `(site, date_logged)`; +`idx_ts_entries_emp_date` on `(employee, date_logged)`. + +### 4.11 `fuel_sheets` (uploaded document) + +Same shape as `timesheets` (§4.9) except `site` is optional — a pump sheet +can span multiple sites, resolved per-row when the actual `fuel_logs` rows +are created. + +Indexes: `idx_fuel_sheets_hash` on `file_hash`; `idx_fuel_sheets_status` on +`(processing_status, date_uploaded)`. + +### 4.12 `fuel_logs` | Field | Type | Notes | | --- | --- | --- | -| `project_code` | text | Denormalised, falls back to `vehicle.project_code` when the source row has no site | -| `site` | relation → `sites` | Optional — empty when the source row had no site column | +| `site` | relation → `sites`, **required** | Resolved per-row at import time | | `vehicle` | relation → `vehicles` | Required by the schema but the importer tolerates a missing relation and falls back to QR | -| `batch_reference` | relation → `data_import_batches` | | | `date_logged` | date | ISO datetime; importer anchors at UTC midnight | | `litres` | number | Required | | `odo_reading` | number | Required — used for variance calculation | -| `latest_consumption` | number | Optional. Empty cells preserved as `undefined` rather than 0 (intentional, see comment at `src/lib/pb-types.ts:173-181`) | +| `latest_consumption` | number | Optional. Empty cells preserved as `undefined` rather than 0 (intentional) | +| `fuel_sheet` | relation → `fuel_sheets`, optional | Which uploaded sheet produced this row, if any | -### 4.10 `plant_hours_log` +Indexes: `idx_fuel_logs_site_date` on `(site, date_logged)`; +`idx_fuel_logs_vehicle_date` on `(vehicle, date_logged)`. + +### 4.13 `plant_hours_sheets` (uploaded document) + +Identical shape to `fuel_sheets` (§4.11), same optional-site reasoning. + +Indexes: `idx_plant_hours_sheets_hash` on `file_hash`; +`idx_plant_hours_sheets_status` on `(processing_status, date_uploaded)`. + +### 4.14 `plant_hours_log` The most disciplined collection. Every site-attribution field is required. @@ -247,37 +321,39 @@ The most disciplined collection. Every site-attribution field is required. | `vehicle_qr` | text | The QR text — part of the **composite unique index** below | | `vehicle` | relation → `vehicles` | Required (minSelect=1) | | `site` | relation → `sites` | Required (minSelect=1) | -| `project_code` | text | Required | | `start_hours`, `end_hours`, `hours` | number | | | `hour_meter_budget`, `cost_per_hour`, `value` | number | `value = hours × cost_per_hour` | | `over_budget`, `hours_mismatch` | bool | Computed at import time | | `vehicle_unit_type_at_import` | select | Snapshot of the vehicle's `unit_type` at the moment of import — see the note in [multi-site-assessment §5](./multi-site-assessment.md#5-vehicles-plant-and-employees-moving-between-sites) about why this discipline should extend to `site` | -| `batch_reference` | relation | Optional | +| `plant_hours_sheet` | relation → `plant_hours_sheets`, optional | Which uploaded sheet produced this row, if any | **Unique index: `(vehicle_qr, start_date)`.** Prevents accidental -double-imports of the same week for the same vehicle -(`scripts/pb-bootstrap-schema.py:257`). Surfaced as a friendly +double-imports of the same week for the same vehicle. Surfaced as a friendly `"duplicate: vehicle_qr=X start_date=Y"` error in the import response. +Additional index: `idx_ph_log_vehicle_start` on `(vehicle, start_date)`. -### 4.11 `delivery_notes` +### 4.15 `delivery_notes` | Field | Type | Notes | | --- | --- | --- | +| `processing_status` | select | `pending` → `ocr-complete` → `auto-verified` / `needs-review` / `manual-verified` / `failed` | | `delivery_note_number` | text | Optional; OCR may fill it in | -| `project_code` | text | Denormalised | -| `site` | relation → `sites` | Resolved at upload time | +| `site` | relation → `sites`, **required** | Resolved at upload time | | `purchase_order` | relation → `purchase_orders` | Filled by OCR (`exact → normalised → Levenshtein`) or by manual `assign-po` route | | `date_received` | date | | | `received_by` | text | The uploader's identity — currently always `"web-upload"` | | `slip_image` | file | The photo/PDF, served via `/api/delivery-notes/:id/slip` to n8n | | `raw_ocr_data` | json | Full OCR payload + match metadata | -| `processing_status` | select | `pending` → `ocr-complete` → `auto-verified` / `needs-review` / `manual-verified` / `failed` | +| `is_verified` | bool | Final flip from `auto-verified` or `manual-verified` | | `ocr_confidence` | number | 0..100 integer | | `ocr_completed_at` | date | | | `received_by_name` | text | Signature OCR'd from the slip | -| `is_verified` | bool | Final flip from `auto-verified` or `manual-verified` | +| `file_hash` | text | SHA-256, used for upload-time dedup | -### 4.12 `received_items` +Indexes: `idx_dn_site_date` on `(site, date_received)`; `idx_dn_status` on +`processing_status`. + +### 4.16 `delivery_note_items` (renamed from `received_items`) Joins DNs to PO line items. `cascadeDelete: true` on the parent DN. @@ -288,41 +364,29 @@ Joins DNs to PO line items. `cascadeDelete: true` on the parent DN. | `qty_received` | number | | | `condition_notes` | text | Free-text audit trail (e.g. `"OCR 92% · desc: Cement 50kg · unit: bag"`) | -### 4.13 `data_import_batches` - -The audit-trail spine. Every uploaded file — Excel, CSV, PDF, image — has a -batch row that tracks its lifecycle. - -| Field | Type | Notes | -| --- | --- | --- | -| `batch_type` | select | `timesheets` / `fuel_logs` / `purchase_orders` / `delivery_notes` / `plant_hours` | -| `content_type` | text | MIME of the original file | -| `original_file` | file | The uploaded bytes; mime-restricted per `batch_type` | -| `date_uploaded` | date | | -| `uploaded_by` | text | Currently always `"web-import"` or `"web-upload"` | -| `status` | select | `pending` / `processing` / `processed` / `failed` / `dead` | -| `attempts` | number | Incremented on each retry | -| `last_error` | text | Last failure reason | -| `last_attempt_at` | date | | -| `next_retry_at` | date | | -| `processed_at` | date | When the batch reached `processed` | -| `processed_record_id` | text | The id of the record this batch created (PO id, DN id, etc.) | -| `file_hash` | text | SHA-256 of the original file; the dedup window for PO re-uploads uses this index | - --- ## 5. Indexes and constraints summary | Index | Where | Purpose | | --- | --- | --- | -| `idx_sites_project_code` | `sites(project_code)` UNIQUE | Prevents duplicate site codes; enforced on the POST /api/sites route via the `idx_sites_project_code` name match (`src/app/api/sites/route.ts:64`) | -| `idx_po_number` | `purchase_orders(purchase_order_number)` UNIQUE | Prevents duplicate PO numbers — works alongside the AI extraction dedup by `purchase_order_number` | -| `idx_plant_hours_log_qr_start` | `plant_hours_log(vehicle_qr, start_date)` UNIQUE | Prevents accidental double-imports of the same week for the same vehicle; surfaced as `duplicate:` reason in the importer | -| `idx_batches_file_hash` | `data_import_batches(file_hash, status)` | Used by the PO upload dedup window (`src/app/api/purchase-orders/upload/route.ts:80-84`) | +| `idx_sites_project_code` | `sites(project_code)` UNIQUE | Prevents duplicate site codes; enforced on the POST /api/sites route | +| `idx_po_number` | `purchase_orders(purchase_order_number)` UNIQUE | Prevents duplicate PO numbers — final backstop behind the app-level dedup in `process-batch` | +| `idx_po_site_status` | `purchase_orders(site, processing_status)` | Per-site pending-PO counts and the n8n cron's fetch-pending query | +| `idx_plant_hours_log_qr_start` | `plant_hours_log(vehicle_qr, start_date)` UNIQUE | Prevents accidental double-imports of the same week for the same vehicle | +| `idx_ph_log_vehicle_start` | `plant_hours_log(vehicle, start_date)` | Per-vehicle history queries | +| `idx_fuel_prices_effective_from` | `fuel_prices(effective_from)` UNIQUE | One price per effective date | +| `idx_dn_site_date` | `delivery_notes(site, date_received)` | Per-site DN listing | +| `idx_dn_status` | `delivery_notes(processing_status)` | Pending-verification dashboards | +| `idx_fuel_logs_site_date`, `idx_fuel_logs_vehicle_date` | `fuel_logs` | Per-site / per-vehicle fuel history | +| `idx_ts_entries_site_date`, `idx_ts_entries_emp_date` | `timesheet_entries` | Per-site / per-employee timesheet queries | +| `idx_timesheets_hash`, `idx_timesheets_status` | `timesheets` (document) | Upload dedup + pending-import dashboards | +| `idx_fuel_sheets_hash`, `idx_fuel_sheets_status` | `fuel_sheets` | Same, for fuel sheet uploads | +| `idx_plant_hours_sheets_hash`, `idx_plant_hours_sheets_status` | `plant_hours_sheets` | Same, for plant hours sheet uploads | No collection outside the above has a server-enforced unique constraint. In -particular, `vehicles.qr_code_id` and `vehicles.project_code` are -**not** unique — see [multi-site-assessment §5](./multi-site-assessment.md#5-vehicles-plant-and-employees-moving-between-sites). +particular, `vehicles.qr_code_id` is **not** unique — see +[multi-site-assessment §5](./multi-site-assessment.md#5-vehicles-plant-and-employees-moving-between-sites). --- @@ -335,7 +399,7 @@ collection: authenticated user) - `updateRule`, `deleteRule` → `null` (superuser only) -The collection list is the full set of 13 — there is no collection where +The collection list is the full set of 16 — there is no collection where clients can update or delete records through the API. The `/api/records/[collection]/[id]/verify` route hand-picks two collections (`purchase_orders`, `delivery_notes`) for the `is_verified` @@ -351,16 +415,29 @@ unused) browser-side `pb` client and the n8n file-proxy route ## 7. Schema drift -The `pb-bootstrap-schema.py` script is the canonical schema definition. The -following migrations exist on disk and are not represented in the script: +`pb-bootstrap-schema.py` is the canonical schema definition — a fresh +PocketBase must come up in the exact shape production is in. That script +had drifted ten-plus migrations behind production before the 2026-09-11 +restructure's final task (Task 20) brought it back to truth: it was missing +`fuel_prices` and `income_claims` entirely, still described the deleted +`data_import_batches` and `plant` collections, still carried `project_code` +on eight collections that no longer have it, still had `qty_outstanding` on +`purchase_order_items`, and was missing every index added across the +restructure. -| Migration | Purpose | -| --- | --- | -| `migrate-delivery-notes-ocr.py` | Adds the `processing_status` / `ocr_*` fields and the OCR webhook hooks to existing DNs | -| `migrate-plant-hours-backfill-vehicle.py` | Backfills `plant_hours_log.vehicle` relations from `vehicle_qr` for pre-relation rows | -| `scripts/migrations/*` (other) | One-off patches for specific schema or data issues | +That drift is exactly the failure mode this section exists to name, so the +rule going forward is unchanged from before: **any time the schema changes, +the new field must be added to both `pb-bootstrap-schema.py`** (so a fresh +PB starts in the right shape) **and `src/lib/pb-types.ts`** (so the +TypeScript code agrees), in the same change that adds it to production. +Forgetting either side is a known foot-gun — it is what produced this +section's rewrite in the first place. `scripts/pb-relax-rules.py`'s +collection list is the third thing that must move in lockstep; it drifted +just as far (missing `fuel_prices`, `income_claims` and `plant_hours_log` +entirely, on top of the same renames/deletions). -Any time the schema changes, the new field must be added to **both** -`pb-bootstrap-schema.py` (so a fresh PB starts in the right shape) **and** -`src/lib/pb-types.ts` (so the TypeScript code agrees). Forgetting either -side is a known foot-gun. +One-off data-migration scripts under `scripts/migrations/*` are expected to +exist and are not a drift signal by themselves — they are how the schema +*got* to its current shape. The drift signal is `pb-bootstrap-schema.py` (or +`pb-relax-rules.py`) disagreeing with what a migration already did to +production. diff --git a/scripts/__pycache__/pb-bootstrap-schema.cpython-314.pyc b/scripts/__pycache__/pb-bootstrap-schema.cpython-314.pyc index 4ac1e7330fb75f20adbe1fc43701ce5900a5a9b0..7c243fdef29f8eb2917a13a2290e2671b77c3429 100644 GIT binary patch literal 16582 zcmb_?d2}0Re%RpT;N(q8q9hJ=(4;Mbwr=aNO_88U>!OE;Wv!qggc*{MKmeW@kRqJj z(5V|!UZk8d?8*Hwnnr_3~?6Hk@v&pJ$nu05O4S(x4UMD@$Kl&)!@AbXSKmGl_ znE?mjQ0tdh+8PYL<9C0*>%$YlfR};4BmUpK-hYl^{szAoe{K59XaA>$Vdfd0QJJd@ zZyU6!Hu`B-?ex>3I_alNb@F!A%{$a8-l?+uC%mhfN!V0R#M8#`ZoZ0VBUQX7!gkva zJ2cp{c7{>CsxRWbdjI$=!-(4%W+xM|4{bAu*!Tn(-iX)F@V{~CE)w@a<&o0=K5hHr*v zZ8xLV@!R?si}V_v}Z^7-b#n?#sSg024XOtfc*{L>O(zS8Pq>Y!U z1zkpsR9MiLEa}wR$*UK@XPv_s||EW&< z9i8^vmVEv*{u}z*U)9&XXW|oSuC$A9g3VTrA=j-u+pP5;`p}3*iyNFq44fnjQtBZ$F00^ zj!V&*uKyjX<9~@dhAgop*HArZ`8)i-(lz(Hy5^Qmn)^KqEp6>ve;;ICnQ#4p#n>ML zpOx$W$U+Z)Y_a!G^!WPUE#j+HjIPj!DQ>Lj{Qhebzee=_9~RL6jRl1!tE`BpMx@=G ztNmLGuKt|`?C+X#l>hk;&Qboog{=S7f`WS%6jbClp#a#+_xSJY_VQ=C{C;4P-=9-U z`wNTw;)itRez~8a*7+YSDEyD*)P$U+SoEI|E96)I>!MAqhgzT^QeVS_ZT)`*RZL@~ zsf~%$MHO z@c1Q$nS?rHCv&!iVYXQRm;8S|4AmC?sV$L)p{KQX^YgI&1u!3A7+Ra{Qg;JSyUcX) z+sxn8)=0}xj~R;g&i_@U6)LR%a?laky@B^^h+#a;ZU$yskN|P-BkR3Fq!D1OMz)vZ zgs~q*TA&7W8!`atwZN*sME#uoriMeWIv+%uBZ0$CjVis~Y@_uiNKNdb_}j`W!$?D< z5h2kkat-uS#xehXfZ=}}sn}Cqk%nz&1F+xAe}K9XU#D2G3NG>=R>Fgw_DFyyMx{t; z+@tP|?17pXcZ1BJCf@r98h=s=Q>U?F56JDKgRTvk0hQxAm;cog!hegjAzx6B4`}i; zKl#5trrQ%pe~XI!s;vy$ou7kEtL_n}`~MrxRoxn}Bk*x#Kb+A9gC>y2Pa|%$Ftb#6 z+g_&g?=sy`%%4f6R7I7eX>KSLyCJD(qKd>Ru_AU*|4d9O06ZqDmr{Ns5xp?a}e?0jhav0&ysGcRHo0UGd~3 zEC!@qGJMLJn3|DG>~|b|^5D^~C!g*-;pMn4Zd{D2Q6WB^PRXipEvkZylyR;DDFMc1 z!1zi=%1B%~l}dDS(WFSxh*DaXVo@LnfNSPYs0!l0mceBj(%C5q%t%Toh9$nD&Izdu_z7SJ ztipoW@p4%jn!qDTiK2*-safD>I-SAe35Qsgq98id@*PIXOW5+Fe| z*^sK}(xUdCmg8WVx=`au0452sL^MvdjzmWjv(Y(4a}ygbMd5%wO2RBInvPTB*2Jt% zfyl0vyr2@3<7xFIjHpxeKmwLT22K$N)Q7wmvU{dfHLaZJ?oM9|O-qx}P&_#yM?=wc zdMa}*6ibD&(QHjr1ef(3ou(s%ed^$$!$*!DJ08=+Gz~LYfMN6WvnF`XGgnHn$Ipz} zFleToV|Hmj2FELUAYzyyGX#A{8#87%T>g+=)|G(4i;?!6xA9Ecb%izNhG=Wt_@;dh zL0GT+dnr5yZE+)fr`3=+|IO*8M#BH2eLZ*uW z1M|>M<89;9&4|MYG3^=g4DH?+H{MJ(X2}8U`UlGJq^-mgP2yV_zLcbaB<#CD3Te-? zp1t~K+!ojQ>~vq~v=b-BUgDmX)Txx1_4lTdDmaHO%pKscrPB#;cj&*muPdn}3G`%O zogB~7v8?^oFL%}u`_LKUq|+`jGo4n5TLS+fO2nO!6X179l`MhFS5#8P>o4A@C<;^H zq$HVC!KzeRN)jh}E#ej>ZL=02i3>xmLR^w8r)1(87#SG~(^n@h>-5TaZ$g~us5(WQ z*HYpfajB^r0L}q$WjxWu0r-hcCJyzcN^FvXN3$LNFm2#CvRm^^3izF_lDq2CGPO{8 zjyTaT$sB9~f59joX!6Z79|fux`W71&-3wjwy~U0LKk7LC*5TVH-aN4=KJc}C;M=q6 z+q34|w<6B>edMjV9l90zz{{c%OxBRlXlpH5Xam?clB8;#UffJ-x=B z`0FaaG4hK`t~EBaZg;s(e-x;{ef8GW4+48v1AFtm9|Stq0v$^|#k%@C?Qge#t>aGj z+uaZ9_Ac}m>l^MIfBX2KINlriwaeeQyySRKSZOHKpZ>otCfEtv;Qt4Kj=%rh&G_qX zpSpGG17F*!ukCZj<~m)huDLyPYv$|fo$Nc=eA~C%zt#R59pCQ$R`-KFhd$VIvasjm zvihL;RH6FR`|PQ|Uw6QkpDE9PNWNe7SYI>q{ZNm$&*A#B<6ijk1Bb7#!SjQl6W<#g z_-tIdpM&B-`B`O~<=Uk)>NuGLV*J6i6VtqhNPk zvvGK&qlz*MX$4{9B3No%og=I9Lw~WFUyo61}6_w84VNd=ttD z?#66Xo`h^44-=$`kjrn*`#DLCg*v^Hsj^HSk|;fs4-qHkh{TDNM5hf-DCMZrDMOve zz{!DOf!J@%65E7=thBcu;N(_1M^jW8**XLNls)j6XV%?}w+4>EOfFMk_s{nfovt?q zUK?28Uq4s$`R4~dgH=q-%r^Qt8kygu=jcjl4*WG07Vm&IsnclYo910`4UBVD&_vk@ zZ6(%XjfeO04&Gl3! z$zi7}%OX47F=Zy}rEgGS#FN===3qOM6hqTW7uF?-Y)|D3M$4Pi31}irM^&XOH38iK z$tXob3Y5hVF|oXzPK4-Zw%QB=FyD2NDgc5d2k~ymh&ZwUGcC ziC`Flf~U7#a}a)t3RDff14SN(i3{*g`7u1^nZIQN3lodimrfU3I#(L+w|(xgdCuE# zbUHV-Bo()9&maHPZr|SZxy$JZeePvE!G&aQVkxu69{C7X-N+qX3a_yT@jaEBS%$!P z6yNFYHTE#P7lLj1H@V+na61_M76^)Lw(vQ`fNf0G$>*J@N|(x+)GGotu)99q8r5JtJXm^upka zqhaVy^@T5S<8kq(Kucr6%*;3kjVYt>?HDhc>Lf0@fq25_ht6GmIUF9Aj{zq6I6km> zE1y8bKBEBvJp>5jDcFSk2e9oSARovlY1-fx_(JNFVidx*460|yGG(zGr@F=8uXq4|A=ot~ zOWPXYCBVh_;5bs@^e6r+=CB{qTo5AvqCC{3Im;>g8+bv z01XOg@qzn*j5d{!v4JYXFRU2(MSKk7vOp5N2pmKe8Zh9js;O(zJr)eegRm&eZ>i68 zO3)b9)L2d`^Z`JjWj!2Yb!T=&hiNhn07l({ma=aN5!YbDO?NV;E?6h2E=Q-3hH^a# z5Q)wYlVJHW(j*Aw>$GS^0^k`zhS3D=0TE9OdXn0!Pn5KL9}Ud`{TJfQB;zVnAFzY? z#kdkf%Zs56`DUVO>5Dr$O?BK*PJ-5uA-?{>OW{7D7gE__;-i5H+GvxgLc~p6*d$Yk zACwD8X9jwM;Cg)%@ta^pxIiK7Sa@Kt_gr|G)EgEqn2x-UwrLXaX=o@C?<+FpMu5aM zKzZ9ZENBCe7pIZ}@PZ8)Et+|dOFf<rNv`Wnuul+5dVz+DKIx0 zJ|_(EgS|jqP@0&)e)0^2b?Byn=CE}{PirEs%2M{ymPTjlGzsNUG@CKaqXkwh=>&OU zt5$1O5%0+0`S9>SI6N{eUqxeoZnI&QI-%ejY0nT{*2W+rDWwzRo!Kuv5`x*f%VBIb zXt46K!haUw_=VmL2`t)WEdrF8&-O#z73nRk7#Zi7(=Oa6`87Fm4Wmhn!6TEq{Qr$Pc z5Ja=qtzmv4<=1WA3u%UlO)v%GHDvdS-09MM?kV)Sr%R3(>Yt=`e@DDPnVzA6V@Qfr z;KnGP7HyQi`LwmvML?nhi?vsm_EC!;0`&JFQM#59CnbnQG^+&5gEYe!(=$ql7Z5pP zQ>4Wpw-h!YsnGA}psd2XHnVofb_m2Kj?D%t>blsRRGPtnRSFQ*C^LEnVL}44sA7Yt zS7fxY4IzPlvhaK9DFLAxo4yd}O1Kc-%E=&s!3(_?&xeKHb3KEQk@)E?iJl%ox(``L zO4QhZ(n|XVN+JG9Ii)Cqo<#U#lC07c4|B+>bTlpssia&7#y|pP*Doa47d|&Q7Uo|T z;0hK3MML?$o1U&}wXktt2Q{d1a|htPN+O;H-9e2&f|zWAdzrxQSxh1|+IP4I1Ph^M zFQmKrr5~ib*e$a1Fk-UOwUkk{*}oV-K~w#k&nAM???W*{Z-%w`9Cngm`7%;pCMZ2& z43sS*je=0NN>3LnQ+3SgNlRsJse#l(GTz)7Op$7ks7PxUdKs#a`ZBayjws`KCk(L* z0&!`+dq*Y-*W;-KUQL3X#iS{?&4s)5coLeIp{)G`m#wMni)P#ahT}eY?`s@!96X|A zumR3}^2_gV@kJ=diR1b3NVc|eS2vXK*b6mw&;L4?t>ZeNK!VUS2RES5`jo^@WBR>2 zT-H-+q{80kU~jg{+lQCvK$PkB6t1_ET&MlQg-++s%K8jxapGrQYIWV$)6(S6JlcCY zIcW^fO4rgd{7MCyp2Q`}(FqkYb>m=QPd_epy5*MN#f5T zphFC`pFT+YNpckqxx(k6cTL!#p5ft_pgxs37?tsgj5x7}O`M=8aNr{s&xS8R1`kyP zRJ!Lb_MQ#*(GCsmyAWqzxPO?qXn{){aZw?T8xj;cVmzkGcmX9piH}1F*MSki5raa) zMJrk2f%{pA3%G&RUSY0rqv1z=0$jO+Z^N4%YPNWHqoj!ouVfBFY(i_o0>`bWV9s#! zl8};j8LeSLyUFaS>68e)^JnBnuvzRPDM>%Wz@>uC_RzV_=KSm+WAp!n@&1Hq{0US4 zv8!|5USxO7yNbTXdG>R6z;^l}#{}yZ+Y39+7VCHAGWqMv9fi7+pE+#7)315wo%7!HH#!^>w1?qh4racFS6su!J$*Yo>|&YH!2Ie%VUI=8ZK-R1Dw=iQ(BUADld z+nQ|d^@9uxXmP!7;rYd}+*r}+{}38Y*nm=3YVX3wpJIuU(dBK9a&?a zg7-pjcb;8_FHhq{BBynvDkloTz4=Qk(EWNw+phJq@cl;a_>u&jFnx!cHnG+lebw&u zZHy~eWchW6y~e%1tG3n-(s}8%z~8xMLM>u zwIIp1k8wmGplwbZ!AcH_9w6vcO{QC*BvIhNY*db1y>-l1u< zwT7d_wyZKc*W1y?&gYKik3Q1I%ywmdHxn_rI7112SYy`B`mSm~&1tCj*EgdvMOK;m z^#h3cg`7L@{-T(5OITx1X?$)dyj+E4rtV$=Aw7#5=m{(~?aWOU8rzrKR>n{dVcUAW zaR$NKcU3naftOdAn)Q7ZqCpSfUT)AtTL`x1w=MVS?!aJuy~+v9dZ~3>)6R3XcY|e6 zfN$Yku07ulEZ<5jHLL|gMQ5PMzEET(&629Bnvj4it4!T`11vB2!3viQ2XH<2_)@0W z(zP;rKZ?}9NUg}M5W^C*1G4!vK4eijH*zQ7)VG9Ir2DTR9NxCx901C9Qp(4RY)qr3 ztzj26m2Iocb~Jk3R6eHEH?9S{DfQ=y?4-8d!)`DO{$sQE+Uy$QxgBu7&~jiUeE&rx zWoQ#ih7ng7cOkeZKd_uxV^8BmBKN{_W3i=sB~uupaE48aEwV2a*@#BsuDT6>_Ba(V z_@kxv3YIPx*^I%`ZYqN9tIQ7ciRBKEo>UR8EHoPWmqy?c1mBS@QD>`g`Af)IgmPw9 zN#UhSIB}UypbZQ#ipj;BOZ#-cVAv(B(HtQx zI*A)F9>Ahw_-@`{G9}I2;qc(1Lhy;DOZS``rw8di0xk@}o>&1}I(kn6b_$||Gn2OU zu5DmX2dObcxZ*1{FzO5W(aG2yKxl2A+*(Q$MYzxGL1AI7yLVO6Zr{^U_H2eYPU7@ z+KLx0Vr+U*i%o^#zWhYVI?YN)`J6}OC}2G*94U;9;`mFHD05UWDMVwKUd4T}I9Qgl z6oRI>TlS?CdDb6m0G+|1fNTT~%#fzD?v@?&05q;LP3x!V0cgt~T{7ZDDwkaj7h4YA z+gIR+k%v)@x4rqxEBlKr$M3sJ*L@iAOo~Dmss-5fKouaa0mLBjMqew%w_}xQ`IM{m z*ymjkrK&vhj*omb^SvMXFx?w|*Ynk9=lhCI_KmZzon3UVIh%`t+Qp9BR~N1>*x;uP zw~yR9ve@+1)43OZ>}$uL;p~5Ve=fLWe^B?tLhr|qJ^t;vZ_WMotE)A87d^S-@cW>p zm%mfJ;(ov5*+R|!V$=3JH{QOHKT~+@RH6O!%4lIv@4ZV8nl8NKc<5p}4nYoC)3B(# z^=h%E{to*NTWC4599te;Xzi}VcTX%h7J6^7V9Sf{MK#y| z?l0tf3r!tMt|f8#@OQ2+T`2@le88S8dIF2}Z?$~5V>kS`qvPq7eJeBHJ+QL7&^}NY z8eQ8lcB^{9wQ#fOt6A)Q>r}D1HSc`)D+|4gj#~pC`fC^abBFWpZ=PCm6m}d~3NLr8 z9R8zF;i;ZN&v2n`& zqlcT= bwr$Z{*zx$%!KHmm(SrNQA2UzUdgT8BiZnW% delta 5456 zcmb_eTTmO<8Quj+h>IjZAV7e)V-wrL#uzXLY=UJQgAF}U?ARhvI zk~VIVrg2X%O4Fpur7m&fmey(GCT*wF^fpsZ0v#Tled~1c)M-bl^W>-gXIF?_vCPz+ z>fwy`-~aOc-}j$$_K#QT52v>8Fq`ry`2EFm{am!2qW(@E)_?V>Klgdd_3iuU_pVRf zjkQuj5h_FFr~=thB~w^Op(<3398L<|8KI|W)X=Gy!=!hOs0lTLbsN0@ ztsU)CmXrKCG;E!+EvCZSrQx_=-ILRZoT%GLOC2154oCw%84YA|(2M#2)Io{4Ln_8% zT7C6v^!2EQZ9oI=VGWxL1Z_q~&{1cWQc%)LETzS0lp48}OM8h_!7|ZuJjpY{*+5~tEjT2j1e`h}E`o)y4btyI`6l|Q5 zG(1fhL23;j(*S-(!**7~7Ms-QhS9K1X#}$xevfO|+#0rN*{140H4C0p_g+Q!52rbo z)^|f*t?qp&wt2VCs5^BU(g((D3YhfA>VncRo>x=m)TcoNQXuC3FCfVL zO9iz+1T}z#$l;8_C?M_SFuI^RPIxu(&YU#tND8)~fE`7BG1vt28t2kQY1$Vw-e_ri zDTA$zGrlHO_Oha^a+cElZ+%35NrLi(29$>b`J@`8#QK$(+NdC`HibNemz19#qgc?F`&nex%S*j;9=u2_#jeO;$1Y6d=DtEiz$ z7}}Ia(d+1q_N-K1J7XQ~#3L2nO~z3` zp|{$ynd*(CLr#jyr5qG`ld$kCOb*5jDc)vm>hkEO zi9$bOYH^RHDfW((<{2psDR>M0IMW}1W-%u8ll4iJK-3BKOkD~Dv<)EYwLwT2mNRpv zFw`Gc2LUrbZPRbC;-ZHh2MI|VKhsQco2dnOIwouM2+@Up7xxwJ-?o)+9^}uLA`GgC zJ>9D*&Pk=8GmS8k)^!vR^9#lR>kTC`L^n_^zlr{X!qr9pynWL4mX0pIecx`#rd#oY z>eg}|NTEdQ;r{V{cJRdL2pVS}?H?aFhAj=YSl+Z-@XWHpr7$NNB3yv;2;*X&;Gg3n zvm7Tx#9UvL3$TIEOhmMVqr7L<9pTsz@8$T2X!dde|2)TEVuK-pi?n2mMxMJ6_4Axp zG|amLQ4Zqg+zVq;UeOfd{WJccJHYz<0Z!Dxx{W*Hr>H}8i;h1CDEWQ@HHU=wFLMwJ zIK}~51TWT>PPNdIlb9b32SRQy=Vhlai8_H>5X2mR(911AIjh$#xLN;PIK&IA)EgM} zV}s+aF|i=bht5HRtS98nD&m<<*2NQ2!d8iLWeX^zLihBk}&Txgn&2yVy( zSvmaIhDrQ!jUy)o7C)$|&f@#=!N!wTF*nTdbM7D)6hwo^9f`UF_-5mCc%Z2nztJ>~ zndb9_mJ7lqHWU>gW6`UP239kH*|duW4WFXwYd15QD- zxdj1W`$Iw2>yOO&BN4#khSc-itltv=`trsHPq@a8xm@FTxW&qdOJ)CG~ z*-gXc8e<0!v#eMkp_dxrytsVF0bISSrcly)l7L|(M0j}DExgU?!=E~f@l5N<;v$gB z!$l%M6H!2jM)13>?RlaZxdvQ=6RyMfwcS^W%sl7eU;w?WU*P5<{3LGM^8r18|Fx%~ zQO_SBse8!MOO`%Z#C(Z1nbH;=?yTEWXmN+b0l&vh7-;oG=5b+LM;#FMAqo0DS^hwl z-@(#i5OczuEMB0k=Y(kRxnYvMEhc`5_?#jOM;58#74q+CvOG(c7s;{#>TU8!K=JRx z($a*>_CASU-g~R!Q+On|PNY7cqLSG(O@F4-)4KZwm~FoqOBPbbyhUBoKW3hELa zOC9%g1)msA*8=flOM+PHT?t?KJe$sq(0>DCC_cZezgJ$7=n~7_pX;-7J@jV=D%UK( zSTlBz$hr8$61Qe-Pg;x?L$a9Co3HW7vVxkdMcsDs(U&ITxMtsU|BBg> zFf5m@89S3jxe!qT5%-KJnVnT=NLE;iAa3#0Ra3H&L=DAl30n$cFz#E5t{L~?kM;#B ztoaaHE*Gn|l#-yct5i|45s!7$8x^j0{Cvmxx@5Hl{A;CBlX9u1(n<-UZIvoZ?#7op zPv@d0(kk?j>ioN z!xs7QYM1@!`FPio_nxgPKC@EVyj*u@f@tg_8n2j}mkf76je|*R9#pW&6)Mdo1oD|x zsxVnkN({%F6V3SU{wkJ`WMAg)Rd)V1x^nt4lIg58E$f3oUwa(W-;r>u$0qOSja1{J_~r2)?dII9MSZgN_T9b*^nU;x C=*cAj diff --git a/scripts/__pycache__/pb-relax-rules.cpython-314.pyc b/scripts/__pycache__/pb-relax-rules.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6f56b98b0d7322c5d0337b183328eea108ca496a GIT binary patch literal 4409 zcmbVP>r)%o6~C+9l~$_<0tA?sS-|FDjAR40F*bEnAq+JSbtP~fmNkN1U>z*UyDJlv zNvm`+Ep(cJB<V*O5p~QaWYyCf#K8 zw9Xhkt23fWH;c4x5gDBmS=}m9I`2iIIbt<__y|A9E>NPSg61mdJhwGc=5zcEG zg6NXAW`UM@$5r-X#EfJ?34KPDgezpxooMP+A zrbAxcWmr5awnbc`aIV}|^?|MW@3hrUYz>%7tW=8~rB+;M4y8<9_#3Z9=oS(?OK0bi zc&wCOFkPub$cl~ve-i065nHhPaK**ZP+{Og;}8uaI_@kax5y)jY~JX7R8 z>?7{tI`!sAb5ZfK%kZ!TB6mFE7P}ls>>i+toNed|m@^{PBlbp`iMQg3NS)XhsTWU1 z8pQs{5z!NA^s*7x5R86wsu~iwRZIdU<{pFjbLc0i^p6S17D0`ikL789OyEbWR( zH*0WpN##G%?UI=|+{!VxzPx&^)@Z9J4&c!%~Rx} zB$;`We4JdbYmmb7Em*3dfVHFfgA@`ZS}R)d!cS8oO0w6u;&jj`QAH}>2ccM1wyOb1 zG*mIgCC#uZNJcGa-W)8JvJw%|B+^5L^|qmTx|k@W?~-BWOJthS|5#&niMhgX{@|it zprTaDl%k`k=y%dGZwZ3+%2u2YLei{k#knL0vXI0p+dOyRS3|`P9jW9+2zH1oCLFR0 zgRs7oNrcibqGFk9L!C%KeW<8+8DiLi6i7_1sLl8hf#U+*iWEvHjZjHZDq0S<6i0`> z8A&0cF=-83zpeC@oUw290HprU7+#guYpfGvgOge_t|OJP1HNzoNKCdn$c09PU@ z$1ww0T*77vk}MI{^q8K~FuR(FNzg&QO{my95e^3f#*iUwy}U{e7-o_&eFZZM327a( zdg5hJM}uC~O(touk0}+?`YSr7WQ|tU-ssQxb3G?N?HT^X(ALP$N4BL0w&q8+j{CNbecQ1Q zrOenDylcy|>3PHp_jzHb_Cs@)7as7JK+IzM)Y6dSY+KGv=k|}X-RJkYkuNRwBJ$&1 zcAxV+F|q9E7mnJk>zmgfIXdq4C+CS)ezw38BA9&yQKIj;H)Nvu(abZt?&~PzZb@4CU#jl^x zAo3sWdk~V3E!|@c=wpwMAEVhn4)Z|Vqithu>pdq!#%`K?8`{P>`rcWeWt^t}G)e*S zXPSjCW`a;)(q3;Ysdx{osOF6xlx59JDq26OtNqECrmaB%#vD{$DDV0@=89O_;hPIj z2;;uVsleEXfGI)kh4o?jav+>;sWfm2%62feERUW(b*h&#IMXeQiPWkj#1p#lvM7Cb zfIe2y<(nF2;<1}DHYrld4BFud@$MY~(*tL;wA+weN~Ge_NkMyANhX!}GG>Fmuzw<5 zUAX`+=E(C!g!d2T$Q#BO@%e{Lew=@=2+;Zmy2WX4_qja#Nd)LqgA#hVEU0`fie zDv)GCtA-2+XOp6yG8T$?=0cSHbHsm+>OM!ckJ;XgDaSQs*qp5{!#%bi%UE-ESH}8; zskWMOjBUrddwx&(sPU7@Y}@=38oJ_q zCU6U4-R0b^1U`z`x^lRUzye~M%Hei^GwfGA4r=tNfSfhkUD>89x!Tsdsht~pJ=vNI z&uGdy`Z}LsGSXw(vN8C^g&bq~*~CvKHeS0MzB|6tzB9ISHtRUC&-CRu`^MEb9U1ni z8CBP9&t+RiK03bdnt0s-(#yFr>16ja+)hnreL1>p#ehZ5)Fm2Civuyjr4ohs!#9A95%DdT0wG xdu+FU_tu_!&zyChd&mtxZ9qr+o;D%MoKYSk*T4VShHU3GlBn+0G+w5e{{q@rJ_`T< literal 0 HcmV?d00001 diff --git a/scripts/pb-bootstrap-schema.py b/scripts/pb-bootstrap-schema.py index a12d0e2..ba2135a 100644 --- a/scripts/pb-bootstrap-schema.py +++ b/scripts/pb-bootstrap-schema.py @@ -4,6 +4,25 @@ PocketBase v0.39 JSON shape: type-specific options (values, maxSelect, mimeTypes, collectionId, cascadeDelete, min, max) live at the field top level — NOT nested under "options". + +Reflects the post-ingestion-restructure schema (2026-09): + - `data_import_batches` (the shared queue pool) and the deprecated + `plant` alias for `vehicles` are gone. + - `timesheets`, `fuel_sheets` and `plant_hours_sheets` are the + uploaded-document collections; each carries its own + `processing_status` queue fields directly (no separate batch row). + - `timesheets` (old) was renamed `timesheet_entries` — the actual + per-employee-per-day time log. + - `received_items` was renamed `delivery_note_items`. + - `purchase_orders` absorbed the queue fields `data_import_batches` + used to carry, plus `file_hash` and `date_uploaded`. + - `project_code` was removed from every collection except `sites` + (the one place it is canonical); relations to `sites` carry site + attribution everywhere else. + - `qty_outstanding` was removed from `purchase_order_items` — it is + now computed at read time via src/lib/po-fulfilment.ts. + - `fuel_prices` and `income_claims` were always live collections but + were missing from this bootstrap script; both are included now. """ import json import sys @@ -52,21 +71,36 @@ def field(name, type_, **kw): return f -DEFS = {} - -DEFS["data_import_batches"] = { - "name": "DATA_IMPORT_BATCHES", "type": "base", - "fields": [ - field("batch_type", "select", required=True, - values=["timesheets", "fuel_logs", "purchase_orders", "delivery_notes", "plant_hours"], maxSelect=1), - field("original_file", "file", required=False, +# Shared field set for the three "uploaded document" collections +# (timesheets, fuel_sheets, plant_hours_sheets) — each is a +# resumable/idempotent import queue item in its own right, not a row +# in a shared pool. +def sheet_fields(site_required): + return [ + field("site", "relation", required=site_required, + collectionId="__SITES_ID__", cascadeDelete=False, maxSelect=1), + field("source_file", "file", required=False, maxSelect=1, maxSize=20971520, - mimeTypes=["text/csv", "text/plain", "application/vnd.ms-excel", "application/csv"]), + mimeTypes=["text/csv", "text/plain", "application/vnd.ms-excel", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "application/csv"]), + field("content_type", "text", required=False), + field("file_hash", "text", required=False), field("date_uploaded", "date", required=False), field("uploaded_by", "text", required=False), - ], - "indexes": [], -} + field("processing_status", "select", required=False, + values=["pending", "processing", "processed", "failed", "dead"], maxSelect=1), + field("processed_at", "date", required=False), + field("attempts", "number", required=False), + field("last_error", "text", required=False), + field("rows_total", "number", required=False), + field("rows_created", "number", required=False), + field("rows_failed", "number", required=False), + field("row_errors", "json", required=False), + ] + + +DEFS = {} DEFS["sites"] = { "name": "SITES", "type": "base", @@ -78,6 +112,8 @@ DEFS["sites"] = { field("work_hours", "number", required=False), field("hours_per_week", "number", required=False), field("wages_hours_per_day", "number", required=False), + field("latitude", "number", required=False), + field("longitude", "number", required=False), field("job_start_date", "date", required=False), ], "indexes": [ @@ -88,7 +124,6 @@ DEFS["sites"] = { DEFS["employees"] = { "name": "EMPLOYEES", "type": "base", "fields": [ - field("project_code", "text", required=False), field("site", "relation", required=False, collectionId="__SITES_ID__", cascadeDelete=False, maxSelect=1), field("name", "text", required=False), @@ -106,8 +141,7 @@ DEFS["purchase_orders"] = { "name": "PURCHASE_ORDERS", "type": "base", "fields": [ field("purchase_order_number", "text", required=True, min=1, max=128), - field("project_code", "text", required=False), - field("site", "relation", required=False, + field("site", "relation", required=True, collectionId="__SITES_ID__", cascadeDelete=False, maxSelect=1), field("supplier", "text", required=False), field("required_by_date", "date", required=False), @@ -119,9 +153,18 @@ DEFS["purchase_orders"] = { mimeTypes=["application/pdf", "image/jpeg", "image/png", "image/webp"]), field("raw_ocr_data", "json", required=False), field("is_verified", "bool", required=False), + field("file_hash", "text", required=False), + field("processing_status", "select", required=False, + values=["pending", "processing", "processed", "failed", "dead"], maxSelect=1), + field("attempts", "number", required=False), + field("last_error", "text", required=False), + field("next_retry_at", "date", required=False), + field("last_attempt_at", "date", required=False), + field("date_uploaded", "date", required=False), ], "indexes": [ "CREATE UNIQUE INDEX `idx_po_number` ON `purchase_orders` (`purchase_order_number`)", + "CREATE INDEX `idx_po_site_status` ON `purchase_orders` (`site`, `processing_status`)", ], } @@ -134,7 +177,6 @@ DEFS["purchase_order_items"] = { field("cost_code", "text", required=False), field("description", "text", required=False), field("qty", "number", required=False), - field("qty_outstanding", "number", required=False), field("unit", "text", required=False), field("rate", "number", required=False), field("disc_percent", "number", required=False), @@ -147,7 +189,6 @@ DEFS["purchase_order_items"] = { DEFS["fixed_costs"] = { "name": "FIXED_COSTS", "type": "base", "fields": [ - field("project_code", "text", required=False), field("site", "relation", required=False, collectionId="__SITES_ID__", cascadeDelete=False, maxSelect=1), field("date_logged", "date", required=False), @@ -166,63 +207,104 @@ DEFS["fixed_costs"] = { "indexes": [], } -DEFS["plant"] = { - "name": "PLANT", "type": "base", - "fields": [ - field("project_code", "text", required=False), - field("site", "relation", required=False, - collectionId="__SITES_ID__", cascadeDelete=False, maxSelect=1), - field("description", "text", required=False), - field("standard_work_hours", "text", required=False), - field("cost_per_hour", "number", required=False), - ], - "indexes": [], -} - DEFS["vehicles"] = { "name": "VEHICLES", "type": "base", "fields": [ - field("project_code", "text", required=False), field("site", "relation", required=False, collectionId="__SITES_ID__", cascadeDelete=False, maxSelect=1), field("description", "text", required=False), field("standard_work_hours", "text", required=False), field("fuel_limits", "text", required=False), + field("qr_code_id", "text", required=False), + field("cost_per_hour", "number", required=False), + field("attention_dismissed_at", "date", required=False), field("unit_type", "select", required=False, values=["fuel", "plant"], maxSelect=1, default="fuel"), - field("attention_dismissed_at", "date", required=False), ], "indexes": [], } +DEFS["fuel_prices"] = { + "name": "FUEL_PRICES", "type": "base", + "fields": [ + field("effective_from", "date", required=True), + field("price_per_litre", "number", required=True), + field("notes", "text", required=False), + ], + "indexes": [ + "CREATE UNIQUE INDEX `idx_fuel_prices_effective_from` ON `fuel_prices` (`effective_from`)", + ], +} + DEFS["timesheets"] = { "name": "TIMESHEETS", "type": "base", + "fields": sheet_fields(site_required=True), + "indexes": [ + "CREATE INDEX `idx_timesheets_hash` ON `timesheets` (`file_hash`)", + "CREATE INDEX `idx_timesheets_status` ON `timesheets` (`processing_status`, `date_uploaded`)", + ], +} + +DEFS["timesheet_entries"] = { + "name": "TIMESHEET_ENTRIES", "type": "base", "fields": [ field("employee", "relation", required=False, collectionId="__EMPLOYEES_ID__", cascadeDelete=False, maxSelect=1), - field("batch_reference", "relation", required=False, - collectionId="__DATA_IMPORT_BATCHES_ID__", cascadeDelete=False, maxSelect=1), field("date_logged", "date", required=False), field("hours_worked", "number", required=False), + # Stamped from the employee's site at import time — NOT joined at + # read time. This ends historical drift when a worker moves sites. + field("site", "relation", required=True, + collectionId="__SITES_ID__", cascadeDelete=False, maxSelect=1), + # Optional forever: empty means captured directly by QR scan, not + # from an uploaded sheet. + field("timesheet", "relation", required=False, + collectionId="__TIMESHEETS_ID__", cascadeDelete=False, maxSelect=1), + ], + "indexes": [ + "CREATE INDEX `idx_ts_entries_site_date` ON `timesheet_entries` (`site`, `date_logged`)", + "CREATE INDEX `idx_ts_entries_emp_date` ON `timesheet_entries` (`employee`, `date_logged`)", + ], +} + +DEFS["fuel_sheets"] = { + "name": "FUEL_SHEETS", "type": "base", + # site is optional: a pump sheet can span sites, resolved per-row. + "fields": sheet_fields(site_required=False), + "indexes": [ + "CREATE INDEX `idx_fuel_sheets_hash` ON `fuel_sheets` (`file_hash`)", + "CREATE INDEX `idx_fuel_sheets_status` ON `fuel_sheets` (`processing_status`, `date_uploaded`)", ], - "indexes": [], } DEFS["fuel_logs"] = { "name": "FUEL_LOGS", "type": "base", "fields": [ - field("project_code", "text", required=False), - field("site", "relation", required=False, + field("site", "relation", required=True, collectionId="__SITES_ID__", cascadeDelete=False, maxSelect=1), field("vehicle", "relation", required=False, collectionId="__VEHICLES_ID__", cascadeDelete=False, maxSelect=1), - field("batch_reference", "relation", required=False, - collectionId="__DATA_IMPORT_BATCHES_ID__", cascadeDelete=False, maxSelect=1), field("date_logged", "date", required=False), field("litres", "number", required=False), field("odo_reading", "number", required=False), + field("latest_consumption", "number", required=False), + field("fuel_sheet", "relation", required=False, + collectionId="__FUEL_SHEETS_ID__", cascadeDelete=False, maxSelect=1), + ], + "indexes": [ + "CREATE INDEX `idx_fuel_logs_site_date` ON `fuel_logs` (`site`, `date_logged`)", + "CREATE INDEX `idx_fuel_logs_vehicle_date` ON `fuel_logs` (`vehicle`, `date_logged`)", + ], +} + +DEFS["plant_hours_sheets"] = { + "name": "PLANT_HOURS_SHEETS", "type": "base", + # site is optional, same reasoning as fuel_sheets. + "fields": sheet_fields(site_required=False), + "indexes": [ + "CREATE INDEX `idx_plant_hours_sheets_hash` ON `plant_hours_sheets` (`file_hash`)", + "CREATE INDEX `idx_plant_hours_sheets_status` ON `plant_hours_sheets` (`processing_status`, `date_uploaded`)", ], - "indexes": [], } DEFS["plant_hours_log"] = { @@ -234,7 +316,6 @@ DEFS["plant_hours_log"] = { collectionId="__VEHICLES_ID__", cascadeDelete=False, maxSelect=1, minSelect=1), field("site", "relation", required=True, collectionId="__SITES_ID__", cascadeDelete=False, maxSelect=1, minSelect=1), - field("project_code", "text", required=True), field("start_date", "date", required=True), field("end_date", "date", required=True), field("start_time", "text", required=False), @@ -246,24 +327,46 @@ DEFS["plant_hours_log"] = { field("hour_meter_budget", "number", required=True), field("cost_per_hour", "number", required=True), field("value", "number", required=True), - field("over_budget", "bool", required=True), - field("hours_mismatch", "bool", required=True), + field("over_budget", "bool", required=False), + field("hours_mismatch", "bool", required=False), field("vehicle_unit_type_at_import", "select", required=True, values=["fuel", "plant"], maxSelect=1), - field("batch_reference", "relation", required=False, - collectionId="__DATA_IMPORT_BATCHES_ID__", cascadeDelete=False, maxSelect=1, minSelect=0), + field("plant_hours_sheet", "relation", required=False, + collectionId="__PLANT_HOURS_SHEETS_ID__", cascadeDelete=False, maxSelect=1, minSelect=0), ], "indexes": [ "CREATE UNIQUE INDEX `idx_plant_hours_log_qr_start` ON `plant_hours_log` (`vehicle_qr`, `start_date`)", + "CREATE INDEX `idx_ph_log_vehicle_start` ON `plant_hours_log` (`vehicle`, `start_date`)", ], } +DEFS["income_claims"] = { + "name": "INCOME_CLAIMS", "type": "base", + "fields": [ + field("claim_number", "text", required=True), + field("site", "relation", required=False, + collectionId="__SITES_ID__", cascadeDelete=False, maxSelect=1), + field("period_start", "date", required=False), + field("period_end", "date", required=False), + field("gross_amount", "number", required=False), + field("certified_by", "text", required=False), + field("paid_on", "date", required=False), + field("status", "select", required=False, + values=["draft", "certified", "paid"], maxSelect=1), + field("is_verified", "bool", required=False), + field("notes", "text", required=False), + ], + "indexes": [], +} + DEFS["delivery_notes"] = { "name": "DELIVERY_NOTES", "type": "base", "fields": [ + field("processing_status", "select", required=False, + values=["pending", "ocr-complete", "auto-verified", + "needs-review", "manual-verified", "failed"], maxSelect=1), field("delivery_note_number", "text", required=False), - field("project_code", "text", required=False), - field("site", "relation", required=False, + field("site", "relation", required=True, collectionId="__SITES_ID__", cascadeDelete=False, maxSelect=1), field("purchase_order", "relation", required=False, collectionId="__PURCHASE_ORDERS_ID__", cascadeDelete=False, maxSelect=1), @@ -274,18 +377,19 @@ DEFS["delivery_notes"] = { mimeTypes=["image/jpeg", "image/png", "image/webp", "application/pdf"]), field("raw_ocr_data", "json", required=False), field("is_verified", "bool", required=False), - field("processing_status", "select", required=False, - values=["pending", "ocr-complete", "auto-verified", - "needs-review", "manual-verified", "failed"], maxSelect=1), field("ocr_confidence", "number", required=False), field("ocr_completed_at", "date", required=False), field("received_by_name", "text", required=False), + field("file_hash", "text", required=False), + ], + "indexes": [ + "CREATE INDEX `idx_dn_site_date` ON `delivery_notes` (`site`, `date_received`)", + "CREATE INDEX `idx_dn_status` ON `delivery_notes` (`processing_status`)", ], - "indexes": [], } -DEFS["received_items"] = { - "name": "RECEIVED_ITEMS", "type": "base", +DEFS["delivery_note_items"] = { + "name": "DELIVERY_NOTE_ITEMS", "type": "base", "fields": [ field("delivery_note", "relation", required=False, collectionId="__DELIVERY_NOTES_ID__", cascadeDelete=True, maxSelect=1), @@ -297,20 +401,26 @@ DEFS["received_items"] = { "indexes": [], } +# Dependency order: a collection referencing another via `relation` must be +# created after the collection it points to, so its `collectionId` +# placeholder can be resolved. ORDER = [ - "data_import_batches", "sites", "employees", "purchase_orders", "purchase_order_items", "fixed_costs", - "plant", "vehicles", + "fuel_prices", "timesheets", + "timesheet_entries", + "fuel_sheets", "fuel_logs", + "plant_hours_sheets", "plant_hours_log", + "income_claims", "delivery_notes", - "received_items", + "delivery_note_items", ] ids = {} diff --git a/scripts/pb-relax-rules.py b/scripts/pb-relax-rules.py index 8282fe0..790d342 100644 --- a/scripts/pb-relax-rules.py +++ b/scripts/pb-relax-rules.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Relax API rules on the 12 MEGA INFRASTRUCTURE collections. +"""Relax API rules on the 16 MEGA INFRASTRUCTURE collections. Pattern: list/view/create → any authenticated user @@ -20,18 +20,22 @@ AUTH_RULE = "@request.auth.id != ''" # any logged-in user NONE_RULE = None # superuser only RULES = { - "DATA_IMPORT_BATCHES": (AUTH_RULE, AUTH_RULE, AUTH_RULE, NONE_RULE, NONE_RULE), "SITES": (AUTH_RULE, AUTH_RULE, AUTH_RULE, NONE_RULE, NONE_RULE), "EMPLOYEES": (AUTH_RULE, AUTH_RULE, AUTH_RULE, NONE_RULE, NONE_RULE), "PURCHASE_ORDERS": (AUTH_RULE, AUTH_RULE, AUTH_RULE, NONE_RULE, NONE_RULE), "PURCHASE_ORDER_ITEMS": (AUTH_RULE, AUTH_RULE, AUTH_RULE, NONE_RULE, NONE_RULE), "FIXED_COSTS": (AUTH_RULE, AUTH_RULE, AUTH_RULE, NONE_RULE, NONE_RULE), - "PLANT": (AUTH_RULE, AUTH_RULE, AUTH_RULE, NONE_RULE, NONE_RULE), "VEHICLES": (AUTH_RULE, AUTH_RULE, AUTH_RULE, NONE_RULE, NONE_RULE), + "FUEL_PRICES": (AUTH_RULE, AUTH_RULE, AUTH_RULE, NONE_RULE, NONE_RULE), "TIMESHEETS": (AUTH_RULE, AUTH_RULE, AUTH_RULE, NONE_RULE, NONE_RULE), + "TIMESHEET_ENTRIES": (AUTH_RULE, AUTH_RULE, AUTH_RULE, NONE_RULE, NONE_RULE), + "FUEL_SHEETS": (AUTH_RULE, AUTH_RULE, AUTH_RULE, NONE_RULE, NONE_RULE), "FUEL_LOGS": (AUTH_RULE, AUTH_RULE, AUTH_RULE, NONE_RULE, NONE_RULE), + "PLANT_HOURS_SHEETS": (AUTH_RULE, AUTH_RULE, AUTH_RULE, NONE_RULE, NONE_RULE), + "PLANT_HOURS_LOG": (AUTH_RULE, AUTH_RULE, AUTH_RULE, NONE_RULE, NONE_RULE), + "INCOME_CLAIMS": (AUTH_RULE, AUTH_RULE, AUTH_RULE, NONE_RULE, NONE_RULE), "DELIVERY_NOTES": (AUTH_RULE, AUTH_RULE, AUTH_RULE, NONE_RULE, NONE_RULE), - "RECEIVED_ITEMS": (AUTH_RULE, AUTH_RULE, AUTH_RULE, NONE_RULE, NONE_RULE), + "DELIVERY_NOTE_ITEMS": (AUTH_RULE, AUTH_RULE, AUTH_RULE, NONE_RULE, NONE_RULE), } diff --git a/src/lib/agent/load.ts b/src/lib/agent/load.ts index cd2f2ef..19b1e69 100644 --- a/src/lib/agent/load.ts +++ b/src/lib/agent/load.ts @@ -344,36 +344,47 @@ export interface ImportItem { * PurchaseOrderRecord.processing_status / DeliveryNoteRecord.processing_status * in src/lib/pb-types.ts) — there is no separate batch row for either. */ -export async function loadImportBatches(args: { status?: string }): Promise { +export async function loadImportBatches(args: { + status?: string; + siteId?: string; +}): Promise { const pb = await pbAdmin(); - const statusFilter = args.status ? `processing_status='${args.status}'` : ""; + const clauses: string[] = []; + if (args.status) clauses.push(`processing_status='${args.status}'`); + // fuel_sheets and plant_hours_sheets carry an optional `site` — a sheet + // with no site set can span sites and is resolved per-row, so it is + // deliberately excluded when scoping to one site rather than included + // by default. + const siteClause = args.siteId ? `site='${args.siteId}'` : ""; + const filterWith = (extra: string) => + [...clauses, ...(extra ? [extra] : [])].join(" && ") || undefined; const docFields = "id,processing_status,date_uploaded,uploaded_by,attempts,last_error,processed_at"; const [timesheets, fuelSheets, plantHoursSheets, purchaseOrders, deliveryNotes] = await Promise.all([ pb.collection("timesheets").getFullList({ - ...(statusFilter ? { filter: statusFilter } : {}), + filter: filterWith(siteClause), sort: "-date_uploaded", fields: docFields, }), pb.collection("fuel_sheets").getFullList({ - ...(statusFilter ? { filter: statusFilter } : {}), + filter: filterWith(siteClause), sort: "-date_uploaded", fields: docFields, }), pb.collection("plant_hours_sheets").getFullList({ - ...(statusFilter ? { filter: statusFilter } : {}), + filter: filterWith(siteClause), sort: "-date_uploaded", fields: docFields, }), pb.collection("purchase_orders").getFullList({ - ...(statusFilter ? { filter: statusFilter } : {}), + filter: filterWith(siteClause), sort: "-date_uploaded", fields: "id,processing_status,date_uploaded,ordered_by,attempts,last_error", }), pb.collection("delivery_notes").getFullList({ - ...(statusFilter ? { filter: statusFilter } : {}), + filter: filterWith(siteClause), sort: "-date_received", fields: "id,processing_status,date_received,received_by,received_by_name", }), diff --git a/src/lib/agent/tools.ts b/src/lib/agent/tools.ts index 2333599..d1ae96f 100644 --- a/src/lib/agent/tools.ts +++ b/src/lib/agent/tools.ts @@ -871,6 +871,7 @@ export const tools: AgentTool[] = [ description: "Document import batches — timesheets, fuel logs, purchase orders, delivery notes and plant hours — with their processing status, attempt count and any error. Use for 'what uploads failed', 'did my timesheet import work', 'what is stuck'.", schema: { + site: z.string().optional().describe("Site name, project code, or id"), status: z .enum(["pending", "processing", "processed", "failed", "dead"]) .optional() @@ -878,12 +879,23 @@ export const tools: AgentTool[] = [ limit: z.number().optional().describe("Defaults to 25"), }, handler: async (args) => { + let siteId: string | undefined; + let scope = "all sites"; + if (args.site) { + const found = await requireSite(String(args.site)); + if (!found.ok) return { error: found.error }; + siteId = found.site.id; + scope = found.site.name; + } + const batches = await loadImportBatches({ status: args.status as string | undefined, + siteId, }); const limit = typeof args.limit === "number" ? args.limit : 25; return { + scope, total: batches.length, showing: Math.min(limit, batches.length), failed: batches.filter((b) => b.status === "failed" || b.status === "dead").length,