From 252dd7eb6a9dda754eb81202da7eb982e7cb52e3 Mon Sep 17 00:00:00 2001 From: DEls Date: Sun, 20 Sep 2026 23:49:58 +0200 Subject: [PATCH] fix(dn): let staged ingestion run against the real PocketBase schema Three faults stopped the staged pipeline from ingesting anything once it met a live PocketBase rather than a test double. PocketBase treats 0 as blank for a required number field, and two staged fields legitimately hold zero: dn_batch_sources.ordinal is 0 for the first source in a batch, and dn_batch_pages.region is 0 for a whole-page read. Both creates were rejected with validation_required, so the first file of every upload failed and no page ever persisted. The migration now declares them optional, and pb-relax-dn-zero-fields.py relaxes instances that already have them required. The staged runner verifies its batch lease and commits the protected write in one transaction, which needs PocketBase's batch API. That is off by default, so the runner failed closed with dn_pipeline_transactions_required and no deployment note explained why. pb-enable-batch-api.py turns it on and raises the 3s transaction timeout, which is tight for a commit carrying page images. compare-dn-pipelines used top-level await while resolving as CJS, so the command in the rollout runbook always failed to transform. Renamed .mts. The tests missed all of this because they mock PocketBase, and the mock accepts what the real server rejects. Co-Authored-By: Claude Opus 5 --- docs/operations.md | 4 +- ...-pipelines.ts => compare-dn-pipelines.mts} | 2 + scripts/pb-add-dn-staged-ingestion.py | 8 +- scripts/pb-enable-batch-api.py | 73 +++++++++++++++++ scripts/pb-relax-dn-zero-fields.py | 78 +++++++++++++++++++ 5 files changed, 161 insertions(+), 4 deletions(-) rename scripts/{compare-dn-pipelines.ts => compare-dn-pipelines.mts} (96%) create mode 100644 scripts/pb-enable-batch-api.py create mode 100644 scripts/pb-relax-dn-zero-fields.py diff --git a/docs/operations.md b/docs/operations.md index 68c3e9e..ebe91c9 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -318,8 +318,8 @@ network/vision problem; fix that before increasing `DN_TICK_BATCH_SIZE`. 2. Run the read-only comparison: ```bash - npx tsx --env-file=.env.local scripts/compare-dn-pipelines.ts - npx tsx --env-file=.env.local scripts/compare-dn-pipelines.ts --all + npx tsx --env-file=.env.local scripts/compare-dn-pipelines.mts + npx tsx --env-file=.env.local scripts/compare-dn-pipelines.mts --all ``` It reports page coverage, candidate count, PO/item links, unit diff --git a/scripts/compare-dn-pipelines.ts b/scripts/compare-dn-pipelines.mts similarity index 96% rename from scripts/compare-dn-pipelines.ts rename to scripts/compare-dn-pipelines.mts index 7626a30..28d071e 100644 --- a/scripts/compare-dn-pipelines.ts +++ b/scripts/compare-dn-pipelines.mts @@ -1,3 +1,5 @@ +// .mts, not .ts: this script uses top-level await, and the package has no +// "type": "module", so tsx would otherwise transform it as CJS and fail. import { pbAdmin } from "../src/lib/pb"; import { classifyUnit, poMatchCounts, verifyPurchaseOrderIds } from "../src/lib/dn-pipeline/compare"; diff --git a/scripts/pb-add-dn-staged-ingestion.py b/scripts/pb-add-dn-staged-ingestion.py index c656a76..c05ff79 100644 --- a/scripts/pb-add-dn-staged-ingestion.py +++ b/scripts/pb-add-dn-staged-ingestion.py @@ -318,7 +318,9 @@ def main(): relation("batch", batch_id, required=True), field("file", "file", required=True, maxSelect=1, maxSize=20971520), field("file_hash", required=True), - field("ordinal", "number", required=True), + # Not required: PocketBase treats 0 as blank for a required + # number, and the first source in a batch is legitimately 0. + field("ordinal", "number"), field("mime", required=True), field("original_name", required=True), field("capture_session_id"), field("delivery_session_id"), field("captured_at", "date"), @@ -337,7 +339,9 @@ def main(): # Assigned by the staged runner after normalisation. Optional so # legacy rows can be backfilled only when their identity is clear. field("global_page", "number"), - field("region", "number", required=True), + # Not required for the same reason as `ordinal`: a whole-page + # read carries region 0, which a required number would reject. + field("region", "number"), field("image", "file", required=True, maxSelect=1, maxSize=20971520), field("image_hash", required=True), field("mime", required=True), field("width", "number"), diff --git a/scripts/pb-enable-batch-api.py b/scripts/pb-enable-batch-api.py new file mode 100644 index 0000000..7f3efba --- /dev/null +++ b/scripts/pb-enable-batch-api.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +"""Enable PocketBase's transactional batch API. + +The staged delivery-note pipeline verifies its batch lease and commits the +protected business write inside one transaction, which requires this endpoint. +With it disabled the runner fails closed with +``dn_pipeline_transactions_required``. + +The default 3s transaction timeout is raised because a staged batch commits +page images alongside its metadata. + +Dry run unless ``--apply`` is supplied, and ``--apply`` additionally requires +``--confirm-host`` matching the configured PocketBase host exactly. +""" +import sys + +from pb_env import PB, http, superuser_token + +WANT_TIMEOUT = 30 + +APPLY = "--apply" in sys.argv + + +def confirmation_host(): + for index, argument in enumerate(sys.argv): + if argument.startswith("--confirm-host="): + return argument.split("=", 1)[1] + if argument == "--confirm-host" and index + 1 < len(sys.argv): + return sys.argv[index + 1] + return None + + +def main(): + if APPLY and confirmation_host() != PB: + sys.exit( + f"REFUSING --apply: pass --confirm-host with the exact configured " + f"host ({PB})." + ) + + token = superuser_token() + print(f"auth ok on {PB}") + print(f"target: {PB} mode: {'APPLY' if APPLY else 'DRY RUN'}") + + code, settings = http("GET", "/api/settings", token=token) + if code != 200: + sys.exit(f"could not read settings ({code})") + + batch = dict(settings.get("batch") or {}) + print(f" current: enabled={batch.get('enabled')} timeout={batch.get('timeout')} maxRequests={batch.get('maxRequests')}") + + desired = dict(batch) + desired["enabled"] = True + if int(batch.get("timeout") or 0) < WANT_TIMEOUT: + desired["timeout"] = WANT_TIMEOUT + + if desired == batch: + print(" already configured; nothing to do") + return + + if not APPLY: + print(f" would set: enabled={desired['enabled']} timeout={desired['timeout']} maxRequests={desired.get('maxRequests')}") + print("\nDRY RUN. No settings written.") + return + + code, response = http("PATCH", "/api/settings", {"batch": desired}, token=token) + if code != 200: + sys.exit(f" PATCH failed {code}: {response}") + applied = (response or {}).get("batch", desired) + print(f" applied: enabled={applied.get('enabled')} timeout={applied.get('timeout')} maxRequests={applied.get('maxRequests')}") + print("\nAPPLIED.") + + +main() diff --git a/scripts/pb-relax-dn-zero-fields.py b/scripts/pb-relax-dn-zero-fields.py new file mode 100644 index 0000000..1fbd279 --- /dev/null +++ b/scripts/pb-relax-dn-zero-fields.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +"""Relax the staged-ingestion number fields that legitimately hold zero. + +PocketBase treats 0 as blank for a ``required`` number field, so +``dn_batch_sources.ordinal`` (0 for the first source in a batch) and +``dn_batch_pages.region`` (0 for a whole-page read) were rejected on every +create. This flips those two fields to optional and touches nothing else. + +Dry run unless ``--apply`` is supplied, and ``--apply`` additionally requires +``--confirm-host`` matching the configured PocketBase host exactly. +""" +import sys + +from pb_env import PB, http, superuser_token + +TARGETS = (("dn_batch_sources", "ordinal"), ("dn_batch_pages", "region")) + +APPLY = "--apply" in sys.argv + + +def confirmation_host(): + for index, argument in enumerate(sys.argv): + if argument.startswith("--confirm-host="): + return argument.split("=", 1)[1] + if argument == "--confirm-host" and index + 1 < len(sys.argv): + return sys.argv[index + 1] + return None + + +def main(): + if APPLY and confirmation_host() != PB: + sys.exit( + f"REFUSING --apply: pass --confirm-host with the exact configured " + f"host ({PB})." + ) + + token = superuser_token() + print(f"auth ok on {PB}") + print(f"target: {PB} mode: {'APPLY' if APPLY else 'DRY RUN'}") + + changed = 0 + for collection_name, field_name in TARGETS: + code, collection = http("GET", f"/api/collections/{collection_name}", token=token) + if code != 200: + sys.exit(f"collection {collection_name} not found ({code})") + + fields = collection.get("fields") or collection.get("schema") or [] + target = next((f for f in fields if f.get("name") == field_name), None) + if target is None: + sys.exit(f"{collection_name}.{field_name} not found") + + if not target.get("required"): + print(f" {collection_name}.{field_name}: already optional") + continue + + changed += 1 + if not APPLY: + print(f" {collection_name}.{field_name}: would set required false") + continue + + target["required"] = False + code, response = http( + "PATCH", + f"/api/collections/{collection['id']}", + {"fields": fields}, + token=token, + ) + if code != 200: + sys.exit(f" {collection_name}.{field_name}: PATCH failed {code}: {response}") + print(f" {collection_name}.{field_name}: required -> false") + + if not APPLY: + print(f"\nDRY RUN. {changed} field(s) would change. No schema changes written.") + else: + print(f"\nAPPLIED. {changed} field(s) changed.") + + +main()