fix(scripts): backfill uploaded_at from real sources, add a dry run
The script read dn_batch_notes.date, which does not exist on that collection — the backfill would have silently set nothing while reporting success. My plan specified the wrong field. Now sources from delivery_notes.date_received via the batch link (11 of 34), falling back to the scanner-generated filename date (17), leaving 6 empty rather than inventing one. Dry-run by default, and it reports the source per batch so the numbers can be checked before writing. Applied: 17 existing fields preserved, uploaded_at added, 28 of 34 batches backfilled. -uploaded_at now sorts newest first. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
c32b41506e
commit
b4b61083b0
1 changed files with 79 additions and 33 deletions
|
|
@ -8,16 +8,40 @@ sort is a no-op and the tick's "oldest first" means nothing. The rest of
|
|||
this codebase already works around the same quirk (delivery_notes sorts by
|
||||
date_received), so an explicit field is the consistent fix.
|
||||
|
||||
Backfill uses the batch's earliest dn_batch_notes date where one exists.
|
||||
Batches with nothing to go on are LEFT EMPTY rather than given an invented
|
||||
timestamp; they sort last.
|
||||
Backfill, in priority order:
|
||||
|
||||
Idempotent.
|
||||
1. The earliest date_received of the delivery_notes linked to the batch.
|
||||
That is the document's own date and the most trustworthy source.
|
||||
2. A date parsed from the stored filename. These are scanner-generated
|
||||
("cam_scanner_2026_09_10_15", "20260910_144808"), so they are
|
||||
unambiguous — but they are the SCAN date, not the upload date. Good
|
||||
enough to order a board by; not to be treated as provenance.
|
||||
3. Nothing. Left empty rather than invented.
|
||||
|
||||
Batches uploaded after this ships get uploaded_at set at upload time, so
|
||||
this only ever runs against the historical set.
|
||||
|
||||
Idempotent. Dry-run by default; pass --apply to write.
|
||||
"""
|
||||
import re
|
||||
import sys
|
||||
|
||||
from pb_env import PB, http, superuser_token
|
||||
|
||||
# A scanner filename date: 20260910, 2026_09_10, 2026-09-10.
|
||||
FILENAME_DATE = re.compile(r"(20\d{2})[_-]?(\d{2})[_-]?(\d{2})")
|
||||
|
||||
|
||||
def date_from_filename(filename: str) -> str | None:
|
||||
m = FILENAME_DATE.search(filename or "")
|
||||
if not m:
|
||||
return None
|
||||
year, month, day = (int(g) for g in m.groups())
|
||||
# Reject a match that is really an invoice or order number.
|
||||
if not (2024 <= year <= 2027 and 1 <= month <= 12 and 1 <= day <= 31):
|
||||
return None
|
||||
return f"{year:04d}-{month:02d}-{day:02d} 00:00:00.000Z"
|
||||
|
||||
|
||||
def get_all(token, coll, fields=None, filt=None):
|
||||
out, page = [], 1
|
||||
|
|
@ -36,16 +60,21 @@ def get_all(token, coll, fields=None, filt=None):
|
|||
page += 1
|
||||
|
||||
|
||||
def main():
|
||||
def main(apply: bool):
|
||||
token = superuser_token()
|
||||
print(f"\n→ dn_batches on {PB}")
|
||||
print(f"\n→ dn_batches on {PB} ({'APPLY' if apply else 'DRY RUN'})")
|
||||
|
||||
code, coll = http("GET", "/api/collections/dn_batches", token=token)
|
||||
if code != 200:
|
||||
sys.exit(f"GET collection failed: {code} {coll}")
|
||||
# Read the live field list and APPEND. PocketBase replaces the whole
|
||||
# array on PATCH, so building it from anything but a fresh read risks
|
||||
# dropping columns.
|
||||
fields = list(coll.get("fields", []))
|
||||
if any(f.get("name") == "uploaded_at" for f in fields):
|
||||
print(" • uploaded_at already exists")
|
||||
elif not apply:
|
||||
print(f" would add uploaded_at (preserving {len(fields)} existing fields)")
|
||||
else:
|
||||
code, resp = http(
|
||||
"PATCH",
|
||||
|
|
@ -55,41 +84,58 @@ def main():
|
|||
)
|
||||
if code != 200:
|
||||
sys.exit(f"PATCH failed: {code} {resp}")
|
||||
print(" ✓ added uploaded_at")
|
||||
print(f" ✓ added uploaded_at (kept {len(fields)} existing fields)")
|
||||
|
||||
notes = get_all(token, "dn_batch_notes", "batch,date")
|
||||
earliest = {}
|
||||
for n in notes:
|
||||
d = (n.get("date") or "").strip()
|
||||
b = n.get("batch")
|
||||
if not d or not b:
|
||||
# Source 1: the earliest date_received among the batch's delivery notes.
|
||||
earliest: dict[str, str] = {}
|
||||
for d in get_all(token, "delivery_notes", "batch,date_received"):
|
||||
batch = d.get("batch")
|
||||
received = (d.get("date_received") or "").strip()
|
||||
if not batch or not received:
|
||||
continue
|
||||
if b not in earliest or d < earliest[b]:
|
||||
earliest[b] = d
|
||||
if batch not in earliest or received < earliest[batch]:
|
||||
earliest[batch] = received
|
||||
|
||||
batches = get_all(token, "dn_batches", "id,uploaded_at,file")
|
||||
counts = {"delivery_note": 0, "filename": 0, "left_empty": 0, "already_set": 0}
|
||||
|
||||
batches = get_all(token, "dn_batches", "id,uploaded_at")
|
||||
filled = skipped = blank = 0
|
||||
for b in batches:
|
||||
if (b.get("uploaded_at") or "").strip():
|
||||
skipped += 1
|
||||
counts["already_set"] += 1
|
||||
continue
|
||||
d = earliest.get(b["id"])
|
||||
if not d:
|
||||
blank += 1
|
||||
continue
|
||||
code, resp = http(
|
||||
"PATCH",
|
||||
f"/api/collections/dn_batches/records/{b['id']}",
|
||||
{"uploaded_at": d},
|
||||
token,
|
||||
)
|
||||
if code != 200:
|
||||
sys.exit(f"PATCH {b['id']} failed: {code} {resp}")
|
||||
filled += 1
|
||||
|
||||
print(f" backfilled {filled}, already set {skipped}, left empty {blank}")
|
||||
value = earliest.get(b["id"])
|
||||
source = "delivery_note"
|
||||
if not value:
|
||||
value = date_from_filename(b.get("file") or "")
|
||||
source = "filename"
|
||||
if not value:
|
||||
counts["left_empty"] += 1
|
||||
print(f" {'left empty':14s} {'-':24s} {(b.get('file') or '')[:40]}")
|
||||
continue
|
||||
|
||||
counts[source] += 1
|
||||
print(f" {source:14s} {value[:10]:24s} {(b.get('file') or '')[:40]}")
|
||||
if apply:
|
||||
code, resp = http(
|
||||
"PATCH",
|
||||
f"/api/collections/dn_batches/records/{b['id']}",
|
||||
{"uploaded_at": value},
|
||||
token,
|
||||
)
|
||||
if code != 200:
|
||||
sys.exit(f"PATCH {b['id']} failed: {code} {resp}")
|
||||
|
||||
print(
|
||||
f"\n from delivery notes: {counts['delivery_note']}"
|
||||
f" from filename: {counts['filename']}"
|
||||
f" left empty: {counts['left_empty']}"
|
||||
f" already set: {counts['already_set']}"
|
||||
)
|
||||
if not apply:
|
||||
print("\n DRY RUN — nothing written. Re-run with --apply.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
main("--apply" in sys.argv)
|
||||
print("\nDone.")
|
||||
|
|
|
|||
Loading…
Reference in a new issue