feat: add delivery note posting and pipeline comparison

This commit is contained in:
DEls 2026-09-19 18:45:30 +02:00
parent 45b3c66480
commit 5d094183e6
4 changed files with 407 additions and 1 deletions

View file

@ -257,9 +257,93 @@ The two routes that should always return 200 in production:
should print `200`. If it doesn't, the container isn't ready (start
period is 40s).
## 8. Staged delivery-note ingestion
The staged pipeline is deliberately off until a representative site has been
observed. It is additive: original source files and the legacy runner remain
available, and financial rows are created only by the posting action.
### 8.1 Migration
Preview the schema first. This command never writes and does not require admin
credentials:
```bash
set -a; source .env.local; set +a
python3 scripts/pb-add-dn-staged-ingestion.py --dry-run
```
Apply only after reviewing the preview and confirming the exact PocketBase host:
```bash
set -a; source .env.local; set +a
python3 scripts/pb-add-dn-staged-ingestion.py --apply \
--confirm-host "$NEXT_PUBLIC_POCKETBASE_URL"
```
The script adds the staged collections, fields, and indexes. Do not remove
collections or edit financial rows directly.
### 8.2 Required staged environment
The server needs the existing PocketBase admin values plus:
| Variable | Required value | Purpose |
| --- | --- | --- |
| `NEXT_PUBLIC_POCKETBASE_URL` | exact browser/server PB host | File reads and migration host guard. |
| `POCKETBASE_ADMIN_EMAIL` + `POCKETBASE_ADMIN_PASSWORD` or `POCKETBASE_ADMIN_TOKEN` | valid superuser credentials | Server reads and fenced writes. |
| `N8N_INGEST_SECRET` | high-entropy shared secret | Authenticates the tick callback. |
| `MINIMAX_API_KEY` | vision-capable API key | Reads decoded page images. |
| `MINIMAX_BASE_URL` | usually `https://api.minimax.io/v1` | Vision endpoint. |
| `MINIMAX_MODEL` | `MiniMax-M3` or approved vision model | Extraction model. |
| `DN_STAGED_INGESTION_ENABLED` | exactly `false` until rollout, then exactly `true` | Selects staged vs legacy tick runner. |
| `DN_PAGE_PROXY_SECRET` | high-entropy secret (or documented n8n fallback) | Signs review page URLs. |
Keep these values server-only except for `NEXT_PUBLIC_POCKETBASE_URL`.
### 8.3 Tick and lock recovery
`POST /api/dn-batches/tick` is the n8n scheduler entry point and requires
`x-ingest-secret`. The staged runner acquires a PocketBase lease per batch and
fences business mutations in the same transaction. A worker that loses its
lease must not retry writes. Expired `dn_pipeline_locks` rows are reclaimed by
the next tick; do not delete a live lock by hand. If a tick is interrupted,
inspect the batch stage, `last_error`, page statuses, and candidate stages, then
let the next tick resume idempotently. Repeated expiry indicates a timeout or
network/vision problem; fix that before increasing `DN_TICK_BATCH_SIZE`.
### 8.4 Shadow rollout and observation
1. Apply the migration and leave `DN_STAGED_INGESTION_ENABLED=false`.
2. Run the read-only comparison:
```bash
npx tsx --env-file=.env.local scripts/compare-dn-pipelines.ts <batch-id>
npx tsx --env-file=.env.local scripts/compare-dn-pipelines.ts --all
```
It reports page coverage, candidate count, PO/item links, unit
compatibility, quantities, computed value, and `writes: false`; it never
promotes or changes PocketBase.
3. Observe one site's page-read failures, open pages, needs-review rate,
exact-PO and item-link rates, duplicates, lock reclaims, promotion failures,
and incorrect automatic financial matches. The last metric must remain zero.
4. Enable staged processing for that site by setting the flag to `true` and
restarting the server. Expand site by site only after the metrics remain
acceptable.
### 8.5 Rollback
Set `DN_STAGED_INGESTION_ENABLED=false` and restart to return to the legacy
runner. Do not delete staged rows: they are the audit trail and can be retried
or compared later. Do not edit `delivery_notes`, `delivery_note_items`, or
supplier ledger rows directly. Use the correction/audit workflow and investigate
the candidate, page, lock, and promotion event first. Posted candidates are
immutable unless an explicit correction process is approved.
---
## 8. What this document does NOT cover
## 9. What this document does NOT cover
- The application logic — see [`architecture.md`](./architecture.md).
- The collections — see [`data-model.md`](./data-model.md).

View file

@ -0,0 +1,62 @@
import { pbAdmin } from "../src/lib/pb";
type Row = Record<string, any>;
const args = process.argv.slice(2);
const batchId = args.find((arg) => !arg.startsWith("-"));
const all = args.includes("--all");
if (!batchId && !all) {
console.error("Usage: npx tsx scripts/compare-dn-pipelines.ts <batch-id> | --all");
process.exit(2);
}
const pb = await pbAdmin();
const batches = (all ? await pb.collection("dn_batches").getFullList({ sort: "created" }) : [await pb.collection("dn_batches").getOne(batchId!)]).map((row) => row as Row);
const reports: Row[] = [];
for (const batch of batches) {
const [pages, notes] = await Promise.all([
pb.collection("dn_batch_pages").getFullList({ filter: `batch = ${JSON.stringify(batch.id)}`, fields: "id,read_status" }).catch(() => []),
pb.collection("dn_batch_notes").getFullList({ filter: `batch = ${JSON.stringify(batch.id)}`, sort: "note_index", fields: "id,stage,status,pages,purchase_order,items,checks,doc_number,delivery_note_number" }).catch(() => []),
]);
const poIds = [...new Set((notes as Row[]).map((note) => typeof note.purchase_order === "string" ? note.purchase_order : "").filter(Boolean))];
const poItems = (await Promise.all(poIds.map((po) => pb.collection("purchase_order_items").getFullList({ filter: `purchase_order = ${JSON.stringify(po)}`, fields: "id,purchase_order,unit,qty,rate,description" }).catch(() => [])))).flat() as Row[];
const itemById = new Map(poItems.map((item) => [item.id, item]));
let matchedItems = 0;
let itemCount = 0;
let unitMatches = 0;
let quantity = 0;
let computedValue = 0;
for (const note of notes as Row[]) {
for (const line of Array.isArray(note.items) ? note.items : []) {
if (!line || typeof line !== "object") continue;
itemCount++;
const linked = typeof line.purchase_order_item === "string" ? line.purchase_order_item : "";
const poItem = itemById.get(linked);
const qty = typeof line.qty === "number" ? line.qty : Number(line.qty ?? 0);
quantity += Number.isFinite(qty) ? qty : 0;
if (poItem) {
matchedItems++;
if (!line.unit || !poItem.unit || String(line.unit).trim().toLowerCase() === String(poItem.unit).trim().toLowerCase()) unitMatches++;
const rate = Number(poItem.rate);
if (Number.isFinite(rate) && Number.isFinite(qty)) computedValue += qty * rate;
}
}
}
const coveredPages = new Set((notes as Row[]).flatMap((note) => Array.isArray(note.pages) ? note.pages.filter((page: unknown) => Number.isInteger(page)) : []));
reports.push({
batchId: batch.id,
site: batch.site ?? null,
pageCoverage: { total: pages.length, assigned: coveredPages.size, open: Math.max(0, pages.length - coveredPages.size), read: (pages as Row[]).filter((page) => page.read_status === "read").length },
candidateCount: notes.length,
stages: Object.fromEntries([...new Set((notes as Row[]).map((note) => String(note.stage ?? note.status ?? "unknown")))].map((stage) => [stage, (notes as Row[]).filter((note) => String(note.stage ?? note.status ?? "unknown") === stage).length])),
poMatch: { candidatesWithPo: (notes as Row[]).filter((note) => typeof note.purchase_order === "string" && note.purchase_order).length, total: notes.length },
itemMatch: { matched: matchedItems, total: itemCount },
units: { compatible: unitMatches, matchedItems },
quantity,
computedValue: Math.round(computedValue * 100) / 100,
writes: false,
});
}
console.log(JSON.stringify({ generatedAt: new Date().toISOString(), reports }, null, 2));

View file

@ -0,0 +1,92 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
const BATCH = "batch0000000001";
const NOTE = "note00000000001";
const SITE = "site00000000001";
const SUPPLIER = "supp00000000001";
const PO = "po0000000000001";
const ITEM = "poi000000000001";
const DELIVERY = "dn0000000000001";
const state = vi.hoisted(() => ({
batch: {} as Record<string, unknown>,
note: {} as Record<string, unknown>,
delivery: {} as Record<string, unknown> | null,
events: [] as Record<string, unknown>[],
poItems: [] as Record<string, unknown>[],
pages: [] as Record<string, unknown>[],
}));
vi.mock("@/lib/pb", () => ({ pbAdmin: vi.fn(async () => ({
authStore: { token: "test" },
files: { getURL: (_record: unknown, filename: string) => `https://pb.test/${filename}` },
collection(name: string) {
if (name === "dn_batches") return { getOne: vi.fn(async () => state.batch) };
if (name === "dn_batch_notes") return { getOne: vi.fn(async () => state.note) };
if (name === "suppliers") return { getOne: vi.fn(async () => ({ id: SUPPLIER, name: "Supplier" })) };
if (name === "purchase_orders") return { getOne: vi.fn(async () => ({ id: PO, site: SITE, supplier_ref: SUPPLIER })) };
if (name === "purchase_order_items") return { getFullList: vi.fn(async () => state.poItems) };
if (name === "dn_batch_pages") return { getFullList: vi.fn(async () => state.pages) };
if (name === "delivery_notes") return { getOne: vi.fn(async () => state.delivery) };
if (name === "delivery_note_items") return { getFullList: vi.fn(async () => [{ id: "item-row" }]) };
if (name === "dn_review_events") return {
getFullList: vi.fn(async () => state.events),
create: vi.fn(async (event: Record<string, unknown>) => { state.events.push(event); return { id: "event" }; }),
};
throw new Error(`unexpected collection ${name}`);
},
})) }));
const promoteNote = vi.hoisted(() => vi.fn(async () => ({ deliveryNoteId: DELIVERY, itemsCreated: 1 })));
vi.mock("@/lib/dn-pipeline/promote", () => ({ promoteNote }));
function request() { return new Request("http://localhost", { method: "POST" }); }
function context() { return { params: Promise.resolve({ id: BATCH, noteId: NOTE }) }; }
beforeEach(() => {
state.batch = { id: BATCH, site: SITE, page_count: 1, assembly_revision: 3 };
state.note = {
id: NOTE, batch: BATCH, status: "candidate", stage: "ready_to_post", assembly_revision: 3,
pages: [1], doc_number: "DN-1", delivery_note_number: "DN-1", supplier: SUPPLIER, purchase_order: PO,
items: [{ description: "Aggregate", qty: 2, unit: "ton", purchase_order_item: ITEM }],
checks: ["po_resolved", "product_matched", "units_convertible", "not_duplicate", "qty_plausible"].map((id) => ({ id, passed: true })),
};
state.delivery = null;
state.events = [];
state.poItems = [{ id: ITEM, purchase_order: PO, description: "Aggregate", unit: "ton", qty: 10, rate: 5 }];
state.pages = [{ id: "page00000000001", image: "page.png", mime: "image/png" }];
promoteNote.mockClear();
const png = Uint8Array.from(Buffer.from("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=", "base64"));
vi.stubGlobal("fetch", vi.fn(async () => new Response(png, { status: 200 })));
});
describe("post delivery note route", () => {
it("returns blocking reasons without promoting a needs-review candidate", async () => {
state.note = { ...state.note, stage: "needs_review", purchase_order: "", checks: [] };
const { POST } = await import("./route");
const response = await POST(request(), context());
expect(response.status).toBe(409);
expect(await response.json()).toMatchObject({ error: "needs_review" });
expect(promoteNote).not.toHaveBeenCalled();
});
it("promotes a ready candidate and records one posting audit", async () => {
const { POST } = await import("./route");
const response = await POST(request(), context());
expect(response.status).toBe(200);
expect(await response.json()).toEqual({ deliveryNoteId: DELIVERY, itemsCreated: 1, stage: "posted" });
expect(promoteNote).toHaveBeenCalledOnce();
expect(state.events).toHaveLength(1);
expect(state.events[0]).toMatchObject({ action: "post", note: NOTE });
});
it("returns the same delivery note on an idempotent retry", async () => {
state.note = { ...state.note, status: "promoted", stage: "posted", delivery_note: DELIVERY };
state.delivery = { id: DELIVERY, batch_note: NOTE };
const { POST } = await import("./route");
const response = await POST(request(), context());
expect(response.status).toBe(200);
expect(await response.json()).toEqual({ deliveryNoteId: DELIVERY, itemsCreated: 1, stage: "posted" });
expect(promoteNote).not.toHaveBeenCalled();
});
});

View file

@ -0,0 +1,168 @@
import { PDFDocument } from "pdf-lib";
import { NextResponse } from "next/server";
import { pbAdmin } from "@/lib/pb";
import { evaluateCandidateReadiness } from "@/lib/dn-pipeline/readiness";
import { promoteNote, type PromotePoItem } from "@/lib/dn-pipeline/promote";
import { pocketBaseFileUrl } from "@/lib/dn-pipeline/run-staged";
import type { NoteSegment, PageItem } from "@/lib/dn-pipeline/types";
export const dynamic = "force-dynamic";
export const runtime = "nodejs";
const idPattern = /^[a-z0-9]{15}$/i;
const object = (value: unknown): value is Record<string, unknown> =>
typeof value === "object" && value !== null && !Array.isArray(value);
const relationId = (value: unknown): string =>
typeof value === "string" ? value : object(value) && typeof value.id === "string" ? value.id : "";
function posted(row: Record<string, unknown>): boolean {
return row.status === "posted" || row.status === "promoted" || row.stage === "posted" ||
Boolean(row.posted_at) || Boolean(row.delivery_note);
}
function filter(pb: unknown, raw: string, params: Record<string, unknown>): string {
const client = pb as { filter?: (query: string, values: Record<string, unknown>) => string };
return typeof client.filter === "function"
? client.filter(raw, params)
: raw.replace(/\{:(\w+)\}/g, (_match, key: string) => JSON.stringify(params[key]));
}
function asItems(value: unknown): PageItem[] {
if (!Array.isArray(value)) return [];
return value.filter(object).map((item) => ({
description: typeof item.description === "string" ? item.description : "",
qty: typeof item.qty === "number" ? item.qty : Number(item.qty ?? 0),
unit: typeof item.unit === "string" ? item.unit : null,
unit_price: typeof item.unit_price === "number" ? item.unit_price : null,
line_amount: typeof item.line_amount === "number" ? item.line_amount : null,
product_code: typeof item.product_code === "string" ? item.product_code : null,
...item,
})) as PageItem[];
}
async function buildPagePdf(pb: Awaited<ReturnType<typeof pbAdmin>>, pages: Record<string, unknown>[]): Promise<Buffer> {
const pdf = await PDFDocument.create();
for (const page of pages) {
if (typeof page.image !== "string" || !page.image) throw new Error("candidate page image missing");
const response = await fetch(pocketBaseFileUrl(pb, page as { id: string }, "dn_batch_pages", page.image), {
headers: { Authorization: pb.authStore.token },
});
if (!response.ok) throw new Error(`candidate page fetch failed: ${response.status}`);
const image = await response.arrayBuffer();
const embedded = await pdf.embedPng(image);
const rendered = pdf.addPage([embedded.width, embedded.height]);
rendered.drawImage(embedded, { x: 0, y: 0, width: embedded.width, height: embedded.height });
}
return Buffer.from(await pdf.save());
}
function toNote(row: Record<string, unknown>): NoteSegment {
const pages = Array.isArray(row.pages) ? row.pages.filter((page): page is number => Number.isInteger(page)) : [];
return {
note_index: Number(row.note_index ?? 0),
delivery_note_number: String(row.delivery_note_number ?? row.doc_number ?? ""),
page_from: Number(row.page_from ?? pages[0] ?? 1),
page_to: Number(row.page_to ?? pages.at(-1) ?? pages[0] ?? 1),
pages,
doc_type: (row.doc_type === "invoice" || row.doc_type === "collection_note" || row.doc_type === "unknown") ? row.doc_type : "delivery_note",
doc_number_alts: Array.isArray(row.doc_number_alts) ? row.doc_number_alts.filter((v): v is string => typeof v === "string") : [],
po_number_raw: typeof row.po_number_raw === "string" ? row.po_number_raw : null,
supplier_name_raw: typeof row.supplier_name_raw === "string" ? row.supplier_name_raw : null,
supplier_registration_raw: typeof row.supplier_registration_raw === "string" ? row.supplier_registration_raw : null,
customer_vat_no: typeof row.customer_vat_no === "string" ? row.customer_vat_no : null,
date: typeof row.date === "string" ? row.date : null,
received_by_name: typeof row.received_by_name === "string" ? row.received_by_name : null,
gross_mass: typeof row.gross_mass === "number" ? row.gross_mass : null,
tare_mass: typeof row.tare_mass === "number" ? row.tare_mass : null,
nett_mass: typeof row.nett_mass === "number" ? row.nett_mass : null,
mass_unit: typeof row.mass_unit === "string" ? row.mass_unit : null,
quantity: typeof row.quantity === "number" ? row.quantity : null,
unit: typeof row.unit === "string" ? row.unit : null,
items: asItems(row.items),
};
}
export async function POST(_request: Request, { params }: { params: Promise<{ id: string; noteId: string }> }) {
const { id, noteId } = await params;
if (!idPattern.test(id) || !idPattern.test(noteId)) return NextResponse.json({ error: "invalid_id" }, { status: 400 });
const pb = await pbAdmin();
const [batch, note] = await Promise.all([
pb.collection("dn_batches").getOne(id, { fields: "id,site,page_count,assembly_revision,revision" }).catch(() => null),
pb.collection("dn_batch_notes").getOne(noteId, { fields: "*" }).catch(() => null),
]) as [Record<string, unknown> | null, Record<string, unknown> | null];
if (!batch) return NextResponse.json({ error: "batch_not_found" }, { status: 404 });
if (!note) return NextResponse.json({ error: "note_not_found" }, { status: 404 });
if (relationId(note.batch) !== id) return NextResponse.json({ error: "note_not_in_batch" }, { status: 400 });
if (posted(note)) {
const deliveryNoteId = relationId(note.delivery_note);
if (!deliveryNoteId) return NextResponse.json({ error: "posted_candidate_missing_delivery_note" }, { status: 409 });
const delivery = await pb.collection("delivery_notes").getOne(deliveryNoteId, { fields: "id,batch_note" }).catch(() => null) as Record<string, unknown> | null;
if (!delivery || relationId(delivery.batch_note) !== noteId) return NextResponse.json({ error: "posted_delivery_note_mismatch" }, { status: 409 });
const items = await pb.collection("delivery_note_items").getFullList({ filter: filter(pb, "delivery_note = {:deliveryNote}", { deliveryNote: deliveryNoteId }), fields: "id" });
await ensurePostedAudit(pb, id, noteId, deliveryNoteId);
return NextResponse.json({ deliveryNoteId, itemsCreated: items.length, stage: "posted" });
}
if (note.status === "dismissed" || note.stage === "dismissed") return NextResponse.json({ error: "dismissed_candidate" }, { status: 409 });
const siteId = relationId(batch.site);
const supplierId = relationId(note.supplier);
const purchaseOrderId = relationId(note.purchase_order);
const [supplier, purchaseOrder, poItems, pages] = await Promise.all([
supplierId ? pb.collection("suppliers").getOne(supplierId, { fields: "id,name" }).catch(() => null) : Promise.resolve(null),
purchaseOrderId ? pb.collection("purchase_orders").getOne(purchaseOrderId, { fields: "id,site,supplier,supplier_ref" }).catch(() => null) : Promise.resolve(null),
purchaseOrderId ? pb.collection("purchase_order_items").getFullList({ filter: filter(pb, "purchase_order = {:purchaseOrder}", { purchaseOrder: purchaseOrderId }), fields: "id,purchase_order,description,unit,material_code,qty,rate" }).catch(() => []) : Promise.resolve([]),
pb.collection("dn_batch_pages").getFullList({ filter: filter(pb, "batch = {:batch}", { batch: id }), sort: "created", fields: "id,image,mime,source_page,region" }).catch(() => []),
]);
const poSiteId = relationId((purchaseOrder as Record<string, unknown> | null)?.site);
const rawPoSupplier = relationId((purchaseOrder as Record<string, unknown> | null)?.supplier);
const poSupplierRef = relationId((purchaseOrder as Record<string, unknown> | null)?.supplier_ref);
const poSupplierId = poSupplierRef || (idPattern.test(rawPoSupplier) ? rawPoSupplier : "");
const checks = Array.isArray(note.checks) ? note.checks.filter(object) : [];
const readiness = evaluateCandidateReadiness({
batch,
note,
supplierId,
purchaseOrderId,
purchaseOrderSiteId: poSiteId,
purchaseOrderSupplierId: poSupplierId,
poItems: poItems as Record<string, unknown>[],
checks,
});
if (!readiness.ready || !supplier || !purchaseOrder || !siteId) {
const reasons = [...readiness.reasons];
if (!siteId && !reasons.includes("batch_site_missing")) reasons.push("batch_site_missing");
if (!supplier && !reasons.includes("supplier_unresolved")) reasons.push("supplier_unresolved");
if (!purchaseOrder && !reasons.includes("purchase_order_unresolved")) reasons.push("purchase_order_unresolved");
return NextResponse.json({ error: "needs_review", reasons }, { status: 409 });
}
if (!Array.isArray(pages) || pages.length === 0) return NextResponse.json({ error: "needs_review", reasons: ["candidate_pages_missing"] }, { status: 409 });
const orderedPages = (pages as Record<string, unknown>[]).slice(0, Math.max(1, Array.isArray(note.pages) ? note.pages.length : 1));
let pageFile: Buffer;
try { pageFile = await buildPagePdf(pb, orderedPages); } catch (error) { return NextResponse.json({ error: "candidate_page_unavailable", detail: error instanceof Error ? error.message : String(error) }, { status: 500 }); }
const normalizedNote = toNote(note);
const poItemById = new Map((poItems as Record<string, unknown>[]).map((item) => [relationId(item.id), item]));
const lineMatches = normalizedNote.items.map((item) => {
const linked = object(item) ? relationId((item as unknown as Record<string, unknown>).purchase_order_item) : "";
const poItem = poItemById.get(linked);
return { description: item.description, match: { kind: "matched" as const, via: "exact" as const, poItem: poItem as unknown as PromotePoItem } };
});
try {
const result = await promoteNote({
pb, note: normalizedNote, batchNoteId: noteId, batchId: id, siteId,
purchaseOrderId, supplierId, poItems: (poItems as Record<string, unknown>[]).map((item) => ({
id: relationId(item.id), description: String(item.description ?? ""), ...(typeof item.unit === "string" ? { unit: item.unit } : {}),
})),
pageFile, pageFileName: `${normalizedNote.delivery_note_number || noteId}.pdf`, pageFileMime: "application/pdf", lineMatches,
});
await ensurePostedAudit(pb, id, noteId, result.deliveryNoteId);
return NextResponse.json({ ...result, stage: "posted" });
} catch (error) {
return NextResponse.json({ error: "promotion_failed", detail: error instanceof Error ? error.message : String(error) }, { status: 500 });
}
}
async function ensurePostedAudit(pb: Awaited<ReturnType<typeof pbAdmin>>, batchId: string, noteId: string, deliveryNoteId: string): Promise<void> {
const events = await pb.collection("dn_review_events").getFullList({ filter: filter(pb, "batch = {:batch} && note = {:note} && action = {:action}", { batch: batchId, note: noteId, action: "post" }), fields: "id" });
if (events.length === 0) await pb.collection("dn_review_events").create({ batch: batchId, note: noteId, action: "post", before: { stage: "ready_to_post" }, after: { stage: "posted", delivery_note: deliveryNoteId }, reason: "candidate posted", actor: "web" });
}