From e6cac6e4477eddd99c9465a8d2bbc1e51a7e8bab Mon Sep 17 00:00:00 2001 From: Dave Date: Sat, 17 Oct 2026 01:18:10 +0200 Subject: [PATCH] feat(collection-ai): data model, settings, migration Co-Authored-By: Claude Sonnet 5.5 --- package.json | 1 + pocketbase/pb_hooks/audit_log_utils.js | 2 +- .../2026-10-17-add-collection-ai.ts | 143 ++++++++++++++++++ src/lib/server/ai-settings-service.test.ts | 73 +++++++++ src/lib/server/ai-settings-service.ts | 66 +++++++- src/lib/server/collection-confirm-service.ts | 15 ++ src/lib/types.ts | 59 ++++++++ 7 files changed, 357 insertions(+), 2 deletions(-) create mode 100644 scripts/migrations/2026-10-17-add-collection-ai.ts diff --git a/package.json b/package.json index ca9ad44..ff8a8c5 100644 --- a/package.json +++ b/package.json @@ -21,6 +21,7 @@ "pb:migrate:audit-log": "npx tsx scripts/migrations/2026-09-25-create-audit-log.ts", "pb:migrate:rep-trip-legs": "npx tsx scripts/migrations/2026-09-28-create-rep-trip-legs.ts", "pb:migrate:quote-assistant": "npx tsx scripts/migrations/2026-10-16-add-quote-assistant-fields.ts", + "pb:migrate:collection-ai": "npx tsx scripts/migrations/2026-10-17-add-collection-ai.ts", "pb:migrate:giellie-chats": "npx tsx scripts/migrations/2026-09-30-create-giellie-chats.ts", "pb:migrate:place-overrides": "npx tsx scripts/migrations/2026-09-28-create-place-overrides.ts", "pb:migrate:drop-orders-status": "npx tsx scripts/migrations/2026-06-27-drop-orders-status-and-wipe.ts", diff --git a/pocketbase/pb_hooks/audit_log_utils.js b/pocketbase/pb_hooks/audit_log_utils.js index caec6d4..26e1a50 100644 --- a/pocketbase/pb_hooks/audit_log_utils.js +++ b/pocketbase/pb_hooks/audit_log_utils.js @@ -19,7 +19,7 @@ const IGNORED_FIELDS = { email_tracking_id: 1, has_new_reply: 1, last_reply_at: 1, pdf_generated_at: 1, quote_matching_attempted: 1, delivery_note_matching_attempted: 1, duplicate_order_matching_attempted: 1, order_number_duplicate_checked: 1, - email_opened: 1, email_opened_at: 1, email_delivered: 1, email_delivered_at: 1, + email_opened: 1, email_opened_at: 1, email_delivered: 1, email_delivered_at: 1, ai_check: 1, }; // Key fields recorded when a line is added or removed, so the trail can diff --git a/scripts/migrations/2026-10-17-add-collection-ai.ts b/scripts/migrations/2026-10-17-add-collection-ai.ts new file mode 100644 index 0000000..14a065c --- /dev/null +++ b/scripts/migrations/2026-10-17-add-collection-ai.ts @@ -0,0 +1,143 @@ +/** + * One-shot migration: schema for the Collection Confirm AI checks. + * + * collection_doc_reads (new collection) one row per invoice photo the AI read + * order_items.ai_check json (per-line check record) + * stock_purchase_items.ai_check json (per-line check record) + * ai_settings.collection_ai_enabled bool + * ai_settings.collection_ai_prompt text (max 50000) + * ai_settings.collection_ai_last_run_at text (max 40) + * ai_settings.collection_ai_last_run_summary json + * + * Run with: + * + * npm run pb:migrate:collection-ai + * + * Idempotent — an existing collection or field is skipped. Manual step on + * the live PB (like select values and pb_hooks). + * + * NOTE: PB text fields with `max: 0` cap at 5000 chars, NOT unlimited, so + * every text field gets an explicit max. + * + * Rollback: delete the collection and the fields in the PB admin UI. + */ +import "./_env-loader"; +import PocketBase from "pocketbase"; + +const PB_URL = process.env.NEXT_PUBLIC_POCKETBASE_URL || process.env.PB_URL || "http://127.0.0.1:8090"; +const ADMIN_EMAIL = process.env.PB_ADMIN_EMAIL; +const ADMIN_PASSWORD = process.env.PB_ADMIN_PASSWORD; + +const TAG = "[collection-ai]"; + +type FieldDef = Record & { name: string }; + +const text = (name: string, max = 500): FieldDef => ({ + name, + type: "text", + required: false, + presentable: false, + max, +}); + +const json = (name: string, maxSize: number): FieldDef => ({ + name, + type: "json", + required: false, + presentable: false, + maxSize, +}); + +const DOC_READ_FIELDS: FieldDef[] = [ + text("source_collection", 100), + text("source_id", 50), + text("filename", 500), + text("content_hash", 100), + text("status", 20), + text("doc_type", 30), + text("document_no", 200), + text("doc_date", 40), + text("supplier_name", 300), + text("model", 100), + text("error", 2000), + text("read_at", 40), + json("po_refs", 200000), + json("lines", 200000), + json("notes", 200000), + json("findings", 200000), +]; + +const AI_CHECK_FIELDS: FieldDef[] = [json("ai_check", 50000)]; + +const AI_SETTINGS_FIELDS: FieldDef[] = [ + { name: "collection_ai_enabled", type: "bool", required: false, presentable: false }, + text("collection_ai_prompt", 50000), + text("collection_ai_last_run_at", 40), + json("collection_ai_last_run_summary", 20000), +]; + +async function addMissingFields(pb: PocketBase, collectionName: string, wanted: FieldDef[]) { + const collection = await pb.collections.getOne(collectionName); + const existing = new Set(collection.fields.map((f: { name: string }) => f.name)); + const missing = wanted.filter((f) => !existing.has(f.name)); + + for (const f of wanted) { + if (existing.has(f.name)) { + console.log(`${TAG} ${collectionName}.${f.name} already exists — skipping.`); + } + } + if (missing.length === 0) return; + + await pb.collections.update(collection.id, { + fields: [...collection.fields, ...missing], + }); + for (const f of missing) { + console.log(`${TAG} ${collectionName}.${f.name} added.`); + } +} + +async function main() { + if (!ADMIN_EMAIL || !ADMIN_PASSWORD) { + throw new Error("Set PB_ADMIN_EMAIL and PB_ADMIN_PASSWORD (e.g. from .env.local)."); + } + const pb = new PocketBase(PB_URL); + await pb.collection("_superusers").authWithPassword(ADMIN_EMAIL, ADMIN_PASSWORD); + console.log(`${TAG} Authenticated against ${PB_URL}`); + + const all = await pb.collections.getFullList(); + if (all.find((c) => c.name === "collection_doc_reads")) { + console.log(`${TAG} collection_doc_reads already exists — skipping.`); + } else { + await pb.collections.create({ + name: "collection_doc_reads", + type: "base", + // Reads are open to signed-in users; every write goes through the + // server's admin client. + listRule: '@request.auth.id != ""', + viewRule: '@request.auth.id != ""', + createRule: null, + updateRule: null, + deleteRule: null, + fields: [ + ...DOC_READ_FIELDS, + { name: "created", type: "autodate", onCreate: true, onUpdate: false }, + { name: "updated", type: "autodate", onCreate: true, onUpdate: true }, + ], + indexes: [ + "CREATE UNIQUE INDEX idx_collection_doc_reads_hash ON collection_doc_reads (content_hash)", + "CREATE INDEX idx_collection_doc_reads_source ON collection_doc_reads (source_collection, source_id, filename)", + ], + }); + console.log(`${TAG} Created collection_doc_reads.`); + } + + await addMissingFields(pb, "order_items", AI_CHECK_FIELDS); + await addMissingFields(pb, "stock_purchase_items", AI_CHECK_FIELDS); + await addMissingFields(pb, "ai_settings", AI_SETTINGS_FIELDS); + console.log(`${TAG} Done.`); +} + +main().catch((err) => { + console.error(`${TAG} Migration failed:`, err); + process.exit(1); +}); diff --git a/src/lib/server/ai-settings-service.test.ts b/src/lib/server/ai-settings-service.test.ts index 90cf59f..7ed109f 100644 --- a/src/lib/server/ai-settings-service.test.ts +++ b/src/lib/server/ai-settings-service.test.ts @@ -2,6 +2,7 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; import { clearAiSettingsCache, getAiSettings, + recordCollectionAiRun, recordQuoteAssistantRun, updateAiSettings, } from "@/lib/server/ai-settings-service"; @@ -168,3 +169,75 @@ describe("recordQuoteAssistantRun", () => { expect(pb2.update).not.toHaveBeenCalled(); }); }); + +describe("collection AI settings", () => { + it("defaults the fields when the row lacks them", async () => { + buildPb([BASE_ROW]); + const s = await getAiSettings(); + expect(s.collection_ai_enabled).toBe(false); + expect(s.collection_ai_prompt).toBe(""); + expect(s.collection_ai_last_run_at).toBe(""); + expect(s.collection_ai_last_run_summary).toBeNull(); + }); + + it("reads stored values", async () => { + const summary = { + started_at: "a", + finished_at: "b", + files_read: 2, + cards_checked: 5, + errors: [], + }; + buildPb([ + { + ...BASE_ROW, + collection_ai_enabled: true, + collection_ai_prompt: "custom", + collection_ai_last_run_at: "2026-10-17T09:00:00.000Z", + collection_ai_last_run_summary: summary, + }, + ]); + const s = await getAiSettings(); + expect(s.collection_ai_enabled).toBe(true); + expect(s.collection_ai_prompt).toBe("custom"); + expect(s.collection_ai_last_run_at).toBe("2026-10-17T09:00:00.000Z"); + expect(s.collection_ai_last_run_summary).toEqual(summary); + }); + + it("caps the prompt at 50 000 characters", async () => { + const pb = buildPb([BASE_ROW]); + await updateAiSettings({ collection_ai_prompt: "x".repeat(60_000) }); + const sent = pb.update.mock.calls[0][1].collection_ai_prompt as string; + expect(sent).toHaveLength(50_000); + }); + + it("writes the enabled flag", async () => { + const pb = buildPb([BASE_ROW]); + await updateAiSettings({ collection_ai_enabled: true }); + expect(pb.update.mock.calls[0][1].collection_ai_enabled).toBe(true); + }); +}); + +describe("recordCollectionAiRun", () => { + it("writes only the last-run fields and refreshes the cache", async () => { + const pb = buildPb([BASE_ROW]); + await recordCollectionAiRun({ last_run_at: "2026-10-17T10:00:00.000Z" }); + expect(pb.update).toHaveBeenCalledTimes(1); + expect(pb.update).toHaveBeenCalledWith("s1", { + collection_ai_last_run_at: "2026-10-17T10:00:00.000Z", + }); + pb.getFullList.mockClear(); + const s = await getAiSettings(); + expect(s.collection_ai_last_run_at).toBe("2026-10-17T10:00:00.000Z"); + expect(pb.getFullList).not.toHaveBeenCalled(); + }); + + it("does nothing when there is no settings row or no fields", async () => { + const pb = buildPb([]); + await recordCollectionAiRun({ last_run_at: "x" }); + expect(pb.update).not.toHaveBeenCalled(); + const pb2 = buildPb([BASE_ROW]); + await recordCollectionAiRun({}); + expect(pb2.update).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/server/ai-settings-service.ts b/src/lib/server/ai-settings-service.ts index 404193e..45316cb 100644 --- a/src/lib/server/ai-settings-service.ts +++ b/src/lib/server/ai-settings-service.ts @@ -30,7 +30,7 @@ * over encryption-at-rest on 2026-08-09. Don't add encryption here * without re-checking with them. */ -import type { QuoteAssistantRunSummary } from "@/lib/types"; +import type { CollectionAiRunSummary, QuoteAssistantRunSummary } from "@/lib/types"; import { getAdminPocketBase } from "./pocketbase-admin"; export type AiProvider = "anthropic" | "minimax"; @@ -67,6 +67,13 @@ export interface AiSettings { /** ISO timestamp of the last tick; "" = never. */ quote_assistant_last_run_at: string; quote_assistant_last_run_summary: QuoteAssistantRunSummary | null; + /** Collection Confirm AI checks: background tick on/off (default off). */ + collection_ai_enabled: boolean; + /** Operator override of the collection-check prompt; "" = use the default. */ + collection_ai_prompt: string; + /** ISO timestamp of the last tick; "" = never. */ + collection_ai_last_run_at: string; + collection_ai_last_run_summary: CollectionAiRunSummary | null; updated_at: string; } @@ -97,6 +104,9 @@ export interface AiSettingsUpdate { quote_assistant_min_confidence?: QuoteAssistantMinConfidence; /** `""` clears the override. Capped at 50 000 chars. */ quote_assistant_prompt?: string; + collection_ai_enabled?: boolean; + /** `""` clears the override. Capped at 50 000 chars. */ + collection_ai_prompt?: string; } const COLLECTION = "ai_settings"; @@ -119,6 +129,10 @@ const DEFAULT_SETTINGS: AiSettings = { quote_assistant_prompt: "", quote_assistant_last_run_at: "", quote_assistant_last_run_summary: null, + collection_ai_enabled: false, + collection_ai_prompt: "", + collection_ai_last_run_at: "", + collection_ai_last_run_summary: null, updated_at: "", }; @@ -149,6 +163,10 @@ interface RawSettingsRow { quote_assistant_prompt?: string; quote_assistant_last_run_at?: string; quote_assistant_last_run_summary?: QuoteAssistantRunSummary | null; + collection_ai_enabled?: boolean; + collection_ai_prompt?: string; + collection_ai_last_run_at?: string; + collection_ai_last_run_summary?: CollectionAiRunSummary | null; updated_at?: string; } @@ -165,6 +183,13 @@ function normaliseRunSummary(value: unknown): QuoteAssistantRunSummary | null { return null; } +function normaliseCollectionRunSummary(value: unknown): CollectionAiRunSummary | null { + if (value && typeof value === "object" && !Array.isArray(value)) { + return value as CollectionAiRunSummary; + } + return null; +} + function rowToSettings(row: RawSettingsRow): AiSettings { return { id: row.id, @@ -189,6 +214,12 @@ function rowToSettings(row: RawSettingsRow): AiSettings { quote_assistant_last_run_summary: normaliseRunSummary( row.quote_assistant_last_run_summary, ), + collection_ai_enabled: !!row.collection_ai_enabled, + collection_ai_prompt: row.collection_ai_prompt ?? "", + collection_ai_last_run_at: row.collection_ai_last_run_at ?? "", + collection_ai_last_run_summary: normaliseCollectionRunSummary( + row.collection_ai_last_run_summary, + ), updated_at: row.updated_at ?? "", }; } @@ -331,6 +362,13 @@ export async function updateAiSettings( // Same defensive cap as system_prompt (PB field max is 50 000). dbPatch.quote_assistant_prompt = patch.quote_assistant_prompt.slice(0, 50_000); } + if (patch.collection_ai_enabled !== undefined) { + dbPatch.collection_ai_enabled = !!patch.collection_ai_enabled; + } + if (patch.collection_ai_prompt !== undefined) { + // Same defensive cap as system_prompt (PB field max is 50 000). + dbPatch.collection_ai_prompt = patch.collection_ai_prompt.slice(0, 50_000); + } dbPatch.updated_at = new Date().toISOString(); @@ -386,6 +424,32 @@ export async function recordQuoteAssistantRun(fields: { cacheSet(rowToSettings(saved)); } +/** + * Persist the outcome of a Collection Confirm AI tick. Mirrors + * `recordQuoteAssistantRun`: writes ONLY the last-run fields (never + * `updated_at`), refreshes the cache, no-op without a row or fields. + */ +export async function recordCollectionAiRun(fields: { + last_run_at?: string; + summary?: CollectionAiRunSummary; +}): Promise { + const dbPatch: Record = {}; + if (fields.last_run_at !== undefined) { + dbPatch.collection_ai_last_run_at = fields.last_run_at; + } + if (fields.summary !== undefined) { + dbPatch.collection_ai_last_run_summary = fields.summary; + } + if (Object.keys(dbPatch).length === 0) return; + + const pb = await getAdminPocketBase(); + const rows = await pb.collection(COLLECTION).getFullList({}); + if (rows.length === 0) return; + const id = (rows[0] as RawSettingsRow).id; + const saved = (await pb.collection(COLLECTION).update(id, dbPatch)) as RawSettingsRow; + cacheSet(rowToSettings(saved)); +} + /** * Convenience for callers that just want the SDK knobs (no record * scaffolding). Equivalent to `getAiSettings({ includeSecret: true })` diff --git a/src/lib/server/collection-confirm-service.ts b/src/lib/server/collection-confirm-service.ts index 282f599..e9b2cc1 100644 --- a/src/lib/server/collection-confirm-service.ts +++ b/src/lib/server/collection-confirm-service.ts @@ -1,6 +1,7 @@ import { getAdminPocketBase } from "@/lib/server/pocketbase-admin"; import { getOrderItemsCollectionName } from "@/lib/server/orders-service"; import { compressFile } from "@/lib/server/attachment-compressor"; +import type { CollectionAiCheck, CollectionDocType } from "@/lib/types"; import { appendCostNote, @@ -109,6 +110,20 @@ export type ConfirmRow = { supplier_sku?: string; supplier?: string; poNumber?: string; + /** AI check of this line against the driver's invoice photo. */ + ai_check?: CollectionAiCheck | null; + /** Document-level summary of the invoice the AI read for this card. */ + ai_doc?: { + read_id: string; + doc_type: CollectionDocType; + document_no: string; + doc_date: string; + supplier_name: string; + findings: string[]; + } | null; + /** True when the line's supplier code matches a stock item (POS). */ + in_pos?: boolean | null; + pos_stock_item_id?: string | null; /** * Lightweight summary of the most recent `price_diff_report` entry * the `po-agent` wrote to this PO's `notes` field. Populated by diff --git a/src/lib/types.ts b/src/lib/types.ts index 2966801..9a268a0 100644 --- a/src/lib/types.ts +++ b/src/lib/types.ts @@ -553,6 +553,65 @@ export interface QuoteAssistantRunSummary { errors: string[]; } +export type CollectionDocType = "tax_invoice" | "delivery_note" | "other"; + +export interface CollectionDocLine { + line_no: number; + code: string; + description: string; + quantity: number | null; + unit: string; + /** Per single unit, already normalised from "per 100" etc. */ + unit_price: number | null; + line_total: number | null; + to_follow: number | null; + price_note: string; +} + +export interface CollectionDocRead { + id: string; + source_collection: string; + source_id: string; + filename: string; + content_hash: string; + status: "read" | "failed" | "skipped"; + doc_type: CollectionDocType | ""; + document_no: string; + doc_date: string; + supplier_name: string; + po_refs: string[]; + lines: CollectionDocLine[]; + notes: string[]; + findings: string[]; + model: string; + error: string; + read_at: string; +} + +export type SameItem = "sure" | "likely" | "different"; + +export interface CollectionAiCheck { + read_id: string; + document_no: string; + doc_date: string; + doc_type: CollectionDocType; + doc_line: CollectionDocLine | null; + same_item: SameItem | null; + reason: string; + sku_filled_value: string; + findings: string[]; + checked_at: string; + model: string; +} + +export interface CollectionAiRunSummary { + started_at: string; + finished_at: string; + files_read: number; + cards_checked: number; + errors: string[]; +} + export interface QuoteItem extends BaseRecord { quote: string; description: string;