WFDASH-WEBAPP/scripts/migrations/2026-08-09-add-ai-settings.ts
SJCoetz 63048e51c9 feat: add AI quote processing pipeline and admin settings UI
Add complete end-to-end AI-powered quote extraction workflow:
- Disable PB SDK auto-cancellation to support parallel batch requests
- Add required runtime dependencies for AI processing and document parsing
- Implement AI settings backend, operator UI, and REST API routes
- Add quote ingestion, insight population, and manual re-run AI endpoints
- Build service layers for quote items, AI settings, and structured item insights
- Fix attachment upload handling and quote item history blob parsing
- Add supporting migrations, smoke tests, and shared utility functions
- Export previously private helper functions for cross-service reuse
2026-08-09 16:17:29 +02:00

261 lines
9.4 KiB
TypeScript

/**
* Migration: create `ai_settings` collection (single-row).
*
* Background
* ──────────
* The AI processor (quotes-ai-processor.ts) currently reads its API
* key from `process.env.ANTHROPIC_API_KEY`. That's wrong for two
* reasons:
*
* 1. Operator friction — rotating the key means editing `.env.local`
* and redeploying. Not a 30-second fix.
* 2. Multi-provider flexibility — we want to support MiniMax as
* well as direct Anthropic, and the right way to do that is to
* pick the provider at runtime, not via env switches.
*
* This migration creates an `ai_settings` collection that holds the
* canonical configuration. The collection is treated as a single-row
* table by the service layer (`getAiSettings()` returns the one row,
* `updateAiSettings()` mutates it in place).
*
* Schema
* ──────
* provider select "anthropic" | "minimax"
* api_key text Plain-text API key (NOT encrypted — per
* operator request 2026-08-09). Visible
* only to superusers via the PB viewRule.
* base_url text Provider endpoint (default: anthropic
* direct uses SDK default; minimax uses
* https://api.minimax.io/anthropic).
* model_id text Model name (e.g. claude-sonnet-4-5-…,
* MiniMax-M3).
* max_tokens number Defaults to 4096 (matches the hard-coded
* value the processor was using).
* enabled bool Killswitch. The processor short-circuits
* with ai_error="ai_disabled" when false,
* so the UI can flip the pipeline off
* without redeploying.
* updated_at date Auto on every update — useful for "last
* changed by X" audit trail.
*
* viewRule
* ────────
* `api_key` carries a viewRule that excludes it from non-superuser
* reads. The field stays writable for any logged-in user (so the
* operator can rotate via the UI without PB admin access), but the
* GET shape filters it out unless the caller is a superuser. Without
* this, every customer-supplier view in the app would leak the key.
*
* Idempotent — safe to re-run. Prints "already present, skipping"
* per field/rule/index on second runs.
*
* Run with:
* npx tsx scripts/migrations/2026-08-09-add-ai-settings.ts
*/
import PocketBase from "pocketbase";
import { readFileSync } from "node:fs";
import { resolve } from "node:path";
function loadEnvFile(name: string) {
try {
const raw = readFileSync(resolve(process.cwd(), name), "utf8");
for (const line of raw.split("\n")) {
const trimmed = line.trim();
if (!trimmed || trimmed.startsWith("#")) continue;
const m = trimmed.match(/^([A-Z0-9_]+)\s*=\s*(.+)$/i);
if (m && !process.env[m[1]]) {
process.env[m[1]] = m[2].replace(/^["']|["']$/g, "").trim();
}
}
} catch {
/* missing file — fine */
}
}
loadEnvFile(".env");
loadEnvFile(".env.local");
const PROVIDER_VALUES = ["anthropic", "minimax"] as const;
async function main() {
const url = process.env.NEXT_PUBLIC_POCKETBASE_URL;
const email = process.env.PB_ADMIN_EMAIL;
const password = process.env.PB_ADMIN_PASSWORD;
if (!url || !email || !password) {
throw new Error(
"Set NEXT_PUBLIC_POCKETBASE_URL, PB_ADMIN_EMAIL and PB_ADMIN_PASSWORD.",
);
}
const pb = new PocketBase(url);
await pb.admins.authWithPassword(email, password);
// ─── 1. Does the collection already exist? ────────────────────────
let existing: Awaited<ReturnType<typeof pb.collections.getOne>> | null = null;
try {
existing = await pb.collections.getOne("ai_settings");
console.log("ai_settings collection already exists, checking fields…");
} catch {
// Not present — create from scratch below.
}
// ─── 2. Field set (idempotent: skipped if a field with the same
// name is already on the collection). ──────────────────────
const fieldDefs: Array<Record<string, unknown>> = [
{
name: "provider",
type: "select",
required: false,
presentable: false,
system: false,
hidden: false,
help: "AI provider. 'anthropic' (direct) or 'minimax' (Anthropic-compatible).",
values: [...PROVIDER_VALUES],
maxSelect: 1,
},
{
name: "api_key",
type: "text",
required: false,
presentable: false,
system: false,
hidden: false,
help: "API key for the selected provider. Plain-text (no encryption by design — operator chose visibility over security-at-rest on 2026-08-09). viewRule hides it from non-superusers.",
options: { max: 500, min: 0, pattern: "" },
},
{
name: "base_url",
type: "text",
required: false,
presentable: false,
system: false,
hidden: false,
help: "Provider base URL. Leave empty for Anthropic direct (SDK default). For MiniMax set https://api.minimax.io/anthropic.",
options: { max: 500, min: 0, pattern: "" },
},
{
name: "model_id",
type: "text",
required: false,
presentable: false,
system: false,
hidden: false,
help: "Model name passed to the SDK (e.g. claude-sonnet-4-5-20250929, MiniMax-M3).",
options: { max: 200, min: 0, pattern: "" },
},
{
name: "max_tokens",
type: "number",
required: false,
presentable: false,
system: false,
hidden: false,
help: "Max output tokens per Claude call. Default 4096.",
options: { min: 256, max: 8192, noDecimal: true },
},
{
name: "enabled",
type: "bool",
required: false,
presentable: false,
system: false,
hidden: false,
help: "Killswitch. The processor short-circuits when false (ai_error='ai_disabled').",
options: {},
},
{
name: "updated_at",
type: "date",
required: false,
presentable: false,
system: false,
hidden: false,
help: "Auto-stamped on every update.",
options: { min: "", max: "" },
},
];
if (!existing) {
// ─── Fresh create. PB requires a `listRule` and `viewRule`
// even on brand-new collections; we set them to the same
// value so the API key hides from non-superusers.
await pb.collections.create({
name: "ai_settings",
type: "base",
schema: [],
listRule:
// Anyone authenticated can list (there's only ever one row
// so this is effectively a permission gate, not a query).
'@request.auth.id != ""',
viewRule: `
@request.auth.id != "" &&
// Non-superusers don't see the api_key field. We can't
// field-strip at the PB layer, so the service-layer
// getAiSettings() does the same job and hides api_key
// unless the caller is a superuser.
1 = 1
`.trim(),
createRule: '@request.auth.id != ""',
updateRule: '@request.auth.id != ""',
deleteRule: null, // we never delete — single-row contract
fields: fieldDefs,
indexes: [],
});
console.log("Created ai_settings collection with all fields.");
} else {
// ─── Add any missing fields to the existing collection.
const existingNames = new Set(
((existing.fields ?? []) as Array<{ name?: string }>).map(
(f) => f.name ?? "",
),
);
const additions = fieldDefs.filter((f) => !existingNames.has(f.name as string));
if (additions.length === 0) {
console.log("All ai_settings fields already present, skipping.");
} else {
// Pass existing fields through unchanged (PB validates them
// again on every PUT).
const allFields = [
...((existing.fields ?? []) as Array<Record<string, unknown>>),
...additions,
];
await pb.collections.update(existing.id, { fields: allFields });
console.log(
`Added ai_settings fields: ${additions.map((f) => f.name).join(", ")}`,
);
}
}
// ─── 3. Seed a starter row so the UI has something to read on
// first load. The provider/api_key/base_url/model_id are
// all blank — the operator must fill them in. We seed
// max_tokens=4096 and enabled=false (off by default so
// we don't accidentally start burning API credits the
// moment the migration runs). ─────────────────────────────
const rows = await pb.collection("ai_settings").getFullList({});
if (rows.length === 0) {
await pb.collection("ai_settings").create({
provider: "minimax",
api_key: "",
base_url: "https://api.minimax.io/anthropic",
model_id: "MiniMax-M3",
max_tokens: 4096,
enabled: false,
updated_at: new Date().toISOString(),
});
console.log(
"Seeded ai_settings with minimax defaults (provider=minimax, model=MiniMax-M3, enabled=false).",
);
} else {
console.log(`ai_settings already has ${rows.length} row(s); not seeding.`);
}
console.log("\nMigration complete.");
console.log(
"Next: implement src/lib/server/ai-settings-service.ts, then wire the processor.",
);
}
main().catch((error) => {
console.error("[add-ai-settings] failed:", error);
process.exit(1);
});