WFDASH-WEBAPP/scripts/smoke/ai-settings-round-trip.ts

/**
 * Smoke test for the AI settings round-trip + processor wiring.
 *
 * What this verifies
 * ──────────────────
 *   1. Migration ran successfully (ai_settings row exists or can be
 *      created from defaults).
 *   2. `updateAiSettings` persists + `getAiSettings` reflects the
 *      change with the 60s cache busted (i.e. no stale read).
 *   3. `getAiSettings()` redacts api_key by default; `getAiSettings(
 *      { includeSecret: true })` returns it.
 *   4. `getActiveAiConfig()` honors the `enabled` killswitch
 *      (throws AiDisabledError when off).
 *   5. The processor's new code path actually reads from ai_settings:
 *      we ingest one quote, call processOneQuote(), and assert that
 *      the resulting ai_error is NOT "ai_disabled" when we set
 *      enabled=true + a bogus api_key (we expect a 401 from Anthropic
 *      or similar, NOT the killswitch path).
 *
 * Cleanup
 * ───────
 * The test mutates ai_settings and creates one quote. Both are
 * reverted at the end (best-effort) so a successful run leaves PB
 * indistinguishable from before. If the script crashes mid-run, the
 * worst case is enabled=false + a placeholder api_key on ai_settings
 * — the killswitch prevents any cost.
 *
 * Run with:
 *   npx tsx scripts/smoke/ai-settings-round-trip.ts
 *
 * Pre-requisites:
 *   1. Migration ran: `npm run pb:migrate:ai-settings`
 *   2. .env.local has NEXT_PUBLIC_POCKETBASE_URL, PB_ADMIN_EMAIL,
 *      PB_ADMIN_PASSWORD.
 */
import "./_env"; // MUST be first
import {
  getAiSettings,
  updateAiSettings,
  getActiveAiConfig,
  AiDisabledError,
  clearAiSettingsCache,
} from "../../src/lib/server/ai-settings-service";
import { ingestQuoteFromEmail } from "../../src/lib/server/quotes-from-email-service";
import { processOneQuote } from "../../src/lib/server/quotes-ai-processor";
import { getAdminPocketBase } from "../../src/lib/server/pocketbase-admin";

const SMOKE_FROM = "smoke-ai-settings@example.invalid";
const SMOKE_SUBJECT = "[SMOKE] AI settings round trip";
const SMOKE_MESSAGE_ID = "smoke-ai-settings-2026-08-09-001@wfdash.local";

async function main() {
  console.log("=== AI settings round-trip smoke test ===\n");

  // ── 1. Capture original settings so we can restore at the end ─────
  clearAiSettingsCache();
  const original = await getAiSettings({ includeSecret: true });
  console.log("[1/8] Captured original ai_settings:");
  console.log(`     provider=${original.provider || "(empty)"}  enabled=${original.enabled}`);
  console.log(`     model_id="${original.model_id}"  base_url="${original.base_url}"`);
  console.log(`     max_tokens=${original.max_tokens}  api_key_set=${original.api_key_set}`);

  let quoteId: string | null = null;
  let customerId: string | null = null;
  let companyId: string | null = null;
  try {
    // ── 2. Update ai_settings via the service ───────────────────────
    console.log("\n[2/8] updateAiSettings({ enabled:true, provider:'anthropic', model_id, api_key, max_tokens:1024 })…");
    await updateAiSettings({
      enabled: true,
      provider: "anthropic",
      model_id: "claude-sonnet-4-5-20250929",
      base_url: "",
      max_tokens: 1024,
      api_key: "smoke-test-key-DO-NOT-USE",
    });
    // Service bumps cache on write; subsequent read must reflect.
    const after = await getAiSettings({ includeSecret: true });
    if (after.enabled !== true) throw new Error("enabled did not flip to true");
    if (after.provider !== "anthropic") throw new Error(`provider stayed "${after.provider}"`);
    if (after.model_id !== "claude-sonnet-4-5-20250929") throw new Error("model_id did not update");
    if (after.api_key !== "smoke-test-key-DO-NOT-USE") throw new Error("api_key did not update");
    if (after.max_tokens !== 1024) throw new Error(`max_tokens stayed ${after.max_tokens}`);
    console.log("     ✓ all fields updated as expected");

    // ── 3. Default read redacts api_key ─────────────────────────────
    const redacted = await getAiSettings();
    if (redacted.api_key !== "") throw new Error(`expected api_key="" in redacted view, got "${redacted.api_key}"`);
    if (!redacted.api_key_set) throw new Error("expected api_key_set=true after we set a key");
    console.log("\n[3/8] Redacted view returns api_key=\"\" + api_key_set=true ✓");

    // ── 4. getActiveAiConfig returns the live values ────────────────
    const active = await getActiveAiConfig();
    if (active.provider !== "anthropic") throw new Error(`active.provider=${active.provider}`);
    if (active.apiKey !== "smoke-test-key-DO-NOT-USE") throw new Error("active.apiKey mismatch");
    if (active.maxTokens !== 1024) throw new Error(`active.maxTokens=${active.maxTokens}`);
    console.log(`\n[4/8] getActiveAiConfig() → provider=${active.provider}, maxTokens=${active.maxTokens} ✓`);

    // ── 5. Killswitch: enabled=false throws AiDisabledError ────────
    await updateAiSettings({ enabled: false });
    let disabledThrown = false;
    try {
      await getActiveAiConfig();
    } catch (e) {
      if (e instanceof AiDisabledError) disabledThrown = true;
      else throw e;
    }
    if (!disabledThrown) throw new Error("AiDisabledError NOT thrown when enabled=false");
    console.log("\n[5/8] Killswitch honored — AiDisabledError thrown when enabled=false ✓");

    // ── 6. Re-enable so the processor actually tries to call ────────
    await updateAiSettings({ enabled: true });

    // ── 7. End-to-end: ingest a quote + run processOneQuote ─────────
    console.log("\n[6/8] Ingesting smoke-test quote + running processOneQuote()…");
    const ingest = await ingestQuoteFromEmail({
      from_email: SMOKE_FROM,
      from_name: "Smoke Settings",
      subject: SMOKE_SUBJECT,
      body_plain:
        "Please quote 5 x M10 x 30mm hex bolts grade 8.8 zinc. " +
        "Need delivery to Pretoria.",
      message_id: SMOKE_MESSAGE_ID,
      company_hint: "Smoke Settings Co",
    });
    quoteId = ingest.quote.id;
    customerId = ingest.customer.id;
    companyId = ingest.company?.id ?? null;
    console.log(`     ingested quote ${ingest.quote.quote_number} (${quoteId})`);

    const proc = await processOneQuote({ quoteId });
    console.log(`     processOneQuote → outcome=${proc.outcome}  error="${proc.error.slice(0, 100)}"`);
    if (proc.outcome !== "failed") {
      throw new Error(`expected outcome=failed (bogus key), got ${proc.outcome}`);
    }
    if (proc.error === "ai_disabled") {
      throw new Error(
        "Processor short-circuited with ai_disabled despite enabled=true — " +
        "cache may not be invalidating on updateAiSettings",
      );
    }
    if (proc.error.length === 0) {
      throw new Error("Processor returned no error message — should have 401 or similar");
    }
    console.log("     ✓ Processor read ai_settings (ai_error ≠ ai_disabled)");

    // Verify the ai_error was stamped on the quote row.
    const pb = await getAdminPocketBase();
    const afterProc = await pb.collection("quotes").getOne(quoteId);
    if (afterProc.ai_status !== "failed") {
      throw new Error(`Expected ai_status="failed", got "${afterProc.ai_status}"`);
    }
    if (!afterProc.ai_error || afterProc.ai_error.length === 0) {
      throw new Error("ai_error not stamped on the quote row");
    }
    console.log(`     ai_status="failed"  ai_error="${(afterProc.ai_error || "").slice(0, 80)}…"`);

    // ── 8. Validation: bogus provider rejected ─────────────────────
    console.log("\n[7/8] Validation: updateAiSettings rejects unknown provider…");
    let rejected = false;
    try {
      // @ts-expect-error -- intentional bad input
      await updateAiSettings({ provider: "openai_compatible" });
    } catch (e) {
      rejected = true;
      console.log(`     ✓ rejected: ${(e as Error).message}`);
    }
    if (!rejected) throw new Error("unknown provider was silently accepted");

    // ── 9. Validation: max_tokens out of range ─────────────────────
    rejected = false;
    try {
      await updateAiSettings({ max_tokens: 99999 });
    } catch (e) {
      rejected = true;
      console.log(`[8/8] Validation: max_tokens out of range rejected ✓ (${(e as Error).message})`);
    }
    if (!rejected) throw new Error("max_tokens=99999 was silently accepted");

    console.log("\n=== Smoke test PASSED ===");
  } finally {
    // ── Cleanup: restore ai_settings to original + delete smoke rows ─
    console.log("\n[cleanup] Restoring ai_settings + deleting smoke rows…");
    try {
      await updateAiSettings({
        provider: original.provider || "",
        model_id: original.model_id,
        base_url: original.base_url,
        max_tokens: original.max_tokens || 4096,
        enabled: original.enabled,
        // Always clear the placeholder test key, then restore the
        // original (if it was set) — `api_key: ""` then a second
        // pass with the real value would race the cache, so we do
        // them sequentially.
        api_key: "",
      });
      if (original.api_key) {
        await updateAiSettings({ api_key: original.api_key });
      }
      console.log("     ✓ ai_settings restored");
    } catch (e) {
      console.error(`     ! could not restore ai_settings: ${(e as Error).message}`);
    }

    if (quoteId) {
      try {
        const pb = await getAdminPocketBase();
        await pb.collection("quotes").delete(quoteId);
        console.log(`     ✓ deleted smoke quote ${quoteId}`);
      } catch (e) {
        console.log(`     ! could not delete quote: ${(e as Error).message}`);
      }
    }
    if (customerId) {
      try {
        const pb = await getAdminPocketBase();
        await pb.collection("customers").delete(customerId);
        console.log(`     ✓ deleted smoke customer ${customerId}`);
      } catch (e) {
        console.log(`     (skipped customer delete: ${(e as Error).message})`);
      }
    }
    if (companyId) {
      try {
        const pb = await getAdminPocketBase();
        await pb.collection("companies").delete(companyId);
        console.log(`     ✓ deleted smoke company ${companyId}`);
      } catch (e) {
        console.log(`     (skipped company delete: ${(e as Error).message})`);
      }
    }
  }
}

main().catch((e) => {
  console.error("\nSmoke test FAILED:", e.message);
  console.error(e.stack);
  process.exit(1);
});