WFDASH-WEBAPP/src/lib/server/attachment-extractor.ts

/**
 * Deterministic text extraction from email attachments.
 *
 * Used by both the ingestion service (which can preview whether a
 * file has readable text) and the AI processor (which feeds the
 * extracted text to Claude). Pure synchronous I/O — no DB, no LLM.
 *
 * Routing strategy (matches what the n8n workflow did):
 *   1. TEXT-FIRST — try cheap deterministic extraction:
 *      - text/plain → utf8
 *      - text/csv   → csv-parse (joined as TSV, preserves structure)
 *      - Excel      → SheetJS → sheet_to_csv per sheet
 *      - Word       → mammoth.extractRawText
 *      - PDF        → pdf-parse v2 (PDFParse.getText), fall through
 *                     to vision if the text layer is too thin (< 100
 *                     non-whitespace chars — typical of scanned PDFs)
 *   2. VISION-FALLBACK — for images and scanned PDFs. Caller decides
 *      whether to feed the raw bytes to Claude (we return null here
 *      so the caller can route accordingly).
 *
 * Per-format errors are swallowed with `console.warn` so one bad
 * file doesn't kill the whole extraction pass. The caller sees `null`
 * and either skips the file or hands it to vision.
 *
 * NB: pdf-parse@2.x has a different API from v1 — it ships a class
 * (`PDFParse`) with explicit `destroy()` for cleanup. The legacy
 * `pdf(buffer)` default export no longer exists. `load()` is called
 * internally by the constructor, so we just instantiate + getText +
 * destroy.
 */

import * as XLSX from "xlsx";
import mammoth from "mammoth";
import { parse as parseCsvSync } from "csv-parse/sync";
// pdf-parse v2 ships the `PDFParse` class; no v1 default export.
// @types/pdf-parse describes the v1 API and doesn't apply here.
//
// LAZY-LOADED: see `loadPdfParse()` below. The module is dynamically
// imported only when a PDF attachment is actually encountered (most
// quotes have none). This avoids bundling pdfjs-dist (a transitive
// dep) at module-load time, which broke Next.js's webpack loader with
// "Object.defineProperty called on non-object" — pdfjs-dist's ESM
// build uses ES2022 features that webpack's `__webpack_require__.r`
// doesn't handle. By deferring the import, we keep pdf-parse out of
// the route's bundle entirely. The cost is one dynamic-import round-
// trip on the first PDF of a batch (≈ 50-100 ms cold, then cached).
let pdfParsePromise: Promise<typeof import("pdf-parse")> | null = null;
function loadPdfParse() {
  if (!pdfParsePromise) {
    pdfParsePromise = import("pdf-parse");
  }
  return pdfParsePromise;
}

// LAZY-LOADED: same webpack rationale as `loadPdfParse()`. pdfjs-dist
// is only needed when a scanned PDF needs to be rendered to an image
// for vision. The `legacy/build/pdf.mjs` entry is CJS-friendly and
// avoids the worker-thread plumbing that's awkward in a Next.js route
// handler. We pair it with `@napi-rs/canvas` (an optional dep of
// pdfjs-dist, already on disk) for the canvas backend — `page.render()`
// writes pixels into a `<canvas>`-shaped buffer, which we then encode
// to JPEG in-process.
let pdfJsPromise:
  | Promise<typeof import("pdfjs-dist/legacy/build/pdf.mjs")>
  | null = null;
function loadPdfJs() {
  if (!pdfJsPromise) {
    pdfJsPromise = import("pdfjs-dist/legacy/build/pdf.mjs");
  }
  return pdfJsPromise;
}

interface PdfRenderLimits {
  /** Hard cap on number of pages rendered per file. */
  maxPages: number;
  /** Resize so the longest edge is at most this many CSS pixels. */
  maxWidthPx: number;
  /** JPEG quality 0..1. */
  jpegQuality: number;
}
const SCANNED_PDF_RENDER_LIMITS: PdfRenderLimits = {
  maxPages: 5, // cap so a 50-page archive PDF doesn't blow the input budget
  maxWidthPx: 1600, // ≈ Anthropic's recommended ceiling for inline vision
  jpegQuality: 0.85,
};

export interface RawAttachment {
  filename: string;
  contentType: string;
  /** Raw bytes — base64-decoded by the caller before passing in. */
  bytes: Buffer;
}

/** Discriminated union: either we got text, or the file needs vision. */
export type ExtractedAttachment =
  | {
      filename: string;
      contentType: string;
      kind: "text" | "doc" | "pdf" | "spreadsheet" | "csv";
      /** Deterministic text extraction succeeded. */
      text: string;
      thin: false;
    }
  | {
      filename: string;
      contentType: string;
      /** No text layer — caller should pass `bytes` to a vision model. */
      kind: "image" | "pdf-thin" | "unknown";
      text: null;
      bytes: Buffer;
    };

const MIN_TEXT_CHARS_FOR_PDF = 100;

/**
 * Extract text (or signal "send to vision") from one attachment.
 * Never throws — returns `text: null` for any failure path so the
 * caller can continue processing the rest of the batch.
 */
export async function extractAttachmentText(
  att: RawAttachment,
): Promise<ExtractedAttachment> {
  const ct = (att.contentType || "").toLowerCase();
  const fn = (att.filename || "").toLowerCase();
  const ext = extname(fn);

  // ── Plain text ────────────────────────────────────────────────
  if (ct === "text/plain" || ext === "txt") {
    try {
      const text = att.bytes.toString("utf8");
      return { filename: att.filename, contentType: att.contentType, kind: "text", text, thin: false };
    } catch (e) {
      console.warn("[attachment-extractor] text decode failed", fn, e);
      return visionFallback(att, "unknown");
    }
  }

  // ── CSV ───────────────────────────────────────────────────────
  if (ct === "text/csv" || ext === "csv") {
    try {
      const rows = parseCsvSync(att.bytes, { columns: false }) as string[][];
      const text = rows.map((r) => r.join("\t")).join("\n");
      return { filename: att.filename, contentType: att.contentType, kind: "csv", text, thin: false };
    } catch (e) {
      console.warn("[attachment-extractor] csv parse failed", fn, e);
      return visionFallback(att, "unknown");
    }
  }

  // ── Spreadsheets (xlsx, xls, ods) ────────────────────────────
  if (
    ct.includes("sheet") ||
    ct.includes("ms-excel") ||
    ext === "xlsx" ||
    ext === "xls" ||
    ext === "ods"
  ) {
    try {
      const wb = XLSX.read(att.bytes, { type: "buffer" });
      const text = wb.SheetNames.map((name) => {
        const sheet = wb.Sheets[name];
        return `# Sheet: ${name}\n${XLSX.utils.sheet_to_csv(sheet)}`;
      }).join("\n\n");
      return { filename: att.filename, contentType: att.contentType, kind: "spreadsheet", text, thin: false };
    } catch (e) {
      console.warn("[attachment-extractor] xlsx parse failed", fn, e);
      return visionFallback(att, "unknown");
    }
  }

  // ── Word documents (docx, doc) ────────────────────────────────
  if (
    ct.includes("officedocument") ||
    ct.includes("msword") ||
    ext === "docx" ||
    ext === "doc"
  ) {
    try {
      const result = await mammoth.extractRawText({ buffer: att.bytes });
      return { filename: att.filename, contentType: att.contentType, kind: "doc", text: result.value, thin: false };
    } catch (e) {
      console.warn("[attachment-extractor] mammoth failed", fn, e);
      return visionFallback(att, "unknown");
    }
  }

  // ── PDF — text first, fall through to vision if too thin ──────
  if (ct === "application/pdf" || ext === "pdf") {
    // Resolve the parser lazily — see the comment at the top of the
    // file for why. The first PDF in a process costs one dynamic-
    // import round-trip; subsequent PDFs reuse the cached module.
    const PDFParse = (await loadPdfParse()).PDFParse;
    let parser: InstanceType<typeof PDFParse> | null = null;
    try {
      parser = new PDFParse({ data: new Uint8Array(att.bytes) });
      const result = await parser.getText();
      const text = (result?.text ?? "").trim();
      const nonWsLen = text.replace(/\s+/g, "").length;
      if (nonWsLen >= MIN_TEXT_CHARS_FOR_PDF) {
        return { filename: att.filename, contentType: att.contentType, kind: "pdf", text, thin: false };
      }
      // Thin text layer — almost always a scanned PDF. Render the
      // first few pages to JPEG and return as `image` so the AI
      // processor's vision pipeline picks it up. We deliberately
      // collapse the multi-page case into a single image (page 1
      // only) to keep the Claude request shape consistent — the
      // page count is logged so the operator can see in the logs
      // when an attachment was truncated.
      console.warn(
        `[attachment-extractor] pdf "${fn}" has thin text layer (${nonWsLen} chars), rendering to vision`,
      );
      return await renderPdfPagesToImages(att, SCANNED_PDF_RENDER_LIMITS);
    } catch (e) {
      console.warn("[attachment-extractor] pdf-parse failed", fn, e);
      // Parser error — try the same vision render path so a corrupt
      // header or unsupported XREF table still produces something
      // visible. If rendering also fails, `renderPdfPagesToImages`
      // returns the legacy `pdf-thin` shape via the outer try/catch.
      try {
        return await renderPdfPagesToImages(att, SCANNED_PDF_RENDER_LIMITS);
      } catch (renderErr) {
        console.warn(
          "[attachment-extractor] pdf-parse + render both failed",
          fn,
          renderErr,
        );
        return {
          filename: att.filename,
          contentType: att.contentType,
          kind: "pdf-thin",
          text: null,
          bytes: att.bytes,
        };
      }
    } finally {
      try { parser?.destroy(); } catch { /* ignore */ }
    }
  }

  // ── Images — no text layer, hand straight to vision ───────────
  if (ct.startsWith("image/")) {
    return visionFallback(att, "image");
  }

  // ── Unknown / unsupported ─────────────────────────────────────
  console.warn(
    `[attachment-extractor] unsupported type "${ct || "(none)"}" for "${fn}" — skipping`,
  );
  return visionFallback(att, "unknown");
}

/**
 * Batch-extract many attachments in parallel. Per-file failures are
 * isolated — the returned array has the same length as the input,
 * with `text: null` slots for any file that failed extraction.
 */
export async function extractAttachmentsBatch(
  atts: RawAttachment[],
): Promise<ExtractedAttachment[]> {
  return Promise.all(atts.map(extractAttachmentText));
}

// ── helpers ────────────────────────────────────────────────────

function extname(fn: string): string {
  const i = fn.lastIndexOf(".");
  return i >= 0 ? fn.slice(i + 1) : "";
}

function visionFallback(
  att: RawAttachment,
  kind: "image" | "unknown",
): ExtractedAttachment {
  return {
    filename: att.filename,
    contentType: att.contentType,
    kind,
    text: null,
    bytes: att.bytes,
  };
}

/**
 * Render a PDF (typically scanned, with no text layer) to a JPEG image
 * suitable for inline Claude vision. Lazy-loads `pdfjs-dist` and
 * `@napi-rs/canvas` on demand — neither is bundled at module-load time.
 *
 * Returns the same `kind: "image"` shape as a native JPEG attachment so
 * the downstream AI processor needs no special casing.
 *
 * Failure modes (all swallowed — return `kind: "unknown"` so the
 * caller can skip the file rather than crash the whole batch):
 *   - `pdfjs.getDocument` throws (encrypted, malformed, XREF error)
 *   - `@napi-rs/canvas` import fails (no native binary on this platform)
 *   - `doc.numPages === 0` (empty document)
 *   - `page.render` rejects (corrupt page stream)
 */
async function renderPdfPagesToImages(
  att: RawAttachment,
  limits: PdfRenderLimits,
): Promise<ExtractedAttachment> {
  const fn = (att.filename || "").toLowerCase();
  const pdfjs = await loadPdfJs();
  // Lazy-load the canvas backend on demand — same rationale as pdfjs.
  const { createCanvas } = await import("@napi-rs/canvas");

  const doc = await pdfjs.getDocument({
    data: new Uint8Array(att.bytes),
    isEvalSupported: false,
    useWorkerFetch: false,
    // Disable pdfjs's worker thread. Node has no DOM Worker but
    // pdfjs's main-thread fallback works fine when we pass our own
    // canvas. This avoids the "Worker not initialized" error some
    // bundlers raise when pdfjs tries to spawn a worker.
    verbosity: 0,
  }).promise;

  const totalPages = doc.numPages;
  if (!totalPages || totalPages <= 0) {
    throw new Error(`pdf has 0 pages (${fn})`);
  }
  const pagesToRender = Math.min(totalPages, limits.maxPages);
  if (totalPages > pagesToRender) {
    console.warn(
      `[attachment-extractor] pdf "${fn}" has ${totalPages} pages, rendering first ${pagesToRender} as vision`,
    );
  }

  // We collapse the multi-page case into a single image (page 1). The
  // Claude API accepts multiple image blocks per message, but the AI
  // processor's contract is one image per attachment — keep that shape
  // until we have a real-world example that needs more.
  const page = await doc.getPage(1);
  const baseViewport = page.getViewport({ scale: 1 });
  const longestEdge = Math.max(baseViewport.width, baseViewport.height);
  const scale = longestEdge > limits.maxWidthPx
    ? limits.maxWidthPx / longestEdge
    : 2.0; // bump small pages up so OCR-able text is sharp
  const viewport = page.getViewport({ scale });

  const canvas = createCanvas(viewport.width, viewport.height);
  const ctx = canvas.getContext("2d");
  // Cast through `unknown` for the render() arg — pdfjs-dist's Node
  // entry types `canvasContext` as `CanvasRenderingContext2D` and
  // `canvas` as `HTMLCanvasElement`, but @napi-rs/canvas implements
  // the same surface under its own `SKRSContext2D` / `Canvas` types.
  // The cast is a type-level workaround; at runtime both objects
  // satisfy pdfjs's render() contract (a 2D context with
  // fillRect/clearRect/etc. and an HTMLCanvas-shaped shape with
  // width/height).
  await page.render({
    canvasContext: ctx as unknown as globalThis.CanvasRenderingContext2D,
    viewport,
    canvas: canvas as unknown as globalThis.HTMLCanvasElement,
  }).promise;

  const jpeg = canvas.toBuffer("image/jpeg", limits.jpegQuality);
  // Try to release the doc. pdfjs keeps parsed pages in memory; this
  // matters on the cron path where many quotes are processed in a row.
  try { await doc.destroy(); } catch { /* ignore */ }

  const baseName = att.filename.replace(/\.pdf$/i, "");
  const synthName = totalPages > 1
    ? `${baseName}-page-1-of-${totalPages}.jpg`
    : `${baseName}.jpg`;

  console.log(
    `[attachment-extractor] rendered ${fn} (page 1/${totalPages}) → JPEG ${(jpeg.length / 1024).toFixed(1)} KB`,
  );

  return {
    filename: synthName,
    contentType: "image/jpeg",
    kind: "image",
    text: null,
    bytes: jpeg,
  };
}