Workflow-App/Workflow/app/api/cipc/report/route.ts

// app/api/cipc/report/route.ts
// CIPC Director Appointment Check — reconciles CIPC BizPortal against the
// Workflow database for every director ID we hold.
//
// WHAT IT REPORTS (and only this):
//
//   1. NEW APPOINTMENT  — CIPC says this person is a director of company X,
//      but the database does not list them as a director of X.
//   2. NO LONGER A DIRECTOR — the database lists this person as a director of
//      company X, but CIPC no longer shows them there (removed from the
//      CIPC record, or shown as resigned).
//
// Everything else is in sync and is NOT reported.
//
// HOW IT WORKS
//   POST  → starts a background run of "Bizportal Directopr id no search.py"
//           in --batch mode, writing a log we poll.
//   GET   → returns run status; once the run finishes, returns the two
//           exception lists. `?demo=1` returns a fixed sample instead.
//
// The .py is co-authored and is NOT modified by this route — it is invoked
// exactly as it already behaves, including its own login rotation.

import { NextRequest, NextResponse } from "next/server";
import { spawn } from "node:child_process";
import fs from "node:fs";
import path from "node:path";
import { pbFetch } from "@/lib/pb";

export const dynamic = "force-dynamic";
export const maxDuration = 300;

const SCRIPTS_DIR = path.join(process.cwd(), "scripts");
const RUN_DIR = path.join(process.cwd(), ".cipc-report");
const SCRIPT_NAME = "Bizportal Directopr id no search.py";
const BATCH_FILE = path.join(RUN_DIR, "ids.txt");
const LOG_FILE = path.join(RUN_DIR, "run.log");
const STATE_FILE = path.join(RUN_DIR, "state.json");
const DEMO_FILE = path.join(SCRIPTS_DIR, "cipc-report", "sample-report.json");

// ===================================================================
// TYPES
// ===================================================================

export interface CbizRow {
  enterprise_number: string;
  enterprise_name: string;
  enterprise_status: string;
  director_status: string;
}

export interface ExceptionRow {
  id_no: string;
  director: string;
  company: string;
  enterprise_no: string;
  status: string;
  /** "active" for a new appointment, "resigned" / "missing" for a departure. */
  reason: "active" | "resigned" | "missing";
  /** ysterbaard_directors record id — empty when we hold no director record. */
  director_pb_id: string;
  /** ysterbaard_companies record id — empty when we hold no company record. */
  company_pb_id: string;
  /** True when the company itself is already on file. */
  company_exists: boolean;
}

export interface ReportPayload {
  ok: boolean;
  status: "idle" | "running" | "done" | "error";
  is_demo?: boolean;
  started_at?: string;
  finished_at?: string;
  total?: number;
  done?: number;
  current_id?: string | null;
  message?: string;
  error?: string;
  checked?: number;
  failed?: number;
  failed_ids?: string[];
  new_appointments?: ExceptionRow[];
  no_longer_director?: ExceptionRow[];
}

// ===================================================================
// PB READ HELPERS
// ===================================================================

async function fetchAll<T>(collection: string): Promise<T[]> {
  const out: T[] = [];
  let page = 1;
  for (;;) {
    const list = await pbFetch<{ items?: T[]; totalPages?: number }>(
      `/api/collections/${collection}/records`,
      { page, perPage: 200 }
    );
    const items = list.items || [];
    out.push(...items);
    if (page >= (list.totalPages || 1) || items.length === 0) break;
    page += 1;
  }
  return out;
}

interface PbDirector {
  id: string;
  id_no?: string;
  first_names?: string;
  surname?: string;
}
interface PbCompany {
  id: string;
  enterprise_no?: string;
  enterprise_name?: string;
  enterprise_status?: string;
}
interface PbLink {
  company?: string;
  director?: string;
}

/** Normalised enterprise number used for matching. */
function entKey(v: unknown): string {
  return String(v ?? "").trim().toUpperCase();
}

/**
 * CIPC reports the director status as ACTIVE / INACTIVE. A naive
 * /active/i test would read "INACTIVE" as active, so check for the
 * negative first.
 */
function directorIsActive(status: unknown): boolean {
  const s = String(status ?? "").trim().toLowerCase();
  if (!s) return false;
  if (s.includes("inactive")) return false;
  return s.includes("active");
}

function fullName(first?: string, surname?: string): string {
  return [first || "", surname || ""].join(" ").replace(/\s+/g, " ").trim();
}

/** A usable full SA ID: 13 digits, not an all-zero placeholder. */
function isFullId(v: unknown): boolean {
  const s = String(v ?? "").trim();
  return /^\d{13}$/.test(s) && !/^0{13}$/.test(s);
}

// ===================================================================
// THE ID LIST
//
// The same population as the "Bizportal Search List" export: every full
// 13-digit ID across the four collections that hold one, de-duplicated.
// ===================================================================

async function collectTargetIds(): Promise<{
  ids: string[];
  names: Map<string, string>;
}> {
  const directors = await fetchAll<PbDirector>("ysterbaard_directors");
  const regDirectors = await fetchAll<PbDirector>("ysterbaard_reg_directors");
  const shareholders = await fetchAll<{ id_or_entity_no?: string; name?: string }>(
    "ysterbaard_reg_shareholders"
  );
  const onboarding = await fetchAll<{ directors_data?: any }>(
    "ysterbaard_onboarding_submissions"
  );

  const names = new Map<string, string>();
  const ids: string[] = [];
  const seen = new Set<string>();

  const push = (id: unknown, name?: string) => {
    const s = String(id ?? "").trim();
    if (!isFullId(s) || seen.has(s)) return;
    seen.add(s);
    ids.push(s);
    if (name && !names.has(s)) names.set(s, name);
  };

  for (const d of directors) push(d.id_no, fullName(d.first_names, d.surname));
  for (const d of regDirectors) push(d.id_no, fullName(d.first_names, d.surname));
  for (const s of shareholders) {
    const raw = String(s.name ?? "").trim();
    const parts = raw.split(/\s+/);
    push(s.id_or_entity_no, parts.length > 1 ? parts.slice(0, -1).join(" ") + " " + parts[parts.length - 1] : raw);
  }
  for (const o of onboarding) {
    const dd = o.directors_data;
    const list = typeof dd === "string" ? safeJson(dd) : dd;
    for (const p of Array.isArray(list) ? list : []) {
      push(p?.id_no, fullName(p?.first_names, p?.surname));
    }
  }

  return { ids, names };
}

function safeJson(s: string): any {
  try {
    return JSON.parse(s);
  } catch {
    return [];
  }
}

// ===================================================================
// LOG PARSING
//
// The .py prints one block per ID:
//
//   ========== Director ID 8901010001087 ==========
//     logged in OK (Login 3)
//     Searches Conducted Today: 44
//
//   | Enterprise Number | Enterprise Name | Enterprise Status | Director Status |
//   |---|---|---|---|
//   | K2026108471 | BRANDLINE HOLDINGS (PTY) LTD | In Business | Active |
//
// ===================================================================

function parseDirectorLog(stdout: string): Map<string, CbizRow[]> {
  const out = new Map<string, CbizRow[]>();
  const marker = /=+\s*Director ID\s+(\d{13})\s*=+/g;
  const bounds: { id: string; start: number; end: number }[] = [];
  let m: RegExpExecArray | null;
  while ((m = marker.exec(stdout)) !== null) {
    bounds.push({ id: m[1], start: m.index, end: -1 });
  }
  bounds.forEach((b, i) => {
    b.end = i + 1 < bounds.length ? bounds[i + 1].start : stdout.length;
  });

  for (const b of bounds) {
    const body = stdout.slice(b.start, b.end);
    const rows: CbizRow[] = [];
    for (const line of body.split(/\r?\n/)) {
      const t = line.trim();
      if (!t.startsWith("|")) continue;
      const cells = t.split("|").slice(1, -1).map((c) => c.trim());
      if (cells.every((c) => /^[-:]+$/.test(c))) continue;
      if (cells[0] === "Enterprise Number") continue;
      if (!/^[KMB]\d{10,}$/.test(cells[0])) continue;
      rows.push({
        enterprise_number: cells[0],
        enterprise_name: cells[1] ?? "",
        enterprise_status: cells[2] ?? "",
        director_status: cells[3] ?? "",
      });
    }
    out.set(b.id, rows);
  }
  return out;
}

/** IDs the .py announced but produced no table for. */
function parseFailedIds(stdout: string): Set<string> {
  const failed = new Set<string>();
  const marker = /=+\s*Director ID\s+(\d{13})\s*=+/g;
  const bounds: { id: string; start: number; end: number }[] = [];
  let m: RegExpExecArray | null;
  while ((m = marker.exec(stdout)) !== null) {
    bounds.push({ id: m[1], start: m.index, end: -1 });
  }
  bounds.forEach((b, i) => {
    b.end = i + 1 < bounds.length ? bounds[i + 1].start : stdout.length;
  });
  for (const b of bounds) {
    const body = stdout.slice(b.start, b.end);
    if (/WARN: no director-table rows found/.test(body) || /ERROR:/.test(body)) {
      failed.add(b.id);
    }
  }
  return failed;
}

// ===================================================================
// RECONCILIATION
// ===================================================================

async function buildReport(stdout: string): Promise<{
  new_appointments: ExceptionRow[];
  no_longer_director: ExceptionRow[];
  checked: number;
  failed: number;
  failed_ids: string[];
}> {
  const { ids, names } = await collectTargetIds();
  const directors = await fetchAll<PbDirector>("ysterbaard_directors");
  const companies = await fetchAll<PbCompany>("ysterbaard_companies");
  const links = await fetchAll<PbLink>("ysterbaard_company_directors");

  const companyById = new Map(companies.map((c) => [c.id, c]));
  const companyByEnt = new Map(companies.map((c) => [entKey(c.enterprise_no), c]));

  // director record id → set of enterprise numbers they are linked to
  const linkedEntByDirector = new Map<string, Set<string>>();
  for (const l of links) {
    if (!l.director || !l.company) continue;
    const comp = companyById.get(l.company);
    if (!comp) continue;
    const key = entKey(comp.enterprise_no);
    if (!key) continue;
    if (!linkedEntByDirector.has(l.director)) linkedEntByDirector.set(l.director, new Set());
    linkedEntByDirector.get(l.director)!.add(key);
  }

  // ID number → director record id (so a person found only in reg_* can still
  // be matched to the directors record that holds their company links)
  const dirRecByIdNo = new Map<string, string>();
  for (const d of directors) {
    const k = String(d.id_no ?? "").trim();
    if (isFullId(k) && !dirRecByIdNo.has(k)) dirRecByIdNo.set(k, d.id);
  }

  const biz = parseDirectorLog(stdout);
  const failedIds = parseFailedIds(stdout);

  const newAppointments: ExceptionRow[] = [];
  const noLonger: ExceptionRow[] = [];
  const failedList: string[] = [];
  let checked = 0;
  let failed = 0;

  for (const id of ids) {
    if (failedIds.has(id)) {
      failed += 1;
      failedList.push(id);
      continue;
    }
    const rows = biz.get(id);
    if (!rows) {
      failed += 1;
      failedList.push(id);
      continue;
    }
    checked += 1;

    const name = names.get(id) || "";
    const dirRec = dirRecByIdNo.get(id);
    const linked = dirRec ? linkedEntByDirector.get(dirRec) || new Set<string>() : new Set<string>();

    // 1. NEW APPOINTMENT — active at CIPC, but not linked in our database.
    for (const r of rows) {
      const key = entKey(r.enterprise_number);
      if (!key || linked.has(key)) continue;
      if (!directorIsActive(r.director_status)) continue;
      const known = companyByEnt.get(key);
      newAppointments.push({
        id_no: id,
        director: name,
        company: r.enterprise_name,
        enterprise_no: r.enterprise_number,
        status: r.enterprise_status || "",
        reason: "active",
        director_pb_id: dirRec || "",
        company_pb_id: known?.id || "",
        company_exists: Boolean(known),
      });
    }

    // 2. NO LONGER A DIRECTOR — linked in our database, but CIPC no longer
    //    shows them there (resigned, or dropped from the record).
    const bizByEnt = new Map(rows.map((r) => [entKey(r.enterprise_number), r]));
    for (const key of linked) {
      const row = bizByEnt.get(key);
      const comp = companyByEnt.get(key);
      if (!row) {
        noLonger.push({
          id_no: id,
          director: name,
          company: comp?.enterprise_name || key,
          enterprise_no: comp?.enterprise_no || key,
          status: comp?.enterprise_status || "",
          reason: "missing",
          director_pb_id: dirRec || "",
          company_pb_id: comp?.id || "",
          company_exists: Boolean(comp),
        });
        continue;
      }
      if (!directorIsActive(row.director_status)) {
        noLonger.push({
          id_no: id,
          director: name,
          company: row.enterprise_name,
          enterprise_no: row.enterprise_number,
          status: row.enterprise_status || "",
          reason: "resigned",
          director_pb_id: dirRec || "",
          company_pb_id: comp?.id || "",
          company_exists: Boolean(comp),
        });
      }
    }
  }

  return {
    new_appointments: newAppointments,
    no_longer_director: noLonger,
    checked,
    failed,
    failed_ids: failedList,
  };
}

// ===================================================================
// RUN STATE
// ===================================================================

interface RunState {
  status: "idle" | "running" | "done" | "error";
  started_at?: string;
  finished_at?: string;
  total?: number;
  error?: string;
  pid?: number;
}

function readState(): RunState {
  try {
    if (!fs.existsSync(STATE_FILE)) return { status: "idle" };
    return JSON.parse(fs.readFileSync(STATE_FILE, "utf-8")) as RunState;
  } catch {
    return { status: "idle" };
  }
}

function writeState(s: RunState): void {
  fs.mkdirSync(RUN_DIR, { recursive: true });
  fs.writeFileSync(STATE_FILE, JSON.stringify(s, null, 2), "utf-8");
}

function progressFromLog(): { done: number; current_id: string | null } {
  try {
    if (!fs.existsSync(LOG_FILE)) return { done: 0, current_id: null };
    const log = fs.readFileSync(LOG_FILE, "utf-8");
    const seen = [...log.matchAll(/=+\s*Director ID\s+(\d{13})\s*=+/g)].map((m) => m[1]);
    const done = seen.length;
    const last = seen.length ? seen[seen.length - 1] : null;
    return { done, current_id: last };
  } catch {
    return { done: 0, current_id: null };
  }
}

function pythonPath(): string {
  return path.join(
    SCRIPTS_DIR,
    ".venv",
    "Scripts",
    process.platform === "win32" ? "python.exe" : "python"
  );
}

// ===================================================================
// ROUTES
// ===================================================================

export async function GET(req: NextRequest) {
  const demo = req.nextUrl.searchParams.get("demo") === "1";
  if (demo) {
    try {
      const payload = JSON.parse(fs.readFileSync(DEMO_FILE, "utf-8"));
      return NextResponse.json({ ...payload, ok: true, is_demo: true, status: "done" });
    } catch (err: any) {
      return NextResponse.json(
        { ok: false, error: `Demo data unavailable: ${err?.message || String(err)}` },
        { status: 500 }
      );
    }
  }

  const state = readState();
  const progress = progressFromLog();

  if (state.status === "running") {
    return NextResponse.json({
      ok: true,
      status: "running",
      started_at: state.started_at,
      total: state.total,
      done: progress.done,
      current_id: progress.current_id,
    } satisfies ReportPayload);
  }

  if (state.status === "error") {
    return NextResponse.json({
      ok: false,
      status: "error",
      error: state.error,
      started_at: state.started_at,
      finished_at: state.finished_at,
      total: state.total,
      done: progress.done,
    } satisfies ReportPayload);
  }

  if (state.status === "done") {
    let stdout = "";
    try {
      stdout = fs.readFileSync(LOG_FILE, "utf-8");
    } catch {
      stdout = "";
    }
    const report = await buildReport(stdout);
    return NextResponse.json({
      ok: true,
      status: "done",
      started_at: state.started_at,
      finished_at: state.finished_at,
      total: state.total,
      done: progress.done,
      checked: report.checked,
      failed: report.failed,
      failed_ids: report.failed_ids,
      new_appointments: report.new_appointments,
      no_longer_director: report.no_longer_director,
    } satisfies ReportPayload);
  }

  return NextResponse.json({ ok: true, status: "idle" } satisfies ReportPayload);
}

export async function POST() {
  const state = readState();
  if (state.status === "running") {
    const progress = progressFromLog();
    return NextResponse.json({
      ok: true,
      status: "running",
      started_at: state.started_at,
      total: state.total,
      done: progress.done,
      current_id: progress.current_id,
    } satisfies ReportPayload);
  }

  const { ids } = await collectTargetIds();
  if (ids.length === 0) {
    return NextResponse.json(
      { ok: false, error: "No director ID numbers with a full 13-digit number were found." },
      { status: 400 }
    );
  }

  fs.mkdirSync(RUN_DIR, { recursive: true });
  fs.writeFileSync(BATCH_FILE, ids.join("\n") + "\n", "utf-8");
  fs.writeFileSync(LOG_FILE, "", "utf-8");

  const py = pythonPath();
  if (!fs.existsSync(py)) {
    return NextResponse.json(
      { ok: false, error: `Python runtime not found at ${py}` },
      { status: 500 }
    );
  }

  const logFd = fs.openSync(LOG_FILE, "a");

  const child = spawn(py, [path.join(SCRIPTS_DIR, SCRIPT_NAME), "--batch", BATCH_FILE], {
    cwd: SCRIPTS_DIR,
    windowsHide: true,
    // The child's stdout/stderr go straight to the log file. Piping them
    // through Node makes the script's own agent-browser subprocess calls
    // block — the .py's first browser call never returned when spawned
    // this way, and the run sat there for 9 minutes with no output.
    stdio: ["ignore", logFd, logFd],
  });

  writeState({
    status: "running",
    started_at: new Date().toISOString(),
    total: ids.length,
    pid: child.pid,
  });

  child.on("close", (code) => {
    const finished = new Date().toISOString();
    if (code === 0 || code === 1) {
      writeState({
        status: "done",
        started_at: readState().started_at,
        finished_at: finished,
        total: ids.length,
      });
    } else {
      writeState({
        status: "error",
        started_at: readState().started_at,
        finished_at: finished,
        total: ids.length,
        error:
          code === 2
            ? "BizPortal stopped the run — the daily search limit was reached or the subscription expired. Run it again tomorrow."
            : `The lookup script exited with code ${code}.`,
      });
    }
  });

  child.on("error", (err) => {
    writeState({
      status: "error",
      started_at: readState().started_at,
      finished_at: new Date().toISOString(),
      total: ids.length,
      error: err?.message || String(err),
    });
  });

  return NextResponse.json({
    ok: true,
    status: "running",
    started_at: new Date().toISOString(),
    total: ids.length,
  } satisfies ReportPayload);
}