WFDASH-WEBAPP/scripts/_scrape-spring-pins.ts

/**
 * Headless scraper for NSS spring pins ISO8752 page.
 *
 * Launches local Playwright Chromium, logs in to nssfasteners.co.za,
 * discovers total pages from .toolbar-amount, walks all pages via
 * in-page fetch() loop, then POSTs the assembled CSV to the local
 * CSV sink (scripts/_csv-sink.ts).
 *
 * Usage:
 *   npx tsx scripts/_scrape-spring-pins.ts
 */
import { chromium } from "@playwright/test";
import { writeFileSync } from "node:fs";

const TARGET_URL =
  "https://nssfasteners.co.za/pins/spring-pins/spring-pin-iso8752.html";
const SINK_URL = "http://127.0.0.1:8765";
const LOGIN_URL = "https://nssfasteners.co.za/customer/account/login/";
const EMAIL = "jd@wfsales.co.za";
const PASSWORD = "WFsales01";
const PROFILE_DIR = "/tmp/nss-spring-pins-profile";

async function main(): Promise<void> {
  const browser = await chromium.launchPersistentContext(PROFILE_DIR, {
    headless: true,
    channel: "chromium",
    viewport: { width: 1280, height: 900 },
    userAgent:
      "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
  });
  const page = browser.pages()[0] ?? (await browser.newPage());

  // Polyfill esbuild's __name helper so arrow-function callbacks inside
  // page.evaluate work (tsx emits __name for class/object-name tracking,
  // but the helper isn't defined in the page's global scope).
  await page.addInitScript(() => {
    (globalThis as unknown as { __name: <T>(v: T) => T }).__name = (v: unknown) =>
      v as never;
  });

  // 1. Visit login page (skip if already logged in)
  await page.goto(LOGIN_URL, { waitUntil: "domcontentloaded", timeout: 60000 });
  if (page.url().includes("/login")) {
    console.log("On login page — filling credentials");
    await page.fill('input[name="login[username]"]', EMAIL);
    await page.fill('input[name="login[password]"]', PASSWORD);
    await Promise.all([
      page.waitForLoadState("networkidle", { timeout: 60000 }),
      page.click('button[name="send"]'),
    ]);
    console.log(`After login URL: ${page.url()}`);
  } else {
    console.log(`Already past login, on: ${page.url()}`);
  }

  // 2. Navigate to target page
  await page.goto(TARGET_URL, { waitUntil: "domcontentloaded", timeout: 60000 });
  console.log(`Loaded: ${page.url()}`);
  await page.waitForLoadState("networkidle", { timeout: 30000 });

  // 3. Discover filter sidebar attributes (best-effort)
  const filterInfo = await page.evaluate(() => {
    const out: Record<string, unknown> = {};
    const sidebar = document.querySelector(
      ".filter-options, .block-filter, aside, .sidebar",
    );
    if (sidebar) {
      const groups = sidebar.querySelectorAll(".filter-options-item, .filter-content");
      out.groupCount = groups.length;
      out.groups = Array.from(groups).slice(0, 12).map((g) => ({
        title:
          g.querySelector(".filter-options-title, h3, h4, .title")?.textContent?.trim() ??
          "",
        options: Array.from(
          g.querySelectorAll(".filter-options-content a, .filter-options-content li, .item"),
        )
          .slice(0, 8)
          .map((o) => o.textContent?.replace(/\s+/g, " ").trim().slice(0, 60) ?? ""),
      }));
    }
    const amount = document.querySelector(".toolbar-amount")?.textContent?.replace(/\s+/g, " ").trim();
    out.toolbar = amount;
    out.itemCount = document.querySelectorAll(".product-item").length;
    out.title = document.title;
    return out;
  });
  console.log("Filter sidebar info:");
  console.log(JSON.stringify(filterInfo, null, 2));

  // 4. Walk all pages and parse items
  const scrape = async (url: string) => {
    const scrapeFn = function (targetUrl: string) {
      return (async () => {
        const res = await fetch(targetUrl, { credentials: "include" });
        if (!res.ok) throw new Error(`HTTP ${res.status} for ${targetUrl}`);
        const html = await res.text();
        const doc = new DOMParser().parseFromString(html, "text/html");

        // Discover total pages from toolbar if this is the first call
        const amountEl = doc.querySelector(".toolbar-amount");
        const m = amountEl && amountEl.textContent && amountEl.textContent.match(/of\s+(\d+)/i);
        const total = m ? parseInt(m[1], 10) : 0;

        const items = Array.from(doc.querySelectorAll(".product-item"));
        const rows = items.map(function (item) {
          const titleEl = item.querySelector(
            ".items-center.justify-center.text-primary",
          );
          const title = titleEl && titleEl.textContent ? titleEl.textContent.trim() : "";
          const sku = title.includes(" - ")
            ? title.split(" - ").pop().trim()
            : "";
          const linkEl = item.querySelector(
            "a.product-item-photo, a.product.photo",
          );
          const productUrl = linkEl ? linkEl.getAttribute("href") || "" : "";
          const block = (item.textContent || "").replace(/\s+/g, " ");
          const get = function (label: string): string {
            const re = new RegExp(
              label +
                "\\s*:?\\s*([^\\s].*?)(?=\\s+(?:Diameter|Thread|Grade|Finish|Sold|Metric|Hi-Tensile|Stainless|Mild|Brass|Length|$))",
              "i",
            );
            const mm = block.match(re);
            return mm ? mm[1].trim() : "";
          };

          const material = /Spring Steel/i.test(block)
            ? "Spring Steel CS70"
            : /Stainless Steel/i.test(block)
              ? "Stainless Steel"
              : /Mild Steel/i.test(block)
                ? "Mild Steel"
                : /Brass/i.test(block)
                  ? "Brass"
                  : /Hi-Tensile/i.test(block)
                    ? "Hi-Tensile"
                    : "";
          const diameter = get("Diameter");
          const length = get("Length");
          // Spring pins don't have a real Grade column — the "Spring Steel
          // CS70" text is a Material attribute, not a grade. Clear it so the
          // PosDescription pattern stays clean (size x length SPRING PIN fin).
          const grade = "";
          const finish = get("Finish");
          const pack = /Sold as a Single Piece/i.test(block)
            ? "Single Piece"
            : /Sold as a Box/i.test(block)
              ? "Box"
              : /Sold as a Pack/i.test(block)
                ? "Pack"
                : "";

          const unitPriceEl = item.querySelector(".unit-price");
          const priceMatch = unitPriceEl && unitPriceEl.textContent
            ? unitPriceEl.textContent.match(/R\s*([\d.]+)/)
            : null;
          const unitPrice = priceMatch ? priceMatch[1] : "";

          const desc = title.includes(" - ")
            ? title.split(" - ")[0].trim()
            : "";

          // Spring pins: size = diameter in mm (e.g. "5"), no "M" prefix
          const sizeMatch = diameter.match(/(\d+(?:\.\d+)?)\s*mm/i);
          const size = sizeMatch ? sizeMatch[1].replace(/\.0$/, "") : diameter;

          // Length for the PosDescription (strip " mm" suffix)
          const lengthMatch = length.match(/(\d+(?:\.\d+)?)\s*mm/i);
          const lengthNum = lengthMatch ? lengthMatch[1].replace(/\.0$/, "") : "";

          // PosDescription pattern: "<size>x<length> SPRING PIN <finish>"
          // (no Grade segment for spring pins).
          const finishCode =
            finish.toLowerCase() === "plain"
              ? "PLN"
              : finish.toLowerCase() === "zinc plated"
                ? "ZP"
                : finish.toUpperCase();
          const posDescription = lengthNum
            ? [size + "x" + lengthNum, "SPRING PIN", finishCode]
              .filter(Boolean)
              .join(" ")
            : [size, "SPRING PIN", finishCode]
              .filter(Boolean)
              .join(" ");

          return {
            sku: sku,
            posDescription: posDescription,
            description: desc,
            material: material,
            grade: grade,
            diameter: diameter,
            length: length,
            finish: finish,
            pack: pack,
            unitPrice: unitPrice ? "R " + unitPrice : "",
            url: productUrl,
          };
        });

        return { total: total, rows: rows };
      })();
    };
    return await page.evaluate(scrapeFn, url);
  };

  const first = await scrape(TARGET_URL);
  console.log(
    `Page 1: ${first.rows.length} items; toolbar reports total ${first.total}`,
  );

  const allRows: Array<Record<string, string>> = [...first.rows];
  const pages = Math.max(1, Math.ceil(first.total / 10));
  for (let p = 2; p <= pages; p++) {
    const sep = TARGET_URL.includes("?") ? "&" : "?";
    const url = `${TARGET_URL}${sep}p=${p}`;
    const out = await scrape(url);
    console.log(`Page ${p}/${pages}: ${out.rows.length} items`);
    allRows.push(...out.rows);
  }
  console.log(`TOTAL ROWS scraped: ${allRows.length}`);

  // Dedupe by SKU
  const seen = new Set<string>();
  const deduped = allRows.filter((r) => {
    const k = r.sku ?? "";
    if (!k) return false;
    if (seen.has(k)) return false;
    seen.add(k);
    return true;
  });
  console.log(`Unique rows after SKU dedupe: ${deduped.length}`);

  // Build CSV
  const headers = [
    "sku",
    "posDescription",
    "description",
    "material",
    "grade",
    "diameter",
    "length",
    "finish",
    "pack",
    "unitPrice",
    "url",
  ];
  const esc = (v: unknown) => {
    const s = (v ?? "").toString().replace(/"/g, '""');
    return /[",\n]/.test(s) ? `"${s}"` : s;
  };
  const csv = [headers.join(",")]
    .concat(
      deduped.map((r) => headers.map((h) => esc((r as Record<string, unknown>)[h])).join(",")),
    )
    .join("\n");

  // Write directly to disk (sandboxed shells can't keep background processes
  // alive across calls, so the CSV-sink pattern doesn't work here).
  const outPath = "scripts/nss-spring-pins-iso8752-pilot.csv";
  writeFileSync(outPath, csv);
  console.log(`Wrote ${deduped.length} rows (${csv.length} bytes) to ${outPath}`);

  await browser.close();
}

main().catch((err) => {
  console.error("FATAL", err);
  process.exit(1);
});