WFDASH-WEBAPP/scripts/_scrape-dowel-pins.ts

/**
 * Headless scraper for NSS solid dowel pins DIN6325 page.
 *
 * Launches local Playwright Chromium, logs in to nssfasteners.co.za,
 * discovers total pages from .toolbar-amount, walks all pages via
 * in-page fetch() loop, then writes the assembled CSV directly to disk.
 *
 * Usage:
 *   npx tsx scripts/_scrape-dowel-pins.ts
 */
import { chromium } from "@playwright/test";
import { writeFileSync } from "node:fs";

const TARGET_URL =
  "https://nssfasteners.co.za/pins/dowel-pins/solid-dowel-pin-din6325.html";
const LOGIN_URL = "https://nssfasteners.co.za/customer/account/login/";
const EMAIL = "jd@wfsales.co.za";
const PASSWORD = "WFsales01";
const PROFILE_DIR = "/tmp/nss-dowel-pins-profile";
const OUT_CSV = "scripts/nss-dowel-pins-din6325-pilot.csv";

async function main(): Promise<void> {
  const browser = await chromium.launchPersistentContext(PROFILE_DIR, {
    headless: true,
    channel: "chromium",
    viewport: { width: 1280, height: 900 },
    userAgent:
      "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
  });
  const page = browser.pages()[0] ?? (await browser.newPage());

  // Polyfill esbuild's __name helper so arrow-function callbacks inside
  // page.evaluate work (tsx emits __name for class/object-name tracking,
  // but the helper isn't defined in the page's global scope).
  await page.addInitScript(() => {
    (globalThis as unknown as { __name: <T>(v: T) => T }).__name = (v: unknown) =>
      v as never;
  });

  // 1. Visit login page (skip if already logged in)
  await page.goto(LOGIN_URL, { waitUntil: "domcontentloaded", timeout: 60000 });
  if (page.url().includes("/login")) {
    console.log("On login page — filling credentials");
    await page.fill('input[name="login[username]"]', EMAIL);
    await page.fill('input[name="login[password]"]', PASSWORD);
    await Promise.all([
      page.waitForLoadState("networkidle", { timeout: 60000 }),
      page.click('button[name="send"]'),
    ]);
    console.log(`After login URL: ${page.url()}`);
  } else {
    console.log(`Already past login, on: ${page.url()}`);
  }

  // 2. Navigate to target page
  await page.goto(TARGET_URL, { waitUntil: "domcontentloaded", timeout: 60000 });
  console.log(`Loaded: ${page.url()}`);
  await page.waitForLoadState("networkidle", { timeout: 30000 });

  // 3. Discover filter sidebar attributes (best-effort)
  const filterInfo = await page.evaluate(() => {
    const out: Record<string, unknown> = {};
    const sidebar = document.querySelector(
      ".filter-options, .block-filter, aside, .sidebar",
    );
    if (sidebar) {
      const groups = sidebar.querySelectorAll(
        ".filter-options-item, .filter-content",
      );
      out.groupCount = groups.length;
      out.groups = Array.from(groups)
        .slice(0, 12)
        .map((g) => ({
          title:
            g.querySelector(".filter-options-title, h3, h4, .title")?.textContent?.trim() ??
            "",
          options: Array.from(
            g.querySelectorAll(
              ".filter-options-content a, .filter-options-content li, .item",
            ),
          )
            .slice(0, 12)
            .map(
              (o) =>
                o.textContent?.replace(/\s+/g, " ").trim().slice(0, 80) ?? "",
            ),
        }));
    }
    const amount = document
      .querySelector(".toolbar-amount")
      ?.textContent?.replace(/\s+/g, " ")
      .trim();
    out.toolbar = amount;
    out.itemCount = document.querySelectorAll(".product-item").length;
    out.title = document.title;
    return out;
  });
  console.log("Filter sidebar info:");
  console.log(JSON.stringify(filterInfo, null, 2));

  // 4. Walk all pages and parse items
  const scrape = async (url: string) => {
    const scrapeFn = function (targetUrl: string) {
      return (async () => {
        const res = await fetch(targetUrl, { credentials: "include" });
        if (!res.ok) throw new Error(`HTTP ${res.status} for ${targetUrl}`);
        const html = await res.text();
        const doc = new DOMParser().parseFromString(html, "text/html");

        const amountEl = doc.querySelector(".toolbar-amount");
        const m =
          amountEl && amountEl.textContent
            ? amountEl.textContent.match(/of\s+(\d+)/i)
            : null;
        const total = m ? parseInt(m[1], 10) : 0;

        const items = Array.from(doc.querySelectorAll(".product-item"));
        const rows = items.map(function (item) {
          const titleEl = item.querySelector(
            ".items-center.justify-center.text-primary",
          );
          const title = titleEl && titleEl.textContent ? titleEl.textContent.trim() : "";
          const sku = title.includes(" - ")
            ? title.split(" - ").pop().trim()
            : "";
          const linkEl = item.querySelector(
            "a.product-item-photo, a.product.photo",
          );
          const productUrl = linkEl ? linkEl.getAttribute("href") || "" : "";
          const block = (item.textContent || "").replace(/\s+/g, " ");
          const get = function (label: string): string {
            const re = new RegExp(
              label +
                "\\s*:?\\s*([^\\s].*?)(?=\\s+(?:Diameter|Thread|Grade|Finish|Sold|Metric|Hi-Tensile|Stainless|Mild|Brass|Length|Case|Hardened|$))",
              "i",
            );
            const mm = block.match(re);
            return mm ? mm[1].trim() : "";
          };

          // Material detection — dowel pins are typically Case-Hardened Steel
          // or Hardened Steel / Stainless Steel / Mild Steel.
          const material = /Case[\s-]?Hardened/i.test(block)
            ? "Case-Hardened Steel"
            : /Hardened\s+Steel/i.test(block)
              ? "Hardened Steel"
              : /Stainless/i.test(block)
                ? "Stainless Steel"
                : /Mild\s+Steel/i.test(block)
                  ? "Mild Steel"
                  : /Hi-Tensile/i.test(block)
                    ? "Hi-Tensile"
                    : "";

          const diameter = get("Diameter");
          const length = get("Length");
          // Dowel pins don't have a meaningful Grade column.
          const grade = "";
          const finish = get("Finish");

          const pack = /Sold as a Single Piece/i.test(block)
            ? "Single Piece"
            : /Sold as a Box/i.test(block)
              ? "Box"
              : /Sold as a Pack/i.test(block)
                ? "Pack"
                : "";

          const unitPriceEl = item.querySelector(".unit-price");
          const priceMatch =
            unitPriceEl && unitPriceEl.textContent
              ? unitPriceEl.textContent.match(/R\s*([\d.]+)/)
              : null;
          const unitPrice = priceMatch ? priceMatch[1] : "";

          const desc = title.includes(" - ")
            ? title.split(" - ")[0].trim()
            : "";

          // Size = nominal diameter in mm
          const sizeMatch = diameter.match(/(\d+(?:\.\d+)?)\s*mm/i);
          const size = sizeMatch ? sizeMatch[1].replace(/\.0$/, "") : diameter;

          const lengthMatch = length.match(/(\d+(?:\.\d+)?)\s*mm/i);
          const lengthNum = lengthMatch
            ? lengthMatch[1].replace(/\.0$/, "")
            : "";

          // PosDescription pattern: "<size>x<length> DOWEL PIN <finish>"
          // (no Grade segment for dowel pins).
          const finishCode =
            finish.toLowerCase() === "plain"
              ? "PLN"
              : finish.toLowerCase() === "zinc plated"
                ? "ZP"
                : finish.toLowerCase() === "hot dip galvanised" ||
                    finish.toLowerCase() === "hot-dip galvanised" ||
                    finish.toLowerCase() === "hot dip galvanized" ||
                    finish.toLowerCase() === "hdg"
                  ? "HDG"
                  : finish
                    ? finish.toUpperCase()
                    : "PLN";

          const posDescription = lengthNum
            ? [size + "x" + lengthNum, "DOWEL PIN", finishCode]
                .filter(Boolean)
                .join(" ")
            : [size, "DOWEL PIN", finishCode].filter(Boolean).join(" ");

          return {
            sku: sku,
            posDescription: posDescription,
            description: desc,
            material: material,
            grade: grade,
            diameter: diameter,
            length: length,
            finish: finish,
            pack: pack,
            unitPrice: unitPrice ? "R " + unitPrice : "",
            url: productUrl,
          };
        });

        return { total: total, rows: rows };
      })();
    };
    return await page.evaluate(scrapeFn, url);
  };

  const first = await scrape(TARGET_URL);
  console.log(
    `Page 1: ${first.rows.length} items; toolbar reports total ${first.total}`,
  );

  const allRows: Array<Record<string, string>> = [...first.rows];
  const pages = Math.max(1, Math.ceil(first.total / 10));
  for (let p = 2; p <= pages; p++) {
    const sep = TARGET_URL.includes("?") ? "&" : "?";
    const url = `${TARGET_URL}${sep}p=${p}`;
    const out = await scrape(url);
    console.log(`Page ${p}/${pages}: ${out.rows.length} items`);
    allRows.push(...out.rows);
  }
  console.log(`TOTAL ROWS scraped: ${allRows.length}`);

  // Dedupe by SKU
  const seen = new Set<string>();
  const deduped = allRows.filter((r) => {
    const k = r.sku ?? "";
    if (!k) return false;
    if (seen.has(k)) return false;
    seen.add(k);
    return true;
  });
  console.log(`Unique rows after SKU dedupe: ${deduped.length}`);

  // Build CSV
  const headers = [
    "sku",
    "posDescription",
    "description",
    "material",
    "grade",
    "diameter",
    "length",
    "finish",
    "pack",
    "unitPrice",
    "url",
  ];
  const esc = (v: unknown) => {
    const s = (v ?? "").toString().replace(/"/g, '""');
    return /[",\n]/.test(s) ? `"${s}"` : s;
  };
  const csv = [headers.join(",")]
    .concat(
      deduped.map((r) =>
        headers.map((h) => esc((r as Record<string, unknown>)[h])).join(","),
      ),
    )
    .join("\n");

  writeFileSync(OUT_CSV, csv);
  console.log(
    `Wrote ${deduped.length} rows (${csv.length} bytes) to ${OUT_CSV}`,
  );

  await browser.close();
}

main().catch((err) => {
  console.error("FATAL", err);
  process.exit(1);
});