WFDASH-WEBAPP/scripts/_scrape-grease-nipples.ts

/**
 * Headless scraper for NSS grease nipples.
 *
 * The main /grease-nipples.html page is a CMS-grid of 5 subcategories
 * (180°, 45°, 90°, Button, Long). Each subcategory is a real Magento
 * category with paged `.product-item` cards. The script:
 *
 *   1. logs in via Playwright persistent profile,
 *   2. loads the landing page and extracts the 5 subcategory URLs,
 *   3. for each subcategory, paginates via in-page fetch() and parses
 *      every card's Diameter/Thread/Grade/Finish/Material/Head,
 *   4. writes a single deduped CSV to disk.
 *
 * Output rows:
 *   sku, posDescription, description, material, grade, diameter,
 *   thread, finish, pack, unitPrice, url
 *
 * PosDescription pattern: "<thread> GREASE NIPPLE <angle>" where
 * <angle> is derived from the subcategory (180/45/90/STRAIGHT).
 *
 * Usage:
 *   npx tsx scripts/_scrape-grease-nipples.ts
 */
import { chromium } from "@playwright/test";
import { writeFileSync } from "node:fs";

const TARGET_URL =
  "https://nssfasteners.co.za/grease-nipples-and-pipe-plugs/grease-nipples.html";
const LOGIN_URL = "https://nssfasteners.co.za/customer/account/login/";
const EMAIL = "jd@wfsales.co.za";
const PASSWORD = "WFsales01";
const PROFILE_DIR = "/tmp/nss-grease-nipples-profile";
const OUT_CSV = "scripts/nss-grease-nipples-pilot.csv";

/**
 * The five subcategories we expect to find on the landing page, in the
 * order the page lists them. We don't rely on finding them dynamically
 * (some labels can change) — but we DO verify each is present and
 * bail out hard if any is missing.
 */
const SUBCAT_EXPECTATIONS: Array<{
  slug: string;
  angle: string;
  label: string;
}> = [
  {
    slug: "grease-nipple-180-degree-din71412a",
    angle: "180",
    label: "Grease Nipple 180 degree DIN71412A",
  },
  {
    slug: "grease-nipple-45-degree-din71412b",
    angle: "45",
    label: "Grease Nipple 45 degree DIN71412B",
  },
  {
    slug: "grease-nipple-90-degree-din71412c",
    angle: "90",
    label: "Grease Nipple 90 degree DIN71412C",
  },
  {
    slug: "button-grease-nipple-din3404",
    angle: "BUTTON",
    label: "Button Grease Nipple DIN3404",
  },
  {
    slug: "long-grease-nipple-180-degree-din71412a",
    angle: "LONG",
    label: "Long Grease Nipple 180 degree DIN71412A",
  },
];

async function main(): Promise<void> {
  const browser = await chromium.launchPersistentContext(PROFILE_DIR, {
    headless: true,
    channel: "chromium",
    viewport: { width: 1280, height: 900 },
    userAgent:
      "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
  });
  const page = browser.pages()[0] ?? (await browser.newPage());

  // Polyfill esbuild's __name helper so arrow-function callbacks inside
  // page.evaluate work (tsx emits __name for class/object-name tracking,
  // but the helper isn't defined in the page's global scope).
  await page.addInitScript(() => {
    (globalThis as unknown as { __name: <T>(v: T) => T }).__name = (v: unknown) =>
      v as never;
  });

  // 1. Visit login page (skip if already logged in)
  await page.goto(LOGIN_URL, { waitUntil: "domcontentloaded", timeout: 60000 });
  if (page.url().includes("/login")) {
    console.log("On login page — filling credentials");
    await page.fill('input[name="login[username]"]', EMAIL);
    await page.fill('input[name="login[password]"]', PASSWORD);
    await Promise.all([
      page.waitForLoadState("networkidle", { timeout: 60000 }),
      page.click('button[name="send"]'),
    ]);
    console.log(`After login URL: ${page.url()}`);
  } else {
    console.log(`Already past login, on: ${page.url()}`);
  }

  // 2. Navigate to the landing page; discover the subcategory URLs.
  await page.goto(TARGET_URL, {
    waitUntil: "domcontentloaded",
    timeout: 60000,
  });
  console.log(`Loaded: ${page.url()}`);
  await page.waitForLoadState("networkidle", { timeout: 30000 });
  await page.waitForTimeout(2000);

  // Grab all hrefs in the page that look like our subcategories.
  const subcatUrls = await page.evaluate((slugs: string[]) => {
    const out: Array<{ slug: string; url: string }> = [];
    const anchors = Array.from(document.querySelectorAll("a[href]"));
    for (const slug of slugs) {
      const match = anchors.find((a) => {
        const href = a.getAttribute("href") || "";
        return href.includes("/" + slug + ".html");
      });
      if (match) {
        out.push({ slug, url: match.getAttribute("href") || "" });
      }
    }
    return out;
  }, SUBCAT_EXPECTATIONS.map((s) => s.slug));

  if (subcatUrls.length === 0) {
    throw new Error(
      "Found no subcategory URLs on the landing page — NSS markup likely changed",
    );
  }
  for (const s of subcatUrls) {
    console.log(`  subcategory: ${s.slug}  →  ${s.url}`);
  }
  const missing = SUBCAT_EXPECTATIONS.filter(
    (e) => !subcatUrls.some((u) => u.slug === e.slug),
  );
  if (missing.length > 0) {
    throw new Error(
      `Missing expected subcategories on landing page: ${missing.map((m) => m.slug).join(", ")}`,
    );
  }

  // 3. Filter sidebar (best-effort) on the landing page
  const filterInfo = await page.evaluate(() => {
    const out: Record<string, unknown> = {};
    const sidebar = document.querySelector(
      ".filter-options, .block-filter, aside, .sidebar",
    );
    if (sidebar) {
      const groups = sidebar.querySelectorAll(
        ".filter-options-item, .filter-content",
      );
      out.groupCount = groups.length;
      out.groups = Array.from(groups)
        .slice(0, 12)
        .map((g) => ({
          title:
            g.querySelector(".filter-options-title, h3, h4, .title")?.textContent?.trim() ??
            "",
          options: Array.from(
            g.querySelectorAll(
              ".filter-options-content a, .filter-options-content li, .item",
            ),
          )
            .slice(0, 12)
            .map(
              (o) =>
                o.textContent?.replace(/\s+/g, " ").trim().slice(0, 80) ?? "",
            ),
        }));
    }
    out.title = document.title;
    return out;
  });
  console.log("Landing-page filter sidebar:");
  console.log(JSON.stringify(filterInfo, null, 2));

  // 4. Walk every subcategory, paginate, parse, accumulate rows.
  const allRows: Array<Record<string, string>> = [];
  const subcategoryStats: Array<{
    slug: string;
    angle: string;
    rows: number;
    pages: number;
  }> = [];

  for (const sub of subcatUrls) {
    const expected = SUBCAT_EXPECTATIONS.find((e) => e.slug === sub.slug);
    if (!expected) continue; // shouldn't happen — guarded above
    const subUrl = sub.url.startsWith("http")
      ? sub.url
      : "https://nssfasteners.co.za" + sub.url;

    await page.goto(subUrl, {
      waitUntil: "domcontentloaded",
      timeout: 60000,
    });
    await page.waitForLoadState("networkidle", { timeout: 30000 });
    await page.waitForTimeout(2000);

    const scrape = async (url: string) => {
      const scrapeFn = function (params: { url: string; angle: string }) {
        const targetUrl = params.url;
        const angle = params.angle;
        return (async () => {
          const res = await fetch(targetUrl, { credentials: "include" });
          if (!res.ok) throw new Error(`HTTP ${res.status} for ${targetUrl}`);
          const html = await res.text();
          const doc = new DOMParser().parseFromString(html, "text/html");

          const amountEl = doc.querySelector(".toolbar-amount");
          const m =
            amountEl && amountEl.textContent
              ? amountEl.textContent.match(/of\s+(\d+)/i)
              : null;
          const total = m ? parseInt(m[1], 10) : 0;

          const items = Array.from(
            doc.querySelectorAll(
              ".item.product.product-item.product_addtocart_form",
            ),
          );
          const rows = items.map(function (item) {
            const titleEl = item.querySelector(
              ".items-center.justify-center.text-primary",
            );
            const title =
              titleEl && titleEl.textContent
                ? titleEl.textContent.trim()
                : "";
            const sku = title.includes(" - ")
              ? title.split(" - ").pop().trim()
              : "";
            const linkEl = item.querySelector(
              "a.product-item-photo, a.product.photo",
            );
            const productUrl = linkEl ? linkEl.getAttribute("href") || "" : "";

            // Specs block: card body has "Imperial\nStainless Steel\nDiameter: 1/4\n..."
            // We grep the whole item's text content and parse each field.
            const block = (item.textContent || "").replace(/\s+/g, " ");

            const get = function (label: string): string {
              const re = new RegExp(
                label +
                  "\\s*:?\\s*([^\\s].*?)(?=\\s+(?:Diameter|Thread|Grade|Finish|Material|Head|Sold|Imperial|Metric|Hi-Tensile|Stainless|Mild|Brass|$))",
                "i",
              );
              const mm = block.match(re);
              return mm ? mm[1].trim() : "";
            };

            const material = /Stainless/i.test(block)
              ? "Stainless Steel"
              : /Mild\s*Steel/i.test(block)
                ? "Mild Steel"
                : /Hi-Tensile/i.test(block)
                  ? "Hi-Tensile"
                  : /Brass/i.test(block)
                    ? "Brass"
                    : "";

            const diameter = get("Diameter");
            const thread = get("Thread");
            const grade = get("Grade");
            const finish = get("Finish");

            const pack = /Sold as a Single Piece/i.test(block)
              ? "Single Piece"
              : /Sold as a Box/i.test(block)
                ? "Box"
                : /Sold as a Pack/i.test(block)
                  ? "Pack"
                  : "";

            // Price: hidden unit-price div carries the number even when qty=0.
            const unitPriceEl = item.querySelector(".unit-price");
            const priceMatch =
              unitPriceEl && unitPriceEl.textContent
                ? unitPriceEl.textContent.match(/R\s*([\d.]+)/)
                : null;
            const unitPrice = priceMatch ? priceMatch[1] : "";

            const desc = title.includes(" - ")
              ? title.split(" - ")[0].trim()
              : "";

            // The thread value carries the size info we need for the
            // PosDescription. For BSP imperial items it's like "BSP"
            // and Diameter holds "1/4". For metric items Thread is
            // "M6" / "M8" etc. We compose: "<thread> <diameter>"
            // when BSP, else just "<thread>".
            const size =
              /BSP/i.test(thread) && diameter
                ? `${thread} ${diameter}`
                : thread;

            const posDescription = [size, "GREASE NIPPLE", angle]
              .filter(Boolean)
              .join(" ");

            return {
              sku: sku,
              posDescription: posDescription,
              description: desc,
              material: material,
              grade: grade,
              diameter: diameter,
              thread: thread,
              finish: finish,
              pack: pack,
              unitPrice: unitPrice ? "R " + unitPrice : "",
              url: productUrl,
            };
          });

          return { total: total, rows: rows };
        })();
      };
      return await page.evaluate(scrapeFn, { url, angle: expected.angle });
    };

    const first = await scrape(subUrl);
    console.log(
      `[${expected.angle}] page 1: ${first.rows.length} items; toolbar reports total ${first.total}`,
    );

    const subRows: Array<Record<string, string>> = [...first.rows];
    const pages = Math.max(1, Math.ceil(first.total / 10));
    for (let p = 2; p <= pages; p++) {
      const sep = subUrl.includes("?") ? "&" : "?";
      const url = `${subUrl}${sep}p=${p}`;
      const out = await scrape(url);
      console.log(
        `[${expected.angle}] page ${p}/${pages}: ${out.rows.length} items`,
      );
      subRows.push(...out.rows);
    }
    subcategoryStats.push({
      slug: expected.slug,
      angle: expected.angle,
      rows: subRows.length,
      pages,
    });
    allRows.push(...subRows);
  }

  console.log(`TOTAL ROWS scraped (pre-dedupe): ${allRows.length}`);
  console.log(`Per-subcategory:`);
  for (const s of subcategoryStats) {
    console.log(`  ${s.angle.padEnd(8)} ${s.slug}  →  ${s.rows} rows across ${s.pages} page(s)`);
  }

  // Dedupe by SKU
  const seen = new Set<string>();
  const deduped = allRows.filter((r) => {
    const k = r.sku ?? "";
    if (!k) return false;
    if (seen.has(k)) return false;
    seen.add(k);
    return true;
  });
  console.log(`Unique rows after SKU dedupe: ${deduped.length}`);

  // Build CSV
  const headers = [
    "sku",
    "posDescription",
    "description",
    "material",
    "grade",
    "diameter",
    "thread",
    "finish",
    "pack",
    "unitPrice",
    "url",
  ];
  const esc = (v: unknown) => {
    const s = (v ?? "").toString().replace(/"/g, '""');
    return /[",\n]/.test(s) ? `"${s}"` : s;
  };
  const csv = [headers.join(",")]
    .concat(
      deduped.map((r) =>
        headers.map((h) => esc((r as Record<string, unknown>)[h])).join(","),
      ),
    )
    .join("\n");

  writeFileSync(OUT_CSV, csv);
  console.log(
    `Wrote ${deduped.length} rows (${csv.length} bytes) to ${OUT_CSV}`,
  );

  await browser.close();
}

main().catch((err) => {
  console.error("FATAL", err);
  process.exit(1);
});