Dave
d8e8714720
- Enhanced `nss-fasteners-pb-import.json` to compute `selling_price` as `cost_price * (1 + 80%)` and included `markup` in the records. - Updated `nss-fasteners-pb-transform.ts` to allow configurable CSV and output file names via environment variables, improving flexibility for different product runs. - Added support for uploading photos in the repair detail page, including validation for file types and sizes, and implemented a dialog for creating customer quotes linked to repairs. - Introduced a new `customer_quote` field in the `Repair` type to link generated quotes back to their respective repairs. These changes streamline the pricing process and enhance the user experience in managing repairs and quotes.
407 lines
No EOL
13 KiB
TypeScript
407 lines
No EOL
13 KiB
TypeScript
/**
|
|
* Headless scraper for NSS grease nipples.
|
|
*
|
|
* The main /grease-nipples.html page is a CMS-grid of 5 subcategories
|
|
* (180°, 45°, 90°, Button, Long). Each subcategory is a real Magento
|
|
* category with paged `.product-item` cards. The script:
|
|
*
|
|
* 1. logs in via Playwright persistent profile,
|
|
* 2. loads the landing page and extracts the 5 subcategory URLs,
|
|
* 3. for each subcategory, paginates via in-page fetch() and parses
|
|
* every card's Diameter/Thread/Grade/Finish/Material/Head,
|
|
* 4. writes a single deduped CSV to disk.
|
|
*
|
|
* Output rows:
|
|
* sku, posDescription, description, material, grade, diameter,
|
|
* thread, finish, pack, unitPrice, url
|
|
*
|
|
* PosDescription pattern: "<thread> GREASE NIPPLE <angle>" where
|
|
* <angle> is derived from the subcategory (180/45/90/STRAIGHT).
|
|
*
|
|
* Usage:
|
|
* npx tsx scripts/_scrape-grease-nipples.ts
|
|
*/
|
|
import { chromium } from "@playwright/test";
|
|
import { writeFileSync } from "node:fs";
|
|
|
|
const TARGET_URL =
|
|
"https://nssfasteners.co.za/grease-nipples-and-pipe-plugs/grease-nipples.html";
|
|
const LOGIN_URL = "https://nssfasteners.co.za/customer/account/login/";
|
|
const EMAIL = "jd@wfsales.co.za";
|
|
const PASSWORD = "WFsales01";
|
|
const PROFILE_DIR = "/tmp/nss-grease-nipples-profile";
|
|
const OUT_CSV = "scripts/nss-grease-nipples-pilot.csv";
|
|
|
|
/**
|
|
* The five subcategories we expect to find on the landing page, in the
|
|
* order the page lists them. We don't rely on finding them dynamically
|
|
* (some labels can change) — but we DO verify each is present and
|
|
* bail out hard if any is missing.
|
|
*/
|
|
const SUBCAT_EXPECTATIONS: Array<{
|
|
slug: string;
|
|
angle: string;
|
|
label: string;
|
|
}> = [
|
|
{
|
|
slug: "grease-nipple-180-degree-din71412a",
|
|
angle: "180",
|
|
label: "Grease Nipple 180 degree DIN71412A",
|
|
},
|
|
{
|
|
slug: "grease-nipple-45-degree-din71412b",
|
|
angle: "45",
|
|
label: "Grease Nipple 45 degree DIN71412B",
|
|
},
|
|
{
|
|
slug: "grease-nipple-90-degree-din71412c",
|
|
angle: "90",
|
|
label: "Grease Nipple 90 degree DIN71412C",
|
|
},
|
|
{
|
|
slug: "button-grease-nipple-din3404",
|
|
angle: "BUTTON",
|
|
label: "Button Grease Nipple DIN3404",
|
|
},
|
|
{
|
|
slug: "long-grease-nipple-180-degree-din71412a",
|
|
angle: "LONG",
|
|
label: "Long Grease Nipple 180 degree DIN71412A",
|
|
},
|
|
];
|
|
|
|
async function main(): Promise<void> {
|
|
const browser = await chromium.launchPersistentContext(PROFILE_DIR, {
|
|
headless: true,
|
|
channel: "chromium",
|
|
viewport: { width: 1280, height: 900 },
|
|
userAgent:
|
|
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
|
|
});
|
|
const page = browser.pages()[0] ?? (await browser.newPage());
|
|
|
|
// Polyfill esbuild's __name helper so arrow-function callbacks inside
|
|
// page.evaluate work (tsx emits __name for class/object-name tracking,
|
|
// but the helper isn't defined in the page's global scope).
|
|
await page.addInitScript(() => {
|
|
(globalThis as unknown as { __name: <T>(v: T) => T }).__name = (v: unknown) =>
|
|
v as never;
|
|
});
|
|
|
|
// 1. Visit login page (skip if already logged in)
|
|
await page.goto(LOGIN_URL, { waitUntil: "domcontentloaded", timeout: 60000 });
|
|
if (page.url().includes("/login")) {
|
|
console.log("On login page — filling credentials");
|
|
await page.fill('input[name="login[username]"]', EMAIL);
|
|
await page.fill('input[name="login[password]"]', PASSWORD);
|
|
await Promise.all([
|
|
page.waitForLoadState("networkidle", { timeout: 60000 }),
|
|
page.click('button[name="send"]'),
|
|
]);
|
|
console.log(`After login URL: ${page.url()}`);
|
|
} else {
|
|
console.log(`Already past login, on: ${page.url()}`);
|
|
}
|
|
|
|
// 2. Navigate to the landing page; discover the subcategory URLs.
|
|
await page.goto(TARGET_URL, {
|
|
waitUntil: "domcontentloaded",
|
|
timeout: 60000,
|
|
});
|
|
console.log(`Loaded: ${page.url()}`);
|
|
await page.waitForLoadState("networkidle", { timeout: 30000 });
|
|
await page.waitForTimeout(2000);
|
|
|
|
// Grab all hrefs in the page that look like our subcategories.
|
|
const subcatUrls = await page.evaluate((slugs: string[]) => {
|
|
const out: Array<{ slug: string; url: string }> = [];
|
|
const anchors = Array.from(document.querySelectorAll("a[href]"));
|
|
for (const slug of slugs) {
|
|
const match = anchors.find((a) => {
|
|
const href = a.getAttribute("href") || "";
|
|
return href.includes("/" + slug + ".html");
|
|
});
|
|
if (match) {
|
|
out.push({ slug, url: match.getAttribute("href") || "" });
|
|
}
|
|
}
|
|
return out;
|
|
}, SUBCAT_EXPECTATIONS.map((s) => s.slug));
|
|
|
|
if (subcatUrls.length === 0) {
|
|
throw new Error(
|
|
"Found no subcategory URLs on the landing page — NSS markup likely changed",
|
|
);
|
|
}
|
|
for (const s of subcatUrls) {
|
|
console.log(` subcategory: ${s.slug} → ${s.url}`);
|
|
}
|
|
const missing = SUBCAT_EXPECTATIONS.filter(
|
|
(e) => !subcatUrls.some((u) => u.slug === e.slug),
|
|
);
|
|
if (missing.length > 0) {
|
|
throw new Error(
|
|
`Missing expected subcategories on landing page: ${missing.map((m) => m.slug).join(", ")}`,
|
|
);
|
|
}
|
|
|
|
// 3. Filter sidebar (best-effort) on the landing page
|
|
const filterInfo = await page.evaluate(() => {
|
|
const out: Record<string, unknown> = {};
|
|
const sidebar = document.querySelector(
|
|
".filter-options, .block-filter, aside, .sidebar",
|
|
);
|
|
if (sidebar) {
|
|
const groups = sidebar.querySelectorAll(
|
|
".filter-options-item, .filter-content",
|
|
);
|
|
out.groupCount = groups.length;
|
|
out.groups = Array.from(groups)
|
|
.slice(0, 12)
|
|
.map((g) => ({
|
|
title:
|
|
g.querySelector(".filter-options-title, h3, h4, .title")?.textContent?.trim() ??
|
|
"",
|
|
options: Array.from(
|
|
g.querySelectorAll(
|
|
".filter-options-content a, .filter-options-content li, .item",
|
|
),
|
|
)
|
|
.slice(0, 12)
|
|
.map(
|
|
(o) =>
|
|
o.textContent?.replace(/\s+/g, " ").trim().slice(0, 80) ?? "",
|
|
),
|
|
}));
|
|
}
|
|
out.title = document.title;
|
|
return out;
|
|
});
|
|
console.log("Landing-page filter sidebar:");
|
|
console.log(JSON.stringify(filterInfo, null, 2));
|
|
|
|
// 4. Walk every subcategory, paginate, parse, accumulate rows.
|
|
const allRows: Array<Record<string, string>> = [];
|
|
const subcategoryStats: Array<{
|
|
slug: string;
|
|
angle: string;
|
|
rows: number;
|
|
pages: number;
|
|
}> = [];
|
|
|
|
for (const sub of subcatUrls) {
|
|
const expected = SUBCAT_EXPECTATIONS.find((e) => e.slug === sub.slug);
|
|
if (!expected) continue; // shouldn't happen — guarded above
|
|
const subUrl = sub.url.startsWith("http")
|
|
? sub.url
|
|
: "https://nssfasteners.co.za" + sub.url;
|
|
|
|
await page.goto(subUrl, {
|
|
waitUntil: "domcontentloaded",
|
|
timeout: 60000,
|
|
});
|
|
await page.waitForLoadState("networkidle", { timeout: 30000 });
|
|
await page.waitForTimeout(2000);
|
|
|
|
const scrape = async (url: string) => {
|
|
const scrapeFn = function (params: { url: string; angle: string }) {
|
|
const targetUrl = params.url;
|
|
const angle = params.angle;
|
|
return (async () => {
|
|
const res = await fetch(targetUrl, { credentials: "include" });
|
|
if (!res.ok) throw new Error(`HTTP ${res.status} for ${targetUrl}`);
|
|
const html = await res.text();
|
|
const doc = new DOMParser().parseFromString(html, "text/html");
|
|
|
|
const amountEl = doc.querySelector(".toolbar-amount");
|
|
const m =
|
|
amountEl && amountEl.textContent
|
|
? amountEl.textContent.match(/of\s+(\d+)/i)
|
|
: null;
|
|
const total = m ? parseInt(m[1], 10) : 0;
|
|
|
|
const items = Array.from(
|
|
doc.querySelectorAll(
|
|
".item.product.product-item.product_addtocart_form",
|
|
),
|
|
);
|
|
const rows = items.map(function (item) {
|
|
const titleEl = item.querySelector(
|
|
".items-center.justify-center.text-primary",
|
|
);
|
|
const title =
|
|
titleEl && titleEl.textContent
|
|
? titleEl.textContent.trim()
|
|
: "";
|
|
const sku = title.includes(" - ")
|
|
? title.split(" - ").pop().trim()
|
|
: "";
|
|
const linkEl = item.querySelector(
|
|
"a.product-item-photo, a.product.photo",
|
|
);
|
|
const productUrl = linkEl ? linkEl.getAttribute("href") || "" : "";
|
|
|
|
// Specs block: card body has "Imperial\nStainless Steel\nDiameter: 1/4\n..."
|
|
// We grep the whole item's text content and parse each field.
|
|
const block = (item.textContent || "").replace(/\s+/g, " ");
|
|
|
|
const get = function (label: string): string {
|
|
const re = new RegExp(
|
|
label +
|
|
"\\s*:?\\s*([^\\s].*?)(?=\\s+(?:Diameter|Thread|Grade|Finish|Material|Head|Sold|Imperial|Metric|Hi-Tensile|Stainless|Mild|Brass|$))",
|
|
"i",
|
|
);
|
|
const mm = block.match(re);
|
|
return mm ? mm[1].trim() : "";
|
|
};
|
|
|
|
const material = /Stainless/i.test(block)
|
|
? "Stainless Steel"
|
|
: /Mild\s*Steel/i.test(block)
|
|
? "Mild Steel"
|
|
: /Hi-Tensile/i.test(block)
|
|
? "Hi-Tensile"
|
|
: /Brass/i.test(block)
|
|
? "Brass"
|
|
: "";
|
|
|
|
const diameter = get("Diameter");
|
|
const thread = get("Thread");
|
|
const grade = get("Grade");
|
|
const finish = get("Finish");
|
|
|
|
const pack = /Sold as a Single Piece/i.test(block)
|
|
? "Single Piece"
|
|
: /Sold as a Box/i.test(block)
|
|
? "Box"
|
|
: /Sold as a Pack/i.test(block)
|
|
? "Pack"
|
|
: "";
|
|
|
|
// Price: hidden unit-price div carries the number even when qty=0.
|
|
const unitPriceEl = item.querySelector(".unit-price");
|
|
const priceMatch =
|
|
unitPriceEl && unitPriceEl.textContent
|
|
? unitPriceEl.textContent.match(/R\s*([\d.]+)/)
|
|
: null;
|
|
const unitPrice = priceMatch ? priceMatch[1] : "";
|
|
|
|
const desc = title.includes(" - ")
|
|
? title.split(" - ")[0].trim()
|
|
: "";
|
|
|
|
// The thread value carries the size info we need for the
|
|
// PosDescription. For BSP imperial items it's like "BSP"
|
|
// and Diameter holds "1/4". For metric items Thread is
|
|
// "M6" / "M8" etc. We compose: "<thread> <diameter>"
|
|
// when BSP, else just "<thread>".
|
|
const size =
|
|
/BSP/i.test(thread) && diameter
|
|
? `${thread} ${diameter}`
|
|
: thread;
|
|
|
|
const posDescription = [size, "GREASE NIPPLE", angle]
|
|
.filter(Boolean)
|
|
.join(" ");
|
|
|
|
return {
|
|
sku: sku,
|
|
posDescription: posDescription,
|
|
description: desc,
|
|
material: material,
|
|
grade: grade,
|
|
diameter: diameter,
|
|
thread: thread,
|
|
finish: finish,
|
|
pack: pack,
|
|
unitPrice: unitPrice ? "R " + unitPrice : "",
|
|
url: productUrl,
|
|
};
|
|
});
|
|
|
|
return { total: total, rows: rows };
|
|
})();
|
|
};
|
|
return await page.evaluate(scrapeFn, { url, angle: expected.angle });
|
|
};
|
|
|
|
const first = await scrape(subUrl);
|
|
console.log(
|
|
`[${expected.angle}] page 1: ${first.rows.length} items; toolbar reports total ${first.total}`,
|
|
);
|
|
|
|
const subRows: Array<Record<string, string>> = [...first.rows];
|
|
const pages = Math.max(1, Math.ceil(first.total / 10));
|
|
for (let p = 2; p <= pages; p++) {
|
|
const sep = subUrl.includes("?") ? "&" : "?";
|
|
const url = `${subUrl}${sep}p=${p}`;
|
|
const out = await scrape(url);
|
|
console.log(
|
|
`[${expected.angle}] page ${p}/${pages}: ${out.rows.length} items`,
|
|
);
|
|
subRows.push(...out.rows);
|
|
}
|
|
subcategoryStats.push({
|
|
slug: expected.slug,
|
|
angle: expected.angle,
|
|
rows: subRows.length,
|
|
pages,
|
|
});
|
|
allRows.push(...subRows);
|
|
}
|
|
|
|
console.log(`TOTAL ROWS scraped (pre-dedupe): ${allRows.length}`);
|
|
console.log(`Per-subcategory:`);
|
|
for (const s of subcategoryStats) {
|
|
console.log(` ${s.angle.padEnd(8)} ${s.slug} → ${s.rows} rows across ${s.pages} page(s)`);
|
|
}
|
|
|
|
// Dedupe by SKU
|
|
const seen = new Set<string>();
|
|
const deduped = allRows.filter((r) => {
|
|
const k = r.sku ?? "";
|
|
if (!k) return false;
|
|
if (seen.has(k)) return false;
|
|
seen.add(k);
|
|
return true;
|
|
});
|
|
console.log(`Unique rows after SKU dedupe: ${deduped.length}`);
|
|
|
|
// Build CSV
|
|
const headers = [
|
|
"sku",
|
|
"posDescription",
|
|
"description",
|
|
"material",
|
|
"grade",
|
|
"diameter",
|
|
"thread",
|
|
"finish",
|
|
"pack",
|
|
"unitPrice",
|
|
"url",
|
|
];
|
|
const esc = (v: unknown) => {
|
|
const s = (v ?? "").toString().replace(/"/g, '""');
|
|
return /[",\n]/.test(s) ? `"${s}"` : s;
|
|
};
|
|
const csv = [headers.join(",")]
|
|
.concat(
|
|
deduped.map((r) =>
|
|
headers.map((h) => esc((r as Record<string, unknown>)[h])).join(","),
|
|
),
|
|
)
|
|
.join("\n");
|
|
|
|
writeFileSync(OUT_CSV, csv);
|
|
console.log(
|
|
`Wrote ${deduped.length} rows (${csv.length} bytes) to ${OUT_CSV}`,
|
|
);
|
|
|
|
await browser.close();
|
|
}
|
|
|
|
main().catch((err) => {
|
|
console.error("FATAL", err);
|
|
process.exit(1);
|
|
}); |