797 lines
28 KiB
JavaScript
797 lines
28 KiB
JavaScript
|
/**
|
||
|
|
* One-time bulk pass: download every attachment from the three
|
||
|
|
* configured PB collections/fields, compress it (PDF → /ebook,
|
||
|
|
* JPG/PNG → 1200px q=70), and re-upload it to the same record
|
||
|
|
* under the same filename.
|
||
|
|
*
|
||
|
|
* Why a one-time script (not an API route) for the first pass:
|
||
|
|
* - We want a single human-driven command that runs to
|
||
|
|
* completion and produces a verifiable report. No cron,
|
||
|
|
* no auth flow, no Next.js context.
|
||
|
|
* - We DON'T yet have the `attachment_compression_log` PB
|
||
|
|
* collection — that belongs to the recurring weekly agent,
|
||
|
|
* where it powers hash-based dedup. A one-time pass just
|
||
|
|
* processes everything once and writes its report to disk.
|
||
|
|
*
|
||
|
|
* Re-uploading under the same filename:
|
||
|
|
* PB file fields are arrays of filenames. Posting FormData
|
||
|
|
* with a file whose name matches an existing entry REPLACES
|
||
|
|
* that entry in place — the file's disk path changes but the
|
||
|
|
* filename + URL stay stable, so every existing download
|
||
|
|
* link / email attachment / audit_log reference keeps
|
||
|
|
* working.
|
||
|
|
*
|
||
|
|
* Skip rules (from the brainstorming session):
|
||
|
|
* - file < 50 KB (noise)
|
||
|
|
* - file > 24 MB (PB hard cap is 25 MB; leave headroom)
|
||
|
|
* - extension not in {pdf, jpg, jpeg, png}
|
||
|
|
* - already-compressed size ≥ original (no-op re-upload)
|
||
|
|
* - compression tool failed / timed out
|
||
|
|
*
|
||
|
|
* Re-upload safety:
|
||
|
|
* PB's FormData update REPLACES the file field array. For
|
||
|
|
* a single-file replacement we re-upload the whole array
|
||
|
|
* with the one matching entry replaced, so sibling files
|
||
|
|
* on the same record are untouched. See the stock-purchases
|
||
|
|
* service for the same pattern (`additional_attachments`).
|
||
|
|
*
|
||
|
|
* URL-stability caveat (PB dedupe-suffix behavior):
|
||
|
|
* PB stores each file on disk as `{recordId}_{filename}`. When
|
||
|
|
* we re-upload a file with the same filename and that exact
|
||
|
|
* on-disk path already exists, PB DOES NOT overwrite in place
|
||
|
|
* — it appends an 8-11 char dedupe suffix to the new file.
|
||
|
|
* The field's stored filename changes, so every URL built
|
||
|
|
* from `record.file[i]` is invalidated for that record.
|
||
|
|
* `reuploadFile()` detects this as `tokenChanged` (in addition
|
||
|
|
* to `lengthChanged`) and the report surfaces it as a warning.
|
||
|
|
* For collections where URL stability matters (anything with
|
||
|
|
* emailed links, external bookmarks, or PII tracking), test
|
||
|
|
* on a single record first with `--record=<id>` before doing
|
||
|
|
* a bulk run. See reports/attachment-compression-2026-09-26.md
|
||
|
|
* for the live example (1,420 quote_attachments URLs changed).
|
||
|
|
*
|
||
|
|
* Run:
|
||
|
|
* npx tsx scripts/compress-attachments-once.mjs
|
||
|
|
* # or
|
||
|
|
* npx tsx scripts/compress-attachments-once.mjs --dry-run
|
||
|
|
* # limit for testing:
|
||
|
|
* npx tsx scripts/compress-attachments-once.mjs --limit 20
|
||
|
|
* # scope to one source for first-pass rollout:
|
||
|
|
* npx tsx scripts/compress-attachments-once.mjs --dry-run --source=quote_attachments
|
||
|
|
* # comma-separated, multiple sources:
|
||
|
|
* npx tsx scripts/compress-attachments-once.mjs --source=quote_attachments,purchase_orders.documents
|
||
|
|
* # scope to a single record (sanity-check URL stability first):
|
||
|
|
* npx tsx scripts/compress-attachments-once.mjs --dry-run --record=8ay88lwwjiwwggc
|
||
|
|
*
|
||
|
|
* Known source labels (must match one of these exactly):
|
||
|
|
* - quote_attachments
|
||
|
|
* - order_item_attachements
|
||
|
|
* - purchase_orders.documents
|
||
|
|
*
|
||
|
|
* Output:
|
||
|
|
* - stdout: per-collection progress + grand total
|
||
|
|
* - reports/attachment-compression-<date>.json: full machine-
|
||
|
|
* readable report (every file processed)
|
||
|
|
* - reports/attachment-compression-<date>.md: human-readable
|
||
|
|
* summary table
|
||
|
|
*/
|
||
|
|
|
||
|
|
// Inline .env/.env.local loader (same pattern as scripts/_env.ts,
|
||
|
|
// kept self-contained so this script can run as a plain node ESM
|
||
|
|
// without relying on a sibling .ts file).
|
||
|
|
import { readFileSync } from "node:fs";
|
||
|
|
import { resolve } from "node:path";
|
||
|
|
for (const name of [".env", ".env.local"]) {
|
||
|
|
try {
|
||
|
|
const raw = readFileSync(resolve(process.cwd(), name), "utf8");
|
||
|
|
for (const line of raw.split("\n")) {
|
||
|
|
const trimmed = line.trim();
|
||
|
|
if (!trimmed || trimmed.startsWith("#")) continue;
|
||
|
|
const m = trimmed.match(/^([A-Z0-9_]+)\s*=\s*(.+)$/i);
|
||
|
|
if (m && !process.env[m[1]]) {
|
||
|
|
process.env[m[1]] = m[2].replace(/^["']|["']$/g, "").trim();
|
||
|
|
}
|
||
|
|
}
|
||
|
|
} catch {}
|
||
|
|
}
|
||
|
|
|
||
|
|
import PocketBase from "pocketbase";
|
||
|
|
import { createHash } from "node:crypto";
|
||
|
|
import { createReadStream } from "node:fs";
|
||
|
|
import { mkdir, writeFile, stat, unlink, readFile } from "node:fs/promises";
|
||
|
|
import { tmpdir } from "node:os";
|
||
|
|
import { join } from "node:path";
|
||
|
|
import { spawn } from "node:child_process";
|
||
|
|
|
||
|
|
const PB_URL =
|
||
|
|
process.env.NEXT_PUBLIC_POCKETBASE_URL || "http://127.0.0.1:8090";
|
||
|
|
const ADMIN_EMAIL = process.env.PB_ADMIN_EMAIL;
|
||
|
|
const ADMIN_PASSWORD = process.env.PB_ADMIN_PASSWORD;
|
||
|
|
|
||
|
|
const DRY_RUN = process.argv.includes("--dry-run");
|
||
|
|
const LIMIT = parseLimit(process.argv);
|
||
|
|
const MAX_PARALLEL = parseIntEnv("COMPRESS_PARALLEL", "4");
|
||
|
|
|
||
|
|
const SKIP_MIN_BYTES = 50 * 1024; // 50 KB
|
||
|
|
const SKIP_MAX_BYTES = 24 * 1024 * 1024; // 24 MB (PB cap is 25)
|
||
|
|
const SUPPORTED_EXT = new Set(["pdf", "jpg", "jpeg", "png"]);
|
||
|
|
const JPG_MAX_WIDTH = 1200;
|
||
|
|
const JPG_QUALITY = 70;
|
||
|
|
const GS_TIMEOUT_MS = 60_000;
|
||
|
|
const SIPS_TIMEOUT_MS = 30_000;
|
||
|
|
|
||
|
|
const SOURCES = [
|
||
|
|
{
|
||
|
|
label: "order_item_attachements",
|
||
|
|
collection: "order_items",
|
||
|
|
field: "order_item_attachements",
|
||
|
|
},
|
||
|
|
{
|
||
|
|
label: "quote_attachments",
|
||
|
|
collection: "quote_attachments",
|
||
|
|
field: "file",
|
||
|
|
},
|
||
|
|
{
|
||
|
|
label: "purchase_orders.documents",
|
||
|
|
collection: "purchase_orders",
|
||
|
|
field: "documents",
|
||
|
|
},
|
||
|
|
];
|
||
|
|
|
||
|
|
function parseLimit(argv) {
|
||
|
|
const i = argv.indexOf("--limit");
|
||
|
|
if (i === -1) return Infinity;
|
||
|
|
const n = parseInt(argv[i + 1], 10);
|
||
|
|
return Number.isFinite(n) && n > 0 ? n : Infinity;
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* --record=<id>
|
||
|
|
*
|
||
|
|
* Scope the run to exactly one PB record (per source). Use with
|
||
|
|
* `--dry-run` first to sanity-check URL stability on a single
|
||
|
|
* record before doing a bulk run on a collection where stable
|
||
|
|
* URLs matter (emailed links, bookmarks).
|
||
|
|
*
|
||
|
|
* Accepts both forms:
|
||
|
|
* --record=8ay88lwwjiwwggc (equals form)
|
||
|
|
* --record 8ay88lwwjiwwggc (space form)
|
||
|
|
*/
|
||
|
|
function parseRecordId(argv) {
|
||
|
|
for (let i = 0; i < argv.length; i++) {
|
||
|
|
const a = argv[i];
|
||
|
|
if (a === "--record") {
|
||
|
|
const next = argv[i + 1];
|
||
|
|
if (!next || next.startsWith("--")) {
|
||
|
|
console.error(
|
||
|
|
"--record requires a value (e.g. --record=8ay88lwwjiwwggc)",
|
||
|
|
);
|
||
|
|
process.exit(1);
|
||
|
|
}
|
||
|
|
return next;
|
||
|
|
} else if (a.startsWith("--record=")) {
|
||
|
|
const v = a.slice("--record=".length).trim();
|
||
|
|
if (!v) {
|
||
|
|
console.error(
|
||
|
|
"--record requires a value (e.g. --record=8ay88lwwjiwwggc)",
|
||
|
|
);
|
||
|
|
process.exit(1);
|
||
|
|
}
|
||
|
|
return v;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return null;
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* --source=<label[,label...]>
|
||
|
|
*
|
||
|
|
* Scope the run to a subset of the configured SOURCES. Strict match
|
||
|
|
* against the labels below — typos exit(1) loudly rather than
|
||
|
|
* silently processing zero files.
|
||
|
|
*
|
||
|
|
* Accepts both forms:
|
||
|
|
* --source=quote_attachments (equals form)
|
||
|
|
* --source quote_attachments (space form)
|
||
|
|
* --source=quote_attachments,order_item_attachements
|
||
|
|
*/
|
||
|
|
function parseSources(argv) {
|
||
|
|
const wanted = [];
|
||
|
|
for (let i = 0; i < argv.length; i++) {
|
||
|
|
const a = argv[i];
|
||
|
|
if (a === "--source") {
|
||
|
|
const next = argv[i + 1];
|
||
|
|
if (!next || next.startsWith("--")) {
|
||
|
|
console.error(
|
||
|
|
"--source requires a value (e.g. --source=quote_attachments)",
|
||
|
|
);
|
||
|
|
process.exit(1);
|
||
|
|
}
|
||
|
|
wanted.push(
|
||
|
|
...next.split(",").map((s) => s.trim()).filter(Boolean),
|
||
|
|
);
|
||
|
|
} else if (a.startsWith("--source=")) {
|
||
|
|
wanted.push(
|
||
|
|
...a
|
||
|
|
.slice("--source=".length)
|
||
|
|
.split(",")
|
||
|
|
.map((s) => s.trim())
|
||
|
|
.filter(Boolean),
|
||
|
|
);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if (wanted.length === 0) return SOURCES;
|
||
|
|
const known = SOURCES.map((s) => s.label);
|
||
|
|
const unknown = wanted.filter((w) => !known.includes(w));
|
||
|
|
if (unknown.length > 0) {
|
||
|
|
console.error(
|
||
|
|
`--source: unknown value(s): ${unknown.join(", ")}. Available: ${known.join(", ")}`,
|
||
|
|
);
|
||
|
|
process.exit(1);
|
||
|
|
}
|
||
|
|
// Preserve the canonical order of SOURCES so reports stay stable
|
||
|
|
// even when the user lists them in a different order.
|
||
|
|
return SOURCES.filter((s) => wanted.includes(s.label));
|
||
|
|
}
|
||
|
|
|
||
|
|
function parseIntEnv(name, fallback) {
|
||
|
|
const v = parseInt(process.env[name] || "", 10);
|
||
|
|
return Number.isFinite(v) && v > 0 ? v : fallback;
|
||
|
|
}
|
||
|
|
|
||
|
|
function todayStamp() {
|
||
|
|
return new Date().toISOString().slice(0, 10);
|
||
|
|
}
|
||
|
|
|
||
|
|
function ensureDir(p) {
|
||
|
|
return mkdir(p, { recursive: true });
|
||
|
|
}
|
||
|
|
|
||
|
|
function bytes(n) {
|
||
|
|
if (n == null) return "?";
|
||
|
|
if (n < 1024) return `${n} B`;
|
||
|
|
if (n < 1024 * 1024) return `${(n / 1024).toFixed(1)} KB`;
|
||
|
|
return `${(n / (1024 * 1024)).toFixed(2)} MB`;
|
||
|
|
}
|
||
|
|
|
||
|
|
function extOf(name) {
|
||
|
|
const dot = name.lastIndexOf(".");
|
||
|
|
return dot === -1 ? "" : name.slice(dot + 1).toLowerCase();
|
||
|
|
}
|
||
|
|
|
||
|
|
function baseNameOf(name) {
|
||
|
|
const slash = Math.max(name.lastIndexOf("/"), name.lastIndexOf("\\"));
|
||
|
|
return slash === -1 ? name : name.slice(slash + 1);
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* PB stores each uploaded file on disk as `{recordId}_{fieldValue}`,
|
||
|
|
* and the field's stored value is `<originalName>_<randomSuffix>.<ext>`
|
||
|
|
* — e.g. `image001_g8etl02zu0.png`. The token is the full field
|
||
|
|
* value (with the random suffix) and that exact string is what
|
||
|
|
* appears in the URL `/api/files/{collection}/{recordId}/{token}`.
|
||
|
|
*
|
||
|
|
* For the compression agent this matters because:
|
||
|
|
* - downloads must use the FULL token (with suffix), not a stripped
|
||
|
|
* version, or PB returns 404
|
||
|
|
* - re-uploads must use the SAME token (with suffix) so PB
|
||
|
|
* overwrites the same on-disk file and the existing URL stays
|
||
|
|
* stable; uploading with the stripped name would orphan the old
|
||
|
|
* file and create a new entry that breaks every existing link
|
||
|
|
*
|
||
|
|
* The "display filename" we log + show in the report is the
|
||
|
|
* best-effort strip of the suffix, purely cosmetic.
|
||
|
|
*/
|
||
|
|
function displayFilenameFromToken(token) {
|
||
|
|
// Drop the random-suffix segment: split at the last `_` before the
|
||
|
|
// extension. The user filename may itself contain underscores
|
||
|
|
// (e.g. `dn26071401_v2.pdf`) so we only strip the last segment.
|
||
|
|
const dot = token.lastIndexOf(".");
|
||
|
|
if (dot === -1) return token;
|
||
|
|
const base = token.slice(0, dot);
|
||
|
|
const ext = token.slice(dot);
|
||
|
|
const lastUnderscore = base.lastIndexOf("_");
|
||
|
|
if (lastUnderscore === -1) return token;
|
||
|
|
const suffix = base.slice(lastUnderscore + 1);
|
||
|
|
// PB random suffixes are 8 lowercase alnum chars — a soft check so
|
||
|
|
// we don't strip a legitimate user-supplied suffix like `_v2`.
|
||
|
|
if (/^[a-z0-9]{8}$/.test(suffix)) {
|
||
|
|
return base.slice(0, lastUnderscore) + ext;
|
||
|
|
}
|
||
|
|
return token;
|
||
|
|
}
|
||
|
|
|
||
|
|
function runCommand(cmd, args, timeoutMs) {
|
||
|
|
return new Promise((resolve) => {
|
||
|
|
const child = spawn(cmd, args, { stdio: ["ignore", "pipe", "pipe"] });
|
||
|
|
let stdout = "";
|
||
|
|
let stderr = "";
|
||
|
|
let killed = false;
|
||
|
|
|
||
|
|
const timer = setTimeout(() => {
|
||
|
|
killed = true;
|
||
|
|
child.kill("SIGKILL");
|
||
|
|
}, timeoutMs);
|
||
|
|
|
||
|
|
child.stdout.on("data", (d) => (stdout += d.toString()));
|
||
|
|
child.stderr.on("data", (d) => (stderr += d.toString()));
|
||
|
|
|
||
|
|
child.on("error", (err) => {
|
||
|
|
clearTimeout(timer);
|
||
|
|
resolve({ ok: false, code: -1, stdout, stderr: stderr || err.message, killed });
|
||
|
|
});
|
||
|
|
|
||
|
|
child.on("close", (code) => {
|
||
|
|
clearTimeout(timer);
|
||
|
|
resolve({ ok: !killed && code === 0, code, stdout, stderr, killed });
|
||
|
|
});
|
||
|
|
});
|
||
|
|
}
|
||
|
|
|
||
|
|
async function compressPdf(srcPath, dstPath) {
|
||
|
|
const args = [
|
||
|
|
"-sDEVICE=pdfwrite",
|
||
|
|
"-dPDFSETTINGS=/ebook",
|
||
|
|
"-dNOPAUSE",
|
||
|
|
"-dBATCH",
|
||
|
|
"-dQUIET",
|
||
|
|
`-sOutputFile=${dstPath}`,
|
||
|
|
srcPath,
|
||
|
|
];
|
||
|
|
return runCommand("gs", args, GS_TIMEOUT_MS);
|
||
|
|
}
|
||
|
|
|
||
|
|
async function compressImage(srcPath, dstPath) {
|
||
|
|
// sips -Z MAX_WIDTH input --out output : resize keeping aspect ratio
|
||
|
|
const r1 = await runCommand(
|
||
|
|
"sips",
|
||
|
|
["-Z", String(JPG_MAX_WIDTH), srcPath, "--out", dstPath],
|
||
|
|
SIPS_TIMEOUT_MS,
|
||
|
|
);
|
||
|
|
if (!r1.ok) return r1;
|
||
|
|
|
||
|
|
// sips -Z only resizes; for JPG re-encode at q=70 via formatOptions
|
||
|
|
const reencode = await runCommand(
|
||
|
|
"sips",
|
||
|
|
[
|
||
|
|
"-s", "format", "jpeg",
|
||
|
|
"-s", "formatOptions", String(JPG_QUALITY),
|
||
|
|
dstPath,
|
||
|
|
"--out", dstPath,
|
||
|
|
],
|
||
|
|
SIPS_TIMEOUT_MS,
|
||
|
|
);
|
||
|
|
return reencode;
|
||
|
|
}
|
||
|
|
|
||
|
|
function compressionKind(filename) {
|
||
|
|
const ext = extOf(filename);
|
||
|
|
if (ext === "pdf") return "pdf";
|
||
|
|
if (SUPPORTED_EXT.has(ext)) return "image";
|
||
|
|
return null;
|
||
|
|
}
|
||
|
|
|
||
|
|
// Cache: collection name → [name, id]
|
||
|
|
const collectionCache = new Map();
|
||
|
|
async function getCollectionCandidates(pb, nameOrId) {
|
||
|
|
if (collectionCache.has(nameOrId)) return collectionCache.get(nameOrId);
|
||
|
|
const all = await pb.collections.getFullList();
|
||
|
|
const match = all.find((c) => c.name === nameOrId || c.id === nameOrId);
|
||
|
|
const result = match ? [nameOrId, match.id].filter((v, i, a) => a.indexOf(v) === i) : [nameOrId];
|
||
|
|
collectionCache.set(nameOrId, result);
|
||
|
|
return result;
|
||
|
|
}
|
||
|
|
|
||
|
|
async function downloadFile(pb, collection, recordId, filename, tmpDir) {
|
||
|
|
const candidates = await getCollectionCandidates(pb, collection);
|
||
|
|
let lastErr;
|
||
|
|
for (const candidate of candidates) {
|
||
|
|
const url = `${PB_URL}/api/files/${candidate}/${recordId}/${encodeURIComponent(filename)}`;
|
||
|
|
try {
|
||
|
|
const res = await fetch(url, {
|
||
|
|
headers: { Authorization: pb.authStore.token },
|
||
|
|
});
|
||
|
|
if (!res.ok) {
|
||
|
|
lastErr = new Error(`HTTP ${res.status} for ${url}`);
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
const buf = Buffer.from(await res.arrayBuffer());
|
||
|
|
const safeName = `${recordId}_${baseNameOf(filename)}`.replace(/[^A-Za-z0-9._-]/g, "_");
|
||
|
|
const localPath = join(tmpDir, safeName);
|
||
|
|
await writeFile(localPath, buf);
|
||
|
|
return { localPath, bytes: buf.byteLength };
|
||
|
|
} catch (err) {
|
||
|
|
lastErr = err;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
throw lastErr || new Error(`Failed to download ${collection}/${recordId}/${filename}`);
|
||
|
|
}
|
||
|
|
|
||
|
|
async function reuploadFile(pb, source, record, newFilePath, newFilename) {
|
||
|
|
const field = source.field;
|
||
|
|
const fd = new FormData();
|
||
|
|
const fileBuf = await readFile(newFilePath);
|
||
|
|
const mime = newFilename.match(/\.pdf$/i)
|
||
|
|
? "application/pdf"
|
||
|
|
: newFilename.match(/\.png$/i)
|
||
|
|
? "image/png"
|
||
|
|
: "image/jpeg";
|
||
|
|
const blob = new Blob([new Uint8Array(fileBuf)], { type: mime });
|
||
|
|
fd.append(field, blob, newFilename);
|
||
|
|
|
||
|
|
const updated = await pb.collection(source.collection).update(record.id, fd);
|
||
|
|
const newFieldRaw = updated?.[field];
|
||
|
|
const newField = Array.isArray(newFieldRaw) ? newFieldRaw : newFieldRaw ? [newFieldRaw] : [];
|
||
|
|
const oldFieldRaw = record[field];
|
||
|
|
const oldField = Array.isArray(oldFieldRaw) ? oldFieldRaw : oldFieldRaw ? [oldFieldRaw] : [];
|
||
|
|
|
||
|
|
// `tokenChanged` catches the case where PB appends a dedupe suffix
|
||
|
|
// to the new filename because the on-disk path already exists
|
||
|
|
// (length stays the same, but the URL breaks). `lengthChanged`
|
||
|
|
// catches the rarer case where PB silently appended instead of
|
||
|
|
// replacing (array grew, the original token is gone entirely).
|
||
|
|
const lengthChanged = newField.length !== oldField.length;
|
||
|
|
const tokenChanged =
|
||
|
|
!lengthChanged &&
|
||
|
|
newField.some((v, i) => oldField[i] !== v);
|
||
|
|
|
||
|
|
return {
|
||
|
|
newCount: newField.length,
|
||
|
|
oldCount: oldField.length,
|
||
|
|
lengthChanged,
|
||
|
|
tokenChanged,
|
||
|
|
};
|
||
|
|
}
|
||
|
|
|
||
|
|
function sha256OfFile(path) {
|
||
|
|
return new Promise((resolve, reject) => {
|
||
|
|
const h = createHash("sha256");
|
||
|
|
const s = createReadStream(path);
|
||
|
|
s.on("data", (c) => h.update(c));
|
||
|
|
s.on("end", () => resolve(h.digest("hex")));
|
||
|
|
s.on("error", reject);
|
||
|
|
});
|
||
|
|
}
|
||
|
|
|
||
|
|
async function processOne(pb, source, record, token, tmpDir) {
|
||
|
|
// Use the FULL token (with PB random suffix) for both download
|
||
|
|
// and re-upload, so URLs stay stable. `displayFilename` is just
|
||
|
|
// for the human-readable report.
|
||
|
|
const displayFilename = displayFilenameFromToken(token);
|
||
|
|
const filename = token; // the on-disk filename PB expects
|
||
|
|
const recordId = record.id;
|
||
|
|
|
||
|
|
const result = {
|
||
|
|
source: source.label,
|
||
|
|
collection: source.collection,
|
||
|
|
field: source.field,
|
||
|
|
recordId,
|
||
|
|
token,
|
||
|
|
filename, // on-disk filename (with PB random suffix)
|
||
|
|
displayFilename, // best-effort user-facing name for the report
|
||
|
|
ext: extOf(filename),
|
||
|
|
originalBytes: null,
|
||
|
|
compressedBytes: null,
|
||
|
|
sourceHash: null,
|
||
|
|
compressedHash: null,
|
||
|
|
action: null,
|
||
|
|
skipReason: null,
|
||
|
|
error: null,
|
||
|
|
ratio: null,
|
||
|
|
};
|
||
|
|
|
||
|
|
let localPath = null;
|
||
|
|
let compressedPath = null;
|
||
|
|
try {
|
||
|
|
const kind = compressionKind(displayFilename);
|
||
|
|
if (!kind) {
|
||
|
|
result.action = "skipped";
|
||
|
|
result.skipReason = "unsupported_extension";
|
||
|
|
return result;
|
||
|
|
}
|
||
|
|
|
||
|
|
const dl = await downloadFile(pb, source.collection, recordId, filename, tmpDir);
|
||
|
|
localPath = dl.localPath;
|
||
|
|
result.originalBytes = dl.bytes;
|
||
|
|
|
||
|
|
if (dl.bytes < SKIP_MIN_BYTES) {
|
||
|
|
result.action = "skipped";
|
||
|
|
result.skipReason = "too_small";
|
||
|
|
return result;
|
||
|
|
}
|
||
|
|
if (dl.bytes > SKIP_MAX_BYTES) {
|
||
|
|
result.action = "skipped";
|
||
|
|
result.skipReason = "too_large";
|
||
|
|
return result;
|
||
|
|
}
|
||
|
|
|
||
|
|
result.sourceHash = await sha256OfFile(localPath);
|
||
|
|
|
||
|
|
compressedPath = localPath + `.compressed.${kind === "pdf" ? "pdf" : "jpg"}`;
|
||
|
|
const cmp = kind === "pdf"
|
||
|
|
? await compressPdf(localPath, compressedPath)
|
||
|
|
: await compressImage(localPath, compressedPath);
|
||
|
|
if (!cmp.ok) {
|
||
|
|
result.action = "error";
|
||
|
|
result.error = cmp.killed
|
||
|
|
? `compression_timeout (${kind})`
|
||
|
|
: `compression_failed (${kind}): ${(cmp.stderr || cmp.stdout || "").slice(0, 200)}`;
|
||
|
|
return result;
|
||
|
|
}
|
||
|
|
|
||
|
|
const compressedStat = await stat(compressedPath);
|
||
|
|
result.compressedBytes = compressedStat.size;
|
||
|
|
result.ratio = +(result.compressedBytes / result.originalBytes).toFixed(3);
|
||
|
|
|
||
|
|
// Don't churn the URL for a 2% saving.
|
||
|
|
if (result.compressedBytes >= result.originalBytes * 0.98) {
|
||
|
|
result.action = "skipped";
|
||
|
|
result.skipReason = "no_meaningful_savings";
|
||
|
|
return result;
|
||
|
|
}
|
||
|
|
|
||
|
|
if (DRY_RUN) {
|
||
|
|
result.action = "dry_run";
|
||
|
|
return result;
|
||
|
|
}
|
||
|
|
|
||
|
|
const reup = await reuploadFile(pb, source, record, compressedPath, filename);
|
||
|
|
result.compressedHash = await sha256OfFile(compressedPath);
|
||
|
|
result.action = "compressed";
|
||
|
|
result.reupload = reup;
|
||
|
|
return result;
|
||
|
|
} catch (err) {
|
||
|
|
result.action = "error";
|
||
|
|
result.error = (err?.message || String(err)).slice(0, 500);
|
||
|
|
return result;
|
||
|
|
} finally {
|
||
|
|
for (const p of [localPath, compressedPath]) {
|
||
|
|
if (p) {
|
||
|
|
try { await unlink(p); } catch {}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
async function runWithConcurrency(items, limit, fn, onProgress) {
|
||
|
|
const results = new Array(items.length);
|
||
|
|
let next = 0;
|
||
|
|
let inFlight = 0;
|
||
|
|
return new Promise((resolve) => {
|
||
|
|
const launch = () => {
|
||
|
|
while (inFlight < limit && next < items.length) {
|
||
|
|
const i = next++;
|
||
|
|
inFlight++;
|
||
|
|
fn(items[i], i)
|
||
|
|
.then((r) => {
|
||
|
|
results[i] = r;
|
||
|
|
inFlight--;
|
||
|
|
onProgress?.(i, r);
|
||
|
|
if (next >= items.length && inFlight === 0) resolve(results);
|
||
|
|
else launch();
|
||
|
|
})
|
||
|
|
.catch((err) => {
|
||
|
|
results[i] = { action: "error", error: err?.message || String(err) };
|
||
|
|
inFlight--;
|
||
|
|
if (next >= items.length && inFlight === 0) resolve(results);
|
||
|
|
else launch();
|
||
|
|
});
|
||
|
|
}
|
||
|
|
};
|
||
|
|
launch();
|
||
|
|
});
|
||
|
|
}
|
||
|
|
|
||
|
|
async function loadAllRecordsWithFiles(pb, source, recordId) {
|
||
|
|
let records;
|
||
|
|
if (recordId) {
|
||
|
|
const r = await pb
|
||
|
|
.collection(source.collection)
|
||
|
|
.getOne(recordId)
|
||
|
|
.catch((err) => {
|
||
|
|
console.error(
|
||
|
|
` --record=${recordId}: not found in ${source.collection} (${err?.message || err})`,
|
||
|
|
);
|
||
|
|
return null;
|
||
|
|
});
|
||
|
|
records = r ? [r] : [];
|
||
|
|
} else {
|
||
|
|
records = await pb.collection(source.collection).getFullList({
|
||
|
|
batch: 500,
|
||
|
|
});
|
||
|
|
}
|
||
|
|
const out = [];
|
||
|
|
for (const r of records) {
|
||
|
|
const fieldRaw = r[source.field];
|
||
|
|
if (!fieldRaw) continue;
|
||
|
|
const arr = Array.isArray(fieldRaw) ? fieldRaw : [fieldRaw];
|
||
|
|
if (arr.length > 0) {
|
||
|
|
out.push({ id: r.id, [source.field]: arr });
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return out;
|
||
|
|
}
|
||
|
|
|
||
|
|
async function main() {
|
||
|
|
if (!ADMIN_EMAIL || !ADMIN_PASSWORD) {
|
||
|
|
throw new Error("Set PB_ADMIN_EMAIL and PB_ADMIN_PASSWORD env vars.");
|
||
|
|
}
|
||
|
|
if (DRY_RUN) console.log("=== DRY RUN — no files will be re-uploaded ===");
|
||
|
|
|
||
|
|
// Validate argv flags BEFORE we authenticate, so a typo or missing
|
||
|
|
// value exits cleanly without a wasted PB admin login.
|
||
|
|
const sources = parseSources(process.argv);
|
||
|
|
const recordId = parseRecordId(process.argv);
|
||
|
|
|
||
|
|
const pb = new PocketBase(PB_URL);
|
||
|
|
// Disable auto-cancellation: the SDK cancels any in-flight request
|
||
|
|
// whenever a new one starts, which breaks Promise.all-style fan-out.
|
||
|
|
// We do several concurrent downloads/uploads so we want them all to
|
||
|
|
// land. Same pattern as src/lib/server/pocketbase-admin.ts.
|
||
|
|
pb.autoCancellation(false);
|
||
|
|
await pb.collection("_superusers").authWithPassword(ADMIN_EMAIL, ADMIN_PASSWORD);
|
||
|
|
console.log(`Authenticated against ${PB_URL}`);
|
||
|
|
console.log(`Concurrency: ${MAX_PARALLEL}, limit per source: ${LIMIT === Infinity ? "∞" : LIMIT}`);
|
||
|
|
console.log();
|
||
|
|
|
||
|
|
const tmpDir = await ensureDir(join(tmpdir(), `wfdash-compress-${process.pid}`));
|
||
|
|
|
||
|
|
const allResults = [];
|
||
|
|
if (sources.length < SOURCES.length) {
|
||
|
|
const labels = sources.map((s) => s.label).join(", ");
|
||
|
|
console.log(`Scoped to: ${labels}`);
|
||
|
|
}
|
||
|
|
if (recordId) console.log(`Scoped to record: ${recordId}`);
|
||
|
|
for (const source of sources) {
|
||
|
|
console.log(`\n=== ${source.label} ===`);
|
||
|
|
const t0 = Date.now();
|
||
|
|
const records = await loadAllRecordsWithFiles(pb, source, recordId);
|
||
|
|
console.log(` Found ${records.length} records with files`);
|
||
|
|
|
||
|
|
const work = [];
|
||
|
|
for (const r of records) {
|
||
|
|
const arr = Array.isArray(r[source.field]) ? r[source.field] : [r[source.field]];
|
||
|
|
for (const token of arr) {
|
||
|
|
if (!token) continue;
|
||
|
|
work.push({ record: r, token });
|
||
|
|
if (work.length >= LIMIT) break;
|
||
|
|
}
|
||
|
|
if (work.length >= LIMIT) break;
|
||
|
|
}
|
||
|
|
console.log(` ${work.length} file(s) to consider`);
|
||
|
|
|
||
|
|
let done = 0;
|
||
|
|
const results = await runWithConcurrency(
|
||
|
|
work,
|
||
|
|
MAX_PARALLEL,
|
||
|
|
(item) => processOne(pb, source, item.record, item.token, tmpDir),
|
||
|
|
() => {
|
||
|
|
done++;
|
||
|
|
if (done % 25 === 0 || done === work.length) {
|
||
|
|
process.stdout.write(` ...${done}/${work.length}\r`);
|
||
|
|
}
|
||
|
|
},
|
||
|
|
);
|
||
|
|
process.stdout.write("\n");
|
||
|
|
|
||
|
|
const summary = summarise(results);
|
||
|
|
const elapsed = ((Date.now() - t0) / 1000).toFixed(1);
|
||
|
|
console.log(` Took ${elapsed}s`);
|
||
|
|
console.log(` Compressed: ${summary.compressed} | Skipped: ${summary.skipped} | Errors: ${summary.errors}`);
|
||
|
|
console.log(` Bytes: ${bytes(summary.originalBytes)} → ${bytes(summary.compressedBytes)} (saved ${bytes(summary.savedBytes)}, ${summary.savedPct}%)`);
|
||
|
|
console.log(` Dry-run only: ${summary.dryRun}`);
|
||
|
|
if (summary.tokensChanged > 0) {
|
||
|
|
console.log(` ⚠️ Tokens changed (URLs invalidated): ${summary.tokensChanged}`);
|
||
|
|
}
|
||
|
|
allResults.push({ source: source.label, summary, results });
|
||
|
|
}
|
||
|
|
|
||
|
|
const grand = allResults.reduce(
|
||
|
|
(acc, r) => ({
|
||
|
|
compressed: acc.compressed + r.summary.compressed,
|
||
|
|
skipped: acc.skipped + r.summary.skipped,
|
||
|
|
errors: acc.errors + r.summary.errors,
|
||
|
|
dryRun: acc.dryRun + r.summary.dryRun,
|
||
|
|
tokensChanged: acc.tokensChanged + r.summary.tokensChanged,
|
||
|
|
originalBytes: acc.originalBytes + r.summary.originalBytes,
|
||
|
|
compressedBytes: acc.compressedBytes + r.summary.compressedBytes,
|
||
|
|
}),
|
||
|
|
{ compressed: 0, skipped: 0, errors: 0, dryRun: 0, tokensChanged: 0, originalBytes: 0, compressedBytes: 0 },
|
||
|
|
);
|
||
|
|
grand.savedBytes = grand.originalBytes - grand.compressedBytes;
|
||
|
|
grand.savedPct = grand.originalBytes > 0
|
||
|
|
? +((grand.savedBytes / grand.originalBytes) * 100).toFixed(1)
|
||
|
|
: 0;
|
||
|
|
|
||
|
|
console.log("\n=== GRAND TOTAL ===");
|
||
|
|
console.log(`Compressed: ${grand.compressed} | Skipped: ${grand.skipped} | Errors: ${grand.errors} | Dry-run: ${grand.dryRun}`);
|
||
|
|
console.log(`Bytes: ${bytes(grand.originalBytes)} → ${bytes(grand.compressedBytes)} (saved ${bytes(grand.savedBytes)}, ${grand.savedPct}%)`);
|
||
|
|
if (grand.tokensChanged > 0) {
|
||
|
|
console.log(`\n⚠️ Total tokens changed (URLs invalidated): ${grand.tokensChanged}`);
|
||
|
|
console.log(` PB appended a dedupe suffix to each new filename because the on-disk path already existed.`);
|
||
|
|
console.log(` Every URL built from these records' file fields just changed. Old links 404.`);
|
||
|
|
}
|
||
|
|
|
||
|
|
const date = todayStamp();
|
||
|
|
const jsonPath = `reports/attachment-compression-${date}.json`;
|
||
|
|
const mdPath = `reports/attachment-compression-${date}.md`;
|
||
|
|
await writeFile(jsonPath, JSON.stringify({ grand, sources: allResults }, null, 2));
|
||
|
|
await writeFile(mdPath, renderMarkdown(grand, allResults));
|
||
|
|
console.log(`\nReport written to ${jsonPath}`);
|
||
|
|
console.log(`Report written to ${mdPath}`);
|
||
|
|
}
|
||
|
|
|
||
|
|
function summarise(results) {
|
||
|
|
const s = { compressed: 0, skipped: 0, errors: 0, dryRun: 0, tokensChanged: 0, originalBytes: 0, compressedBytes: 0 };
|
||
|
|
for (const r of results) {
|
||
|
|
if (r.action === "compressed" || r.action === "dry_run") {
|
||
|
|
if (r.action === "compressed") s.compressed++;
|
||
|
|
else s.dryRun++;
|
||
|
|
s.originalBytes += r.originalBytes || 0;
|
||
|
|
s.compressedBytes += r.compressedBytes || 0;
|
||
|
|
if (r.reupload?.tokenChanged) s.tokensChanged++;
|
||
|
|
} else if (r.action === "skipped") {
|
||
|
|
s.skipped++;
|
||
|
|
if (r.originalBytes != null) s.originalBytes += r.originalBytes;
|
||
|
|
if (r.compressedBytes != null) s.compressedBytes += r.compressedBytes;
|
||
|
|
} else if (r.action === "error") {
|
||
|
|
s.errors++;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
s.savedBytes = s.originalBytes - s.compressedBytes;
|
||
|
|
s.savedPct = s.originalBytes > 0
|
||
|
|
? +((s.savedBytes / s.originalBytes) * 100).toFixed(1)
|
||
|
|
: 0;
|
||
|
|
return s;
|
||
|
|
}
|
||
|
|
|
||
|
|
function renderMarkdown(grand, sources) {
|
||
|
|
let out = `# Attachment compression report — ${todayStamp()}\n\n`;
|
||
|
|
out += `_${DRY_RUN ? "DRY RUN — no files were re-uploaded" : "One-time bulk pass"}_\n\n`;
|
||
|
|
out += `## Grand total\n\n`;
|
||
|
|
out += `| Compressed | Skipped | Errors | Dry-run | Tokens changed |\n|---:|---:|---:|---:|---:|\n`;
|
||
|
|
out += `| ${grand.compressed} | ${grand.skipped} | ${grand.errors} | ${grand.dryRun} | ${grand.tokensChanged} |\n\n`;
|
||
|
|
out += `| Original | Compressed | Saved | % |\n|---:|---:|---:|---:|\n`;
|
||
|
|
out += `| ${bytes(grand.originalBytes)} | ${bytes(grand.compressedBytes)} | ${bytes(grand.savedBytes)} | ${grand.savedPct}% |\n\n`;
|
||
|
|
if (grand.tokensChanged > 0) {
|
||
|
|
out += `> ⚠️ **${grand.tokensChanged} token(s) changed.** PB appended dedupe suffixes to those filenames because the on-disk path already existed. Every URL built from these records' file fields is invalidated; old links 404.\n\n`;
|
||
|
|
}
|
||
|
|
|
||
|
|
out += `## Per-source\n\n`;
|
||
|
|
for (const s of sources) {
|
||
|
|
out += `### ${s.source}\n\n`;
|
||
|
|
out += `| Compressed | Skipped | Errors | Tokens changed | Original | Compressed | Saved | % |\n|---:|---:|---:|---:|---:|---:|---:|---:|\n`;
|
||
|
|
out += `| ${s.summary.compressed} | ${s.summary.skipped} | ${s.summary.errors} | ${s.summary.tokensChanged} | ${bytes(s.summary.originalBytes)} | ${bytes(s.summary.compressedBytes)} | ${bytes(s.summary.savedBytes)} | ${s.summary.savedPct}% |\n\n`;
|
||
|
|
|
||
|
|
const skipReasons = {};
|
||
|
|
const errorSamples = [];
|
||
|
|
for (const r of s.results) {
|
||
|
|
if (r.action === "skipped" && r.skipReason) {
|
||
|
|
skipReasons[r.skipReason] = (skipReasons[r.skipReason] || 0) + 1;
|
||
|
|
} else if (r.action === "error") {
|
||
|
|
if (errorSamples.length < 10) errorSamples.push(r);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if (Object.keys(skipReasons).length > 0) {
|
||
|
|
out += `**Skip reasons:**\n`;
|
||
|
|
for (const [reason, count] of Object.entries(skipReasons).sort((a, b) => b[1] - a[1])) {
|
||
|
|
out += `- \`${reason}\`: ${count}\n`;
|
||
|
|
}
|
||
|
|
out += `\n`;
|
||
|
|
}
|
||
|
|
if (errorSamples.length > 0) {
|
||
|
|
out += `**Errors (first 10):**\n`;
|
||
|
|
for (const e of errorSamples) {
|
||
|
|
out += `- \`${e.displayFilename || e.filename}\`: ${e.error}\n`;
|
||
|
|
}
|
||
|
|
out += `\n`;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return out;
|
||
|
|
}
|
||
|
|
|
||
|
|
main().catch((err) => {
|
||
|
|
console.error("FATAL:", err);
|
||
|
|
process.exit(1);
|
||
|
|
});
|