Dave
7555196166
- Added an audit trail feature that logs every create, update, and delete action on delivery notes and orders, capturing detailed changes per field. - Introduced the `AuditTimeline` component to display the history of changes for delivery notes and orders, enhancing traceability. - Updated the `page.tsx` files for delivery notes and orders to fetch and display audit logs, ensuring users can view the history of modifications. - Enhanced the `package.json` with a new migration script for setting up the audit log functionality. This feature improves accountability and transparency in the management of delivery notes and orders.
796 lines
28 KiB
JavaScript
796 lines
28 KiB
JavaScript
/**
|
|
* One-time bulk pass: download every attachment from the three
|
|
* configured PB collections/fields, compress it (PDF → /ebook,
|
|
* JPG/PNG → 1200px q=70), and re-upload it to the same record
|
|
* under the same filename.
|
|
*
|
|
* Why a one-time script (not an API route) for the first pass:
|
|
* - We want a single human-driven command that runs to
|
|
* completion and produces a verifiable report. No cron,
|
|
* no auth flow, no Next.js context.
|
|
* - We DON'T yet have the `attachment_compression_log` PB
|
|
* collection — that belongs to the recurring weekly agent,
|
|
* where it powers hash-based dedup. A one-time pass just
|
|
* processes everything once and writes its report to disk.
|
|
*
|
|
* Re-uploading under the same filename:
|
|
* PB file fields are arrays of filenames. Posting FormData
|
|
* with a file whose name matches an existing entry REPLACES
|
|
* that entry in place — the file's disk path changes but the
|
|
* filename + URL stay stable, so every existing download
|
|
* link / email attachment / audit_log reference keeps
|
|
* working.
|
|
*
|
|
* Skip rules (from the brainstorming session):
|
|
* - file < 50 KB (noise)
|
|
* - file > 24 MB (PB hard cap is 25 MB; leave headroom)
|
|
* - extension not in {pdf, jpg, jpeg, png}
|
|
* - already-compressed size ≥ original (no-op re-upload)
|
|
* - compression tool failed / timed out
|
|
*
|
|
* Re-upload safety:
|
|
* PB's FormData update REPLACES the file field array. For
|
|
* a single-file replacement we re-upload the whole array
|
|
* with the one matching entry replaced, so sibling files
|
|
* on the same record are untouched. See the stock-purchases
|
|
* service for the same pattern (`additional_attachments`).
|
|
*
|
|
* URL-stability caveat (PB dedupe-suffix behavior):
|
|
* PB stores each file on disk as `{recordId}_{filename}`. When
|
|
* we re-upload a file with the same filename and that exact
|
|
* on-disk path already exists, PB DOES NOT overwrite in place
|
|
* — it appends an 8-11 char dedupe suffix to the new file.
|
|
* The field's stored filename changes, so every URL built
|
|
* from `record.file[i]` is invalidated for that record.
|
|
* `reuploadFile()` detects this as `tokenChanged` (in addition
|
|
* to `lengthChanged`) and the report surfaces it as a warning.
|
|
* For collections where URL stability matters (anything with
|
|
* emailed links, external bookmarks, or PII tracking), test
|
|
* on a single record first with `--record=<id>` before doing
|
|
* a bulk run. See reports/attachment-compression-2026-09-26.md
|
|
* for the live example (1,420 quote_attachments URLs changed).
|
|
*
|
|
* Run:
|
|
* npx tsx scripts/compress-attachments-once.mjs
|
|
* # or
|
|
* npx tsx scripts/compress-attachments-once.mjs --dry-run
|
|
* # limit for testing:
|
|
* npx tsx scripts/compress-attachments-once.mjs --limit 20
|
|
* # scope to one source for first-pass rollout:
|
|
* npx tsx scripts/compress-attachments-once.mjs --dry-run --source=quote_attachments
|
|
* # comma-separated, multiple sources:
|
|
* npx tsx scripts/compress-attachments-once.mjs --source=quote_attachments,purchase_orders.documents
|
|
* # scope to a single record (sanity-check URL stability first):
|
|
* npx tsx scripts/compress-attachments-once.mjs --dry-run --record=8ay88lwwjiwwggc
|
|
*
|
|
* Known source labels (must match one of these exactly):
|
|
* - quote_attachments
|
|
* - order_item_attachements
|
|
* - purchase_orders.documents
|
|
*
|
|
* Output:
|
|
* - stdout: per-collection progress + grand total
|
|
* - reports/attachment-compression-<date>.json: full machine-
|
|
* readable report (every file processed)
|
|
* - reports/attachment-compression-<date>.md: human-readable
|
|
* summary table
|
|
*/
|
|
|
|
// Inline .env/.env.local loader (same pattern as scripts/_env.ts,
|
|
// kept self-contained so this script can run as a plain node ESM
|
|
// without relying on a sibling .ts file).
|
|
import { readFileSync } from "node:fs";
|
|
import { resolve } from "node:path";
|
|
for (const name of [".env", ".env.local"]) {
|
|
try {
|
|
const raw = readFileSync(resolve(process.cwd(), name), "utf8");
|
|
for (const line of raw.split("\n")) {
|
|
const trimmed = line.trim();
|
|
if (!trimmed || trimmed.startsWith("#")) continue;
|
|
const m = trimmed.match(/^([A-Z0-9_]+)\s*=\s*(.+)$/i);
|
|
if (m && !process.env[m[1]]) {
|
|
process.env[m[1]] = m[2].replace(/^["']|["']$/g, "").trim();
|
|
}
|
|
}
|
|
} catch {}
|
|
}
|
|
|
|
import PocketBase from "pocketbase";
|
|
import { createHash } from "node:crypto";
|
|
import { createReadStream } from "node:fs";
|
|
import { mkdir, writeFile, stat, unlink, readFile } from "node:fs/promises";
|
|
import { tmpdir } from "node:os";
|
|
import { join } from "node:path";
|
|
import { spawn } from "node:child_process";
|
|
|
|
const PB_URL =
|
|
process.env.NEXT_PUBLIC_POCKETBASE_URL || "http://127.0.0.1:8090";
|
|
const ADMIN_EMAIL = process.env.PB_ADMIN_EMAIL;
|
|
const ADMIN_PASSWORD = process.env.PB_ADMIN_PASSWORD;
|
|
|
|
const DRY_RUN = process.argv.includes("--dry-run");
|
|
const LIMIT = parseLimit(process.argv);
|
|
const MAX_PARALLEL = parseIntEnv("COMPRESS_PARALLEL", "4");
|
|
|
|
const SKIP_MIN_BYTES = 50 * 1024; // 50 KB
|
|
const SKIP_MAX_BYTES = 24 * 1024 * 1024; // 24 MB (PB cap is 25)
|
|
const SUPPORTED_EXT = new Set(["pdf", "jpg", "jpeg", "png"]);
|
|
const JPG_MAX_WIDTH = 1200;
|
|
const JPG_QUALITY = 70;
|
|
const GS_TIMEOUT_MS = 60_000;
|
|
const SIPS_TIMEOUT_MS = 30_000;
|
|
|
|
const SOURCES = [
|
|
{
|
|
label: "order_item_attachements",
|
|
collection: "order_items",
|
|
field: "order_item_attachements",
|
|
},
|
|
{
|
|
label: "quote_attachments",
|
|
collection: "quote_attachments",
|
|
field: "file",
|
|
},
|
|
{
|
|
label: "purchase_orders.documents",
|
|
collection: "purchase_orders",
|
|
field: "documents",
|
|
},
|
|
];
|
|
|
|
function parseLimit(argv) {
|
|
const i = argv.indexOf("--limit");
|
|
if (i === -1) return Infinity;
|
|
const n = parseInt(argv[i + 1], 10);
|
|
return Number.isFinite(n) && n > 0 ? n : Infinity;
|
|
}
|
|
|
|
/**
|
|
* --record=<id>
|
|
*
|
|
* Scope the run to exactly one PB record (per source). Use with
|
|
* `--dry-run` first to sanity-check URL stability on a single
|
|
* record before doing a bulk run on a collection where stable
|
|
* URLs matter (emailed links, bookmarks).
|
|
*
|
|
* Accepts both forms:
|
|
* --record=8ay88lwwjiwwggc (equals form)
|
|
* --record 8ay88lwwjiwwggc (space form)
|
|
*/
|
|
function parseRecordId(argv) {
|
|
for (let i = 0; i < argv.length; i++) {
|
|
const a = argv[i];
|
|
if (a === "--record") {
|
|
const next = argv[i + 1];
|
|
if (!next || next.startsWith("--")) {
|
|
console.error(
|
|
"--record requires a value (e.g. --record=8ay88lwwjiwwggc)",
|
|
);
|
|
process.exit(1);
|
|
}
|
|
return next;
|
|
} else if (a.startsWith("--record=")) {
|
|
const v = a.slice("--record=".length).trim();
|
|
if (!v) {
|
|
console.error(
|
|
"--record requires a value (e.g. --record=8ay88lwwjiwwggc)",
|
|
);
|
|
process.exit(1);
|
|
}
|
|
return v;
|
|
}
|
|
}
|
|
return null;
|
|
}
|
|
|
|
/**
|
|
* --source=<label[,label...]>
|
|
*
|
|
* Scope the run to a subset of the configured SOURCES. Strict match
|
|
* against the labels below — typos exit(1) loudly rather than
|
|
* silently processing zero files.
|
|
*
|
|
* Accepts both forms:
|
|
* --source=quote_attachments (equals form)
|
|
* --source quote_attachments (space form)
|
|
* --source=quote_attachments,order_item_attachements
|
|
*/
|
|
function parseSources(argv) {
|
|
const wanted = [];
|
|
for (let i = 0; i < argv.length; i++) {
|
|
const a = argv[i];
|
|
if (a === "--source") {
|
|
const next = argv[i + 1];
|
|
if (!next || next.startsWith("--")) {
|
|
console.error(
|
|
"--source requires a value (e.g. --source=quote_attachments)",
|
|
);
|
|
process.exit(1);
|
|
}
|
|
wanted.push(
|
|
...next.split(",").map((s) => s.trim()).filter(Boolean),
|
|
);
|
|
} else if (a.startsWith("--source=")) {
|
|
wanted.push(
|
|
...a
|
|
.slice("--source=".length)
|
|
.split(",")
|
|
.map((s) => s.trim())
|
|
.filter(Boolean),
|
|
);
|
|
}
|
|
}
|
|
if (wanted.length === 0) return SOURCES;
|
|
const known = SOURCES.map((s) => s.label);
|
|
const unknown = wanted.filter((w) => !known.includes(w));
|
|
if (unknown.length > 0) {
|
|
console.error(
|
|
`--source: unknown value(s): ${unknown.join(", ")}. Available: ${known.join(", ")}`,
|
|
);
|
|
process.exit(1);
|
|
}
|
|
// Preserve the canonical order of SOURCES so reports stay stable
|
|
// even when the user lists them in a different order.
|
|
return SOURCES.filter((s) => wanted.includes(s.label));
|
|
}
|
|
|
|
function parseIntEnv(name, fallback) {
|
|
const v = parseInt(process.env[name] || "", 10);
|
|
return Number.isFinite(v) && v > 0 ? v : fallback;
|
|
}
|
|
|
|
function todayStamp() {
|
|
return new Date().toISOString().slice(0, 10);
|
|
}
|
|
|
|
function ensureDir(p) {
|
|
return mkdir(p, { recursive: true });
|
|
}
|
|
|
|
function bytes(n) {
|
|
if (n == null) return "?";
|
|
if (n < 1024) return `${n} B`;
|
|
if (n < 1024 * 1024) return `${(n / 1024).toFixed(1)} KB`;
|
|
return `${(n / (1024 * 1024)).toFixed(2)} MB`;
|
|
}
|
|
|
|
function extOf(name) {
|
|
const dot = name.lastIndexOf(".");
|
|
return dot === -1 ? "" : name.slice(dot + 1).toLowerCase();
|
|
}
|
|
|
|
function baseNameOf(name) {
|
|
const slash = Math.max(name.lastIndexOf("/"), name.lastIndexOf("\\"));
|
|
return slash === -1 ? name : name.slice(slash + 1);
|
|
}
|
|
|
|
/**
|
|
* PB stores each uploaded file on disk as `{recordId}_{fieldValue}`,
|
|
* and the field's stored value is `<originalName>_<randomSuffix>.<ext>`
|
|
* — e.g. `image001_g8etl02zu0.png`. The token is the full field
|
|
* value (with the random suffix) and that exact string is what
|
|
* appears in the URL `/api/files/{collection}/{recordId}/{token}`.
|
|
*
|
|
* For the compression agent this matters because:
|
|
* - downloads must use the FULL token (with suffix), not a stripped
|
|
* version, or PB returns 404
|
|
* - re-uploads must use the SAME token (with suffix) so PB
|
|
* overwrites the same on-disk file and the existing URL stays
|
|
* stable; uploading with the stripped name would orphan the old
|
|
* file and create a new entry that breaks every existing link
|
|
*
|
|
* The "display filename" we log + show in the report is the
|
|
* best-effort strip of the suffix, purely cosmetic.
|
|
*/
|
|
function displayFilenameFromToken(token) {
|
|
// Drop the random-suffix segment: split at the last `_` before the
|
|
// extension. The user filename may itself contain underscores
|
|
// (e.g. `dn26071401_v2.pdf`) so we only strip the last segment.
|
|
const dot = token.lastIndexOf(".");
|
|
if (dot === -1) return token;
|
|
const base = token.slice(0, dot);
|
|
const ext = token.slice(dot);
|
|
const lastUnderscore = base.lastIndexOf("_");
|
|
if (lastUnderscore === -1) return token;
|
|
const suffix = base.slice(lastUnderscore + 1);
|
|
// PB random suffixes are 8 lowercase alnum chars — a soft check so
|
|
// we don't strip a legitimate user-supplied suffix like `_v2`.
|
|
if (/^[a-z0-9]{8}$/.test(suffix)) {
|
|
return base.slice(0, lastUnderscore) + ext;
|
|
}
|
|
return token;
|
|
}
|
|
|
|
function runCommand(cmd, args, timeoutMs) {
|
|
return new Promise((resolve) => {
|
|
const child = spawn(cmd, args, { stdio: ["ignore", "pipe", "pipe"] });
|
|
let stdout = "";
|
|
let stderr = "";
|
|
let killed = false;
|
|
|
|
const timer = setTimeout(() => {
|
|
killed = true;
|
|
child.kill("SIGKILL");
|
|
}, timeoutMs);
|
|
|
|
child.stdout.on("data", (d) => (stdout += d.toString()));
|
|
child.stderr.on("data", (d) => (stderr += d.toString()));
|
|
|
|
child.on("error", (err) => {
|
|
clearTimeout(timer);
|
|
resolve({ ok: false, code: -1, stdout, stderr: stderr || err.message, killed });
|
|
});
|
|
|
|
child.on("close", (code) => {
|
|
clearTimeout(timer);
|
|
resolve({ ok: !killed && code === 0, code, stdout, stderr, killed });
|
|
});
|
|
});
|
|
}
|
|
|
|
async function compressPdf(srcPath, dstPath) {
|
|
const args = [
|
|
"-sDEVICE=pdfwrite",
|
|
"-dPDFSETTINGS=/ebook",
|
|
"-dNOPAUSE",
|
|
"-dBATCH",
|
|
"-dQUIET",
|
|
`-sOutputFile=${dstPath}`,
|
|
srcPath,
|
|
];
|
|
return runCommand("gs", args, GS_TIMEOUT_MS);
|
|
}
|
|
|
|
async function compressImage(srcPath, dstPath) {
|
|
// sips -Z MAX_WIDTH input --out output : resize keeping aspect ratio
|
|
const r1 = await runCommand(
|
|
"sips",
|
|
["-Z", String(JPG_MAX_WIDTH), srcPath, "--out", dstPath],
|
|
SIPS_TIMEOUT_MS,
|
|
);
|
|
if (!r1.ok) return r1;
|
|
|
|
// sips -Z only resizes; for JPG re-encode at q=70 via formatOptions
|
|
const reencode = await runCommand(
|
|
"sips",
|
|
[
|
|
"-s", "format", "jpeg",
|
|
"-s", "formatOptions", String(JPG_QUALITY),
|
|
dstPath,
|
|
"--out", dstPath,
|
|
],
|
|
SIPS_TIMEOUT_MS,
|
|
);
|
|
return reencode;
|
|
}
|
|
|
|
function compressionKind(filename) {
|
|
const ext = extOf(filename);
|
|
if (ext === "pdf") return "pdf";
|
|
if (SUPPORTED_EXT.has(ext)) return "image";
|
|
return null;
|
|
}
|
|
|
|
// Cache: collection name → [name, id]
|
|
const collectionCache = new Map();
|
|
async function getCollectionCandidates(pb, nameOrId) {
|
|
if (collectionCache.has(nameOrId)) return collectionCache.get(nameOrId);
|
|
const all = await pb.collections.getFullList();
|
|
const match = all.find((c) => c.name === nameOrId || c.id === nameOrId);
|
|
const result = match ? [nameOrId, match.id].filter((v, i, a) => a.indexOf(v) === i) : [nameOrId];
|
|
collectionCache.set(nameOrId, result);
|
|
return result;
|
|
}
|
|
|
|
async function downloadFile(pb, collection, recordId, filename, tmpDir) {
|
|
const candidates = await getCollectionCandidates(pb, collection);
|
|
let lastErr;
|
|
for (const candidate of candidates) {
|
|
const url = `${PB_URL}/api/files/${candidate}/${recordId}/${encodeURIComponent(filename)}`;
|
|
try {
|
|
const res = await fetch(url, {
|
|
headers: { Authorization: pb.authStore.token },
|
|
});
|
|
if (!res.ok) {
|
|
lastErr = new Error(`HTTP ${res.status} for ${url}`);
|
|
continue;
|
|
}
|
|
const buf = Buffer.from(await res.arrayBuffer());
|
|
const safeName = `${recordId}_${baseNameOf(filename)}`.replace(/[^A-Za-z0-9._-]/g, "_");
|
|
const localPath = join(tmpDir, safeName);
|
|
await writeFile(localPath, buf);
|
|
return { localPath, bytes: buf.byteLength };
|
|
} catch (err) {
|
|
lastErr = err;
|
|
}
|
|
}
|
|
throw lastErr || new Error(`Failed to download ${collection}/${recordId}/${filename}`);
|
|
}
|
|
|
|
async function reuploadFile(pb, source, record, newFilePath, newFilename) {
|
|
const field = source.field;
|
|
const fd = new FormData();
|
|
const fileBuf = await readFile(newFilePath);
|
|
const mime = newFilename.match(/\.pdf$/i)
|
|
? "application/pdf"
|
|
: newFilename.match(/\.png$/i)
|
|
? "image/png"
|
|
: "image/jpeg";
|
|
const blob = new Blob([new Uint8Array(fileBuf)], { type: mime });
|
|
fd.append(field, blob, newFilename);
|
|
|
|
const updated = await pb.collection(source.collection).update(record.id, fd);
|
|
const newFieldRaw = updated?.[field];
|
|
const newField = Array.isArray(newFieldRaw) ? newFieldRaw : newFieldRaw ? [newFieldRaw] : [];
|
|
const oldFieldRaw = record[field];
|
|
const oldField = Array.isArray(oldFieldRaw) ? oldFieldRaw : oldFieldRaw ? [oldFieldRaw] : [];
|
|
|
|
// `tokenChanged` catches the case where PB appends a dedupe suffix
|
|
// to the new filename because the on-disk path already exists
|
|
// (length stays the same, but the URL breaks). `lengthChanged`
|
|
// catches the rarer case where PB silently appended instead of
|
|
// replacing (array grew, the original token is gone entirely).
|
|
const lengthChanged = newField.length !== oldField.length;
|
|
const tokenChanged =
|
|
!lengthChanged &&
|
|
newField.some((v, i) => oldField[i] !== v);
|
|
|
|
return {
|
|
newCount: newField.length,
|
|
oldCount: oldField.length,
|
|
lengthChanged,
|
|
tokenChanged,
|
|
};
|
|
}
|
|
|
|
function sha256OfFile(path) {
|
|
return new Promise((resolve, reject) => {
|
|
const h = createHash("sha256");
|
|
const s = createReadStream(path);
|
|
s.on("data", (c) => h.update(c));
|
|
s.on("end", () => resolve(h.digest("hex")));
|
|
s.on("error", reject);
|
|
});
|
|
}
|
|
|
|
async function processOne(pb, source, record, token, tmpDir) {
|
|
// Use the FULL token (with PB random suffix) for both download
|
|
// and re-upload, so URLs stay stable. `displayFilename` is just
|
|
// for the human-readable report.
|
|
const displayFilename = displayFilenameFromToken(token);
|
|
const filename = token; // the on-disk filename PB expects
|
|
const recordId = record.id;
|
|
|
|
const result = {
|
|
source: source.label,
|
|
collection: source.collection,
|
|
field: source.field,
|
|
recordId,
|
|
token,
|
|
filename, // on-disk filename (with PB random suffix)
|
|
displayFilename, // best-effort user-facing name for the report
|
|
ext: extOf(filename),
|
|
originalBytes: null,
|
|
compressedBytes: null,
|
|
sourceHash: null,
|
|
compressedHash: null,
|
|
action: null,
|
|
skipReason: null,
|
|
error: null,
|
|
ratio: null,
|
|
};
|
|
|
|
let localPath = null;
|
|
let compressedPath = null;
|
|
try {
|
|
const kind = compressionKind(displayFilename);
|
|
if (!kind) {
|
|
result.action = "skipped";
|
|
result.skipReason = "unsupported_extension";
|
|
return result;
|
|
}
|
|
|
|
const dl = await downloadFile(pb, source.collection, recordId, filename, tmpDir);
|
|
localPath = dl.localPath;
|
|
result.originalBytes = dl.bytes;
|
|
|
|
if (dl.bytes < SKIP_MIN_BYTES) {
|
|
result.action = "skipped";
|
|
result.skipReason = "too_small";
|
|
return result;
|
|
}
|
|
if (dl.bytes > SKIP_MAX_BYTES) {
|
|
result.action = "skipped";
|
|
result.skipReason = "too_large";
|
|
return result;
|
|
}
|
|
|
|
result.sourceHash = await sha256OfFile(localPath);
|
|
|
|
compressedPath = localPath + `.compressed.${kind === "pdf" ? "pdf" : "jpg"}`;
|
|
const cmp = kind === "pdf"
|
|
? await compressPdf(localPath, compressedPath)
|
|
: await compressImage(localPath, compressedPath);
|
|
if (!cmp.ok) {
|
|
result.action = "error";
|
|
result.error = cmp.killed
|
|
? `compression_timeout (${kind})`
|
|
: `compression_failed (${kind}): ${(cmp.stderr || cmp.stdout || "").slice(0, 200)}`;
|
|
return result;
|
|
}
|
|
|
|
const compressedStat = await stat(compressedPath);
|
|
result.compressedBytes = compressedStat.size;
|
|
result.ratio = +(result.compressedBytes / result.originalBytes).toFixed(3);
|
|
|
|
// Don't churn the URL for a 2% saving.
|
|
if (result.compressedBytes >= result.originalBytes * 0.98) {
|
|
result.action = "skipped";
|
|
result.skipReason = "no_meaningful_savings";
|
|
return result;
|
|
}
|
|
|
|
if (DRY_RUN) {
|
|
result.action = "dry_run";
|
|
return result;
|
|
}
|
|
|
|
const reup = await reuploadFile(pb, source, record, compressedPath, filename);
|
|
result.compressedHash = await sha256OfFile(compressedPath);
|
|
result.action = "compressed";
|
|
result.reupload = reup;
|
|
return result;
|
|
} catch (err) {
|
|
result.action = "error";
|
|
result.error = (err?.message || String(err)).slice(0, 500);
|
|
return result;
|
|
} finally {
|
|
for (const p of [localPath, compressedPath]) {
|
|
if (p) {
|
|
try { await unlink(p); } catch {}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
async function runWithConcurrency(items, limit, fn, onProgress) {
|
|
const results = new Array(items.length);
|
|
let next = 0;
|
|
let inFlight = 0;
|
|
return new Promise((resolve) => {
|
|
const launch = () => {
|
|
while (inFlight < limit && next < items.length) {
|
|
const i = next++;
|
|
inFlight++;
|
|
fn(items[i], i)
|
|
.then((r) => {
|
|
results[i] = r;
|
|
inFlight--;
|
|
onProgress?.(i, r);
|
|
if (next >= items.length && inFlight === 0) resolve(results);
|
|
else launch();
|
|
})
|
|
.catch((err) => {
|
|
results[i] = { action: "error", error: err?.message || String(err) };
|
|
inFlight--;
|
|
if (next >= items.length && inFlight === 0) resolve(results);
|
|
else launch();
|
|
});
|
|
}
|
|
};
|
|
launch();
|
|
});
|
|
}
|
|
|
|
async function loadAllRecordsWithFiles(pb, source, recordId) {
|
|
let records;
|
|
if (recordId) {
|
|
const r = await pb
|
|
.collection(source.collection)
|
|
.getOne(recordId)
|
|
.catch((err) => {
|
|
console.error(
|
|
` --record=${recordId}: not found in ${source.collection} (${err?.message || err})`,
|
|
);
|
|
return null;
|
|
});
|
|
records = r ? [r] : [];
|
|
} else {
|
|
records = await pb.collection(source.collection).getFullList({
|
|
batch: 500,
|
|
});
|
|
}
|
|
const out = [];
|
|
for (const r of records) {
|
|
const fieldRaw = r[source.field];
|
|
if (!fieldRaw) continue;
|
|
const arr = Array.isArray(fieldRaw) ? fieldRaw : [fieldRaw];
|
|
if (arr.length > 0) {
|
|
out.push({ id: r.id, [source.field]: arr });
|
|
}
|
|
}
|
|
return out;
|
|
}
|
|
|
|
async function main() {
|
|
if (!ADMIN_EMAIL || !ADMIN_PASSWORD) {
|
|
throw new Error("Set PB_ADMIN_EMAIL and PB_ADMIN_PASSWORD env vars.");
|
|
}
|
|
if (DRY_RUN) console.log("=== DRY RUN — no files will be re-uploaded ===");
|
|
|
|
// Validate argv flags BEFORE we authenticate, so a typo or missing
|
|
// value exits cleanly without a wasted PB admin login.
|
|
const sources = parseSources(process.argv);
|
|
const recordId = parseRecordId(process.argv);
|
|
|
|
const pb = new PocketBase(PB_URL);
|
|
// Disable auto-cancellation: the SDK cancels any in-flight request
|
|
// whenever a new one starts, which breaks Promise.all-style fan-out.
|
|
// We do several concurrent downloads/uploads so we want them all to
|
|
// land. Same pattern as src/lib/server/pocketbase-admin.ts.
|
|
pb.autoCancellation(false);
|
|
await pb.collection("_superusers").authWithPassword(ADMIN_EMAIL, ADMIN_PASSWORD);
|
|
console.log(`Authenticated against ${PB_URL}`);
|
|
console.log(`Concurrency: ${MAX_PARALLEL}, limit per source: ${LIMIT === Infinity ? "∞" : LIMIT}`);
|
|
console.log();
|
|
|
|
const tmpDir = await ensureDir(join(tmpdir(), `wfdash-compress-${process.pid}`));
|
|
|
|
const allResults = [];
|
|
if (sources.length < SOURCES.length) {
|
|
const labels = sources.map((s) => s.label).join(", ");
|
|
console.log(`Scoped to: ${labels}`);
|
|
}
|
|
if (recordId) console.log(`Scoped to record: ${recordId}`);
|
|
for (const source of sources) {
|
|
console.log(`\n=== ${source.label} ===`);
|
|
const t0 = Date.now();
|
|
const records = await loadAllRecordsWithFiles(pb, source, recordId);
|
|
console.log(` Found ${records.length} records with files`);
|
|
|
|
const work = [];
|
|
for (const r of records) {
|
|
const arr = Array.isArray(r[source.field]) ? r[source.field] : [r[source.field]];
|
|
for (const token of arr) {
|
|
if (!token) continue;
|
|
work.push({ record: r, token });
|
|
if (work.length >= LIMIT) break;
|
|
}
|
|
if (work.length >= LIMIT) break;
|
|
}
|
|
console.log(` ${work.length} file(s) to consider`);
|
|
|
|
let done = 0;
|
|
const results = await runWithConcurrency(
|
|
work,
|
|
MAX_PARALLEL,
|
|
(item) => processOne(pb, source, item.record, item.token, tmpDir),
|
|
() => {
|
|
done++;
|
|
if (done % 25 === 0 || done === work.length) {
|
|
process.stdout.write(` ...${done}/${work.length}\r`);
|
|
}
|
|
},
|
|
);
|
|
process.stdout.write("\n");
|
|
|
|
const summary = summarise(results);
|
|
const elapsed = ((Date.now() - t0) / 1000).toFixed(1);
|
|
console.log(` Took ${elapsed}s`);
|
|
console.log(` Compressed: ${summary.compressed} | Skipped: ${summary.skipped} | Errors: ${summary.errors}`);
|
|
console.log(` Bytes: ${bytes(summary.originalBytes)} → ${bytes(summary.compressedBytes)} (saved ${bytes(summary.savedBytes)}, ${summary.savedPct}%)`);
|
|
console.log(` Dry-run only: ${summary.dryRun}`);
|
|
if (summary.tokensChanged > 0) {
|
|
console.log(` ⚠️ Tokens changed (URLs invalidated): ${summary.tokensChanged}`);
|
|
}
|
|
allResults.push({ source: source.label, summary, results });
|
|
}
|
|
|
|
const grand = allResults.reduce(
|
|
(acc, r) => ({
|
|
compressed: acc.compressed + r.summary.compressed,
|
|
skipped: acc.skipped + r.summary.skipped,
|
|
errors: acc.errors + r.summary.errors,
|
|
dryRun: acc.dryRun + r.summary.dryRun,
|
|
tokensChanged: acc.tokensChanged + r.summary.tokensChanged,
|
|
originalBytes: acc.originalBytes + r.summary.originalBytes,
|
|
compressedBytes: acc.compressedBytes + r.summary.compressedBytes,
|
|
}),
|
|
{ compressed: 0, skipped: 0, errors: 0, dryRun: 0, tokensChanged: 0, originalBytes: 0, compressedBytes: 0 },
|
|
);
|
|
grand.savedBytes = grand.originalBytes - grand.compressedBytes;
|
|
grand.savedPct = grand.originalBytes > 0
|
|
? +((grand.savedBytes / grand.originalBytes) * 100).toFixed(1)
|
|
: 0;
|
|
|
|
console.log("\n=== GRAND TOTAL ===");
|
|
console.log(`Compressed: ${grand.compressed} | Skipped: ${grand.skipped} | Errors: ${grand.errors} | Dry-run: ${grand.dryRun}`);
|
|
console.log(`Bytes: ${bytes(grand.originalBytes)} → ${bytes(grand.compressedBytes)} (saved ${bytes(grand.savedBytes)}, ${grand.savedPct}%)`);
|
|
if (grand.tokensChanged > 0) {
|
|
console.log(`\n⚠️ Total tokens changed (URLs invalidated): ${grand.tokensChanged}`);
|
|
console.log(` PB appended a dedupe suffix to each new filename because the on-disk path already existed.`);
|
|
console.log(` Every URL built from these records' file fields just changed. Old links 404.`);
|
|
}
|
|
|
|
const date = todayStamp();
|
|
const jsonPath = `reports/attachment-compression-${date}.json`;
|
|
const mdPath = `reports/attachment-compression-${date}.md`;
|
|
await writeFile(jsonPath, JSON.stringify({ grand, sources: allResults }, null, 2));
|
|
await writeFile(mdPath, renderMarkdown(grand, allResults));
|
|
console.log(`\nReport written to ${jsonPath}`);
|
|
console.log(`Report written to ${mdPath}`);
|
|
}
|
|
|
|
function summarise(results) {
|
|
const s = { compressed: 0, skipped: 0, errors: 0, dryRun: 0, tokensChanged: 0, originalBytes: 0, compressedBytes: 0 };
|
|
for (const r of results) {
|
|
if (r.action === "compressed" || r.action === "dry_run") {
|
|
if (r.action === "compressed") s.compressed++;
|
|
else s.dryRun++;
|
|
s.originalBytes += r.originalBytes || 0;
|
|
s.compressedBytes += r.compressedBytes || 0;
|
|
if (r.reupload?.tokenChanged) s.tokensChanged++;
|
|
} else if (r.action === "skipped") {
|
|
s.skipped++;
|
|
if (r.originalBytes != null) s.originalBytes += r.originalBytes;
|
|
if (r.compressedBytes != null) s.compressedBytes += r.compressedBytes;
|
|
} else if (r.action === "error") {
|
|
s.errors++;
|
|
}
|
|
}
|
|
s.savedBytes = s.originalBytes - s.compressedBytes;
|
|
s.savedPct = s.originalBytes > 0
|
|
? +((s.savedBytes / s.originalBytes) * 100).toFixed(1)
|
|
: 0;
|
|
return s;
|
|
}
|
|
|
|
function renderMarkdown(grand, sources) {
|
|
let out = `# Attachment compression report — ${todayStamp()}\n\n`;
|
|
out += `_${DRY_RUN ? "DRY RUN — no files were re-uploaded" : "One-time bulk pass"}_\n\n`;
|
|
out += `## Grand total\n\n`;
|
|
out += `| Compressed | Skipped | Errors | Dry-run | Tokens changed |\n|---:|---:|---:|---:|---:|\n`;
|
|
out += `| ${grand.compressed} | ${grand.skipped} | ${grand.errors} | ${grand.dryRun} | ${grand.tokensChanged} |\n\n`;
|
|
out += `| Original | Compressed | Saved | % |\n|---:|---:|---:|---:|\n`;
|
|
out += `| ${bytes(grand.originalBytes)} | ${bytes(grand.compressedBytes)} | ${bytes(grand.savedBytes)} | ${grand.savedPct}% |\n\n`;
|
|
if (grand.tokensChanged > 0) {
|
|
out += `> ⚠️ **${grand.tokensChanged} token(s) changed.** PB appended dedupe suffixes to those filenames because the on-disk path already existed. Every URL built from these records' file fields is invalidated; old links 404.\n\n`;
|
|
}
|
|
|
|
out += `## Per-source\n\n`;
|
|
for (const s of sources) {
|
|
out += `### ${s.source}\n\n`;
|
|
out += `| Compressed | Skipped | Errors | Tokens changed | Original | Compressed | Saved | % |\n|---:|---:|---:|---:|---:|---:|---:|---:|\n`;
|
|
out += `| ${s.summary.compressed} | ${s.summary.skipped} | ${s.summary.errors} | ${s.summary.tokensChanged} | ${bytes(s.summary.originalBytes)} | ${bytes(s.summary.compressedBytes)} | ${bytes(s.summary.savedBytes)} | ${s.summary.savedPct}% |\n\n`;
|
|
|
|
const skipReasons = {};
|
|
const errorSamples = [];
|
|
for (const r of s.results) {
|
|
if (r.action === "skipped" && r.skipReason) {
|
|
skipReasons[r.skipReason] = (skipReasons[r.skipReason] || 0) + 1;
|
|
} else if (r.action === "error") {
|
|
if (errorSamples.length < 10) errorSamples.push(r);
|
|
}
|
|
}
|
|
if (Object.keys(skipReasons).length > 0) {
|
|
out += `**Skip reasons:**\n`;
|
|
for (const [reason, count] of Object.entries(skipReasons).sort((a, b) => b[1] - a[1])) {
|
|
out += `- \`${reason}\`: ${count}\n`;
|
|
}
|
|
out += `\n`;
|
|
}
|
|
if (errorSamples.length > 0) {
|
|
out += `**Errors (first 10):**\n`;
|
|
for (const e of errorSamples) {
|
|
out += `- \`${e.displayFilename || e.filename}\`: ${e.error}\n`;
|
|
}
|
|
out += `\n`;
|
|
}
|
|
}
|
|
return out;
|
|
}
|
|
|
|
main().catch((err) => {
|
|
console.error("FATAL:", err);
|
|
process.exit(1);
|
|
});
|