/** * One-off backfill helper. * * Not part of the product — this is a CLI shim that performs the same flow * the ingest core does, without pulling server-only modules. Run with: * set -a && source .env.local && set +a * npx tsx scripts/backfill.ts */ import PocketBase from "pocketbase"; const PB_URL = process.env.NEXT_PUBLIC_PB_URL!; const FORGEJO_URL = process.env.FORGEJO_URL!; const PAGE_SIZE = 50; // Conventional-commit parser (mirrors src/lib/git/conventional.ts) function parseCommitMessage(message: string) { const firstLine = (message ?? "").split("\n")[0].trim(); const match = /^([a-zA-Z]+)(?:\(([^)]+)\))?(!?):\s+(.*)$/.exec(firstLine); if (!match) { return { type: "other", scope: "", subject: firstLine.slice(0, 200), breaking: false }; } const [, type, scope, bang, subject] = match; return { type: type.toLowerCase(), scope: (scope ?? "").toLowerCase().trim(), subject: subject.trim().slice(0, 200), breaking: bang === "!", }; } async function fetchCommitPage(owner: string, repo: string, page: number) { const url = `${FORGEJO_URL}/api/v1/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/commits` + `?page=${page}&limit=${PAGE_SIZE}&stat=false&files=false&verification=false`; const response = await fetch(url, { headers: { accept: "application/json" }, cache: "no-store" }); if (!response.ok) throw new Error(`Forgejo returned ${response.status}`); const commits = await response.json(); return { commits, hasMore: response.headers.get("x-hasmore") === "true", pageCount: Number(response.headers.get("x-pagecount") ?? 1), }; } async function main() { const projectId = process.argv[2]; if (!projectId) throw new Error("Usage: tsx scripts/backfill.ts "); const pb = new PocketBase(PB_URL); pb.autoCancellation(false); await pb.collection("_superusers").authWithPassword( process.env.PB_SUPERUSER_EMAIL!, process.env.PB_SUPERUSER_PASSWORD! ); const project = await pb.collection("projects").getOne(projectId); if (!project.repo_owner || !project.repo_name) { throw new Error("Project has no repo linked"); } console.log(`Backfilling ${project.repo_owner}/${project.repo_name}...`); let page = 1; let created = 0; let skipped = 0; for (;;) { const result = await fetchCommitPage(project.repo_owner, project.repo_name, page); if (result.commits.length === 0) break; const shas = result.commits.map((c) => `sha = "${c.sha}"`).join(" || "); const existing = await pb.collection("commits").getFullList({ filter: `project = "${projectId}" && (${shas})`, fields: "sha", }); const known = new Set(existing.map((r) => r.sha as string)); for (const commit of result.commits) { if (known.has(commit.sha)) { skipped += 1; continue; } const parsed = parseCommitMessage(commit.commit.message); const isMerge = (commit.parents?.length ?? 0) > 1; await pb.collection("commits").create({ project: projectId, sha: commit.sha, short_sha: commit.sha.slice(0, 7), message: commit.commit.message.slice(0, 50000), subject: parsed.subject, type: isMerge ? "merge" : parsed.type, scope: parsed.scope, author_name: commit.commit.author.name, author_email: commit.commit.author.email, authored_at: commit.commit.author.date, is_merge: isMerge, additions: commit.stats?.additions ?? 0, deletions: commit.stats?.deletions ?? 0, link_source: "none", url: commit.html_url, }); created += 1; } console.log(` page ${page}: ${result.commits.length} fetched, ${created} created so far`); if (!result.hasMore || page >= result.pageCount) break; page += 1; } await pb.collection("projects").update(projectId, { last_ingest_at: new Date().toISOString(), }); console.log(`Done. created=${created} skipped=${skipped} pagesRead=${page}`); } main().catch((e) => { console.error(e); process.exit(1); });