diff --git a/scripts/backfill.ts b/scripts/backfill.ts new file mode 100644 index 0000000..d7f92d3 --- /dev/null +++ b/scripts/backfill.ts @@ -0,0 +1,122 @@ +/** + * One-off backfill helper. + * + * Not part of the product — this is a CLI shim that performs the same flow + * the ingest core does, without pulling server-only modules. Run with: + * set -a && source .env.local && set +a + * npx tsx scripts/backfill.ts + */ + +import PocketBase from "pocketbase"; + +const PB_URL = process.env.NEXT_PUBLIC_PB_URL!; +const FORGEJO_URL = process.env.FORGEJO_URL!; +const PAGE_SIZE = 50; + +// Conventional-commit parser (mirrors src/lib/git/conventional.ts) +function parseCommitMessage(message: string) { + const firstLine = (message ?? "").split("\n")[0].trim(); + const match = /^([a-zA-Z]+)(?:\(([^)]+)\))?(!?):\s+(.*)$/.exec(firstLine); + if (!match) { + return { type: "other", scope: "", subject: firstLine.slice(0, 200), breaking: false }; + } + const [, type, scope, bang, subject] = match; + return { + type: type.toLowerCase(), + scope: (scope ?? "").toLowerCase().trim(), + subject: subject.trim().slice(0, 200), + breaking: bang === "!", + }; +} + +async function fetchCommitPage(owner: string, repo: string, page: number) { + const url = + `${FORGEJO_URL}/api/v1/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/commits` + + `?page=${page}&limit=${PAGE_SIZE}&stat=false&files=false&verification=false`; + const response = await fetch(url, { headers: { accept: "application/json" }, cache: "no-store" }); + if (!response.ok) throw new Error(`Forgejo returned ${response.status}`); + const commits = await response.json(); + return { + commits, + hasMore: response.headers.get("x-hasmore") === "true", + pageCount: Number(response.headers.get("x-pagecount") ?? 1), + }; +} + +async function main() { + const projectId = process.argv[2]; + if (!projectId) throw new Error("Usage: tsx scripts/backfill.ts "); + + const pb = new PocketBase(PB_URL); + pb.autoCancellation(false); + await pb.collection("_superusers").authWithPassword( + process.env.PB_SUPERUSER_EMAIL!, + process.env.PB_SUPERUSER_PASSWORD! + ); + + const project = await pb.collection("projects").getOne(projectId); + if (!project.repo_owner || !project.repo_name) { + throw new Error("Project has no repo linked"); + } + + console.log(`Backfilling ${project.repo_owner}/${project.repo_name}...`); + + let page = 1; + let created = 0; + let skipped = 0; + + for (;;) { + const result = await fetchCommitPage(project.repo_owner, project.repo_name, page); + if (result.commits.length === 0) break; + + const shas = result.commits.map((c) => `sha = "${c.sha}"`).join(" || "); + const existing = await pb.collection("commits").getFullList({ + filter: `project = "${projectId}" && (${shas})`, + fields: "sha", + }); + const known = new Set(existing.map((r) => r.sha as string)); + + for (const commit of result.commits) { + if (known.has(commit.sha)) { + skipped += 1; + continue; + } + const parsed = parseCommitMessage(commit.commit.message); + const isMerge = (commit.parents?.length ?? 0) > 1; + await pb.collection("commits").create({ + project: projectId, + sha: commit.sha, + short_sha: commit.sha.slice(0, 7), + message: commit.commit.message.slice(0, 50000), + subject: parsed.subject, + type: isMerge ? "merge" : parsed.type, + scope: parsed.scope, + author_name: commit.commit.author.name, + author_email: commit.commit.author.email, + authored_at: commit.commit.author.date, + is_merge: isMerge, + additions: commit.stats?.additions ?? 0, + deletions: commit.stats?.deletions ?? 0, + link_source: "none", + url: commit.html_url, + }); + created += 1; + } + + console.log(` page ${page}: ${result.commits.length} fetched, ${created} created so far`); + + if (!result.hasMore || page >= result.pageCount) break; + page += 1; + } + + await pb.collection("projects").update(projectId, { + last_ingest_at: new Date().toISOString(), + }); + + console.log(`Done. created=${created} skipped=${skipped} pagesRead=${page}`); +} + +main().catch((e) => { + console.error(e); + process.exit(1); +}); \ No newline at end of file