kanban-dev-webapp/scripts/backfill.ts

/**
 * One-off backfill helper.
 *
 * Not part of the product — this is a CLI shim that performs the same flow
 * the ingest core does, without pulling server-only modules. Run with:
 *   set -a && source .env.local && set +a
 *   npx tsx scripts/backfill.ts <projectId>
 */

import PocketBase from "pocketbase";

const PB_URL = process.env.NEXT_PUBLIC_PB_URL!;
const FORGEJO_URL = process.env.FORGEJO_URL!;
const PAGE_SIZE = 50;

// Conventional-commit parser (mirrors src/lib/git/conventional.ts)
function parseCommitMessage(message: string) {
  const firstLine = (message ?? "").split("\n")[0].trim();
  const match = /^([a-zA-Z]+)(?:\(([^)]+)\))?(!?):\s+(.*)$/.exec(firstLine);
  if (!match) {
    return { type: "other", scope: "", subject: firstLine.slice(0, 200), breaking: false };
  }
  const [, type, scope, bang, subject] = match;
  return {
    type: type.toLowerCase(),
    scope: (scope ?? "").toLowerCase().trim(),
    subject: subject.trim().slice(0, 200),
    breaking: bang === "!",
  };
}

async function fetchCommitPage(owner: string, repo: string, page: number) {
  const url =
    `${FORGEJO_URL}/api/v1/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/commits` +
    `?page=${page}&limit=${PAGE_SIZE}&stat=false&files=false&verification=false`;
  const response = await fetch(url, { headers: { accept: "application/json" }, cache: "no-store" });
  if (!response.ok) throw new Error(`Forgejo returned ${response.status}`);
  const commits = await response.json();
  return {
    commits,
    hasMore: response.headers.get("x-hasmore") === "true",
    pageCount: Number(response.headers.get("x-pagecount") ?? 1),
  };
}

async function main() {
  const projectId = process.argv[2];
  if (!projectId) throw new Error("Usage: tsx scripts/backfill.ts <projectId>");

  const pb = new PocketBase(PB_URL);
  pb.autoCancellation(false);
  await pb.collection("_superusers").authWithPassword(
    process.env.PB_SUPERUSER_EMAIL!,
    process.env.PB_SUPERUSER_PASSWORD!
  );

  const project = await pb.collection("projects").getOne(projectId);
  if (!project.repo_owner || !project.repo_name) {
    throw new Error("Project has no repo linked");
  }

  console.log(`Backfilling ${project.repo_owner}/${project.repo_name}...`);

  let page = 1;
  let created = 0;
  let skipped = 0;

  for (;;) {
    const result = await fetchCommitPage(project.repo_owner, project.repo_name, page);
    if (result.commits.length === 0) break;

    const shas = result.commits.map((c: { sha: string }) => `sha = "${c.sha}"`).join(" || ");
    const existing = await pb.collection("commits").getFullList({
      filter: `project = "${projectId}" && (${shas})`,
      fields: "sha",
    });
    const known = new Set(existing.map((r) => r.sha as string));

    for (const commit of result.commits as Array<{ sha: string; commit: { message: string; author: { name: string; email: string; date: string } }; parents?: Array<{ sha: string }>; stats?: { additions: number; deletions: number }; html_url: string }>) {
      if (known.has(commit.sha)) {
        skipped += 1;
        continue;
      }
      const parsed = parseCommitMessage(commit.commit.message);
      const isMerge = (commit.parents?.length ?? 0) > 1;
      await pb.collection("commits").create({
        project: projectId,
        sha: commit.sha,
        short_sha: commit.sha.slice(0, 7),
        message: commit.commit.message.slice(0, 50000),
        subject: parsed.subject,
        type: isMerge ? "merge" : parsed.type,
        scope: parsed.scope,
        author_name: commit.commit.author.name,
        author_email: commit.commit.author.email,
        authored_at: commit.commit.author.date,
        is_merge: isMerge,
        additions: commit.stats?.additions ?? 0,
        deletions: commit.stats?.deletions ?? 0,
        link_source: "none",
        url: commit.html_url,
      });
      created += 1;
    }

    console.log(`  page ${page}: ${result.commits.length} fetched, ${created} created so far`);

    if (!result.hasMore || page >= result.pageCount) break;
    page += 1;
  }

  await pb.collection("projects").update(projectId, {
    last_ingest_at: new Date().toISOString(),
  });

  console.log(`Done. created=${created} skipped=${skipped} pagesRead=${page}`);
}

main().catch((e) => {
  console.error(e);
  process.exit(1);
});