From 2ace4365e6191b20dfb706b5ac93b68d44ff5e95 Mon Sep 17 00:00:00 2001 From: DEls Date: Wed, 26 Aug 2026 12:44:02 +0200 Subject: [PATCH] feat(timesheets): enhance import functionality with site linking and employee creation - Updated the timesheet import route to support optional metadata fields for site linking and auto-creation of missing employees. - Enhanced parsing to detect multiple file formats, including Hills Casual Matrix and long-format CSV/XLSX. - Modified response structure to include detected format and created employees in the JSON response. - Improved UI components to allow project selection and options for creating new workers during import. - Refactored import logic to ensure a seamless user experience when handling timesheet uploads. --- src/app/api/timesheets/import/route.ts | 259 +++++++++++++----- src/app/dashboard/employees/page.tsx | 13 +- src/components/import-timesheets-button.tsx | 152 +++++++++-- src/lib/timesheet-casual-parse.test.ts | 138 ++++++++++ src/lib/timesheet-casual-parse.ts | 282 ++++++++++++++++++++ 5 files changed, 754 insertions(+), 90 deletions(-) create mode 100644 src/lib/timesheet-casual-parse.test.ts create mode 100644 src/lib/timesheet-casual-parse.ts diff --git a/src/app/api/timesheets/import/route.ts b/src/app/api/timesheets/import/route.ts index 2e8ff34..802e54a 100644 --- a/src/app/api/timesheets/import/route.ts +++ b/src/app/api/timesheets/import/route.ts @@ -2,39 +2,67 @@ import { NextResponse } from "next/server"; import Papa from "papaparse"; import * as XLSX from "xlsx"; import { pbAdmin } from "@/lib/pb"; -import { parseHillsTimesheet, type ParsedTimesheetRow } from "@/lib/timesheet-grid-parse"; +import { + parseHillsTimesheet, + type ParsedTimesheetRow, +} from "@/lib/timesheet-grid-parse"; +import { + looksLikeCasualMatrix, + parseCasualTimesheet, +} from "@/lib/timesheet-casual-parse"; /** * POST /api/timesheets/import * - * Accepts multipart/form-data with a single `file` field. Two formats: + * Accepts multipart/form-data with a single `file` field, plus optional + * metadata fields: * - * 1. Long-format CSV/XLSX/XLS (one row per employee per day): - * employee_no - matches EMPLOYEES.employee_no - * date - yyyy-mm-dd - * hours_worked - number + * siteId - when set, any missing employees get linked + * to this site on auto-create + * createMissingEmployees - "true"/"false". When true, employee_nos + * that don't exist yet are created as casual + * workers tied to the supplied siteId * - * 2. Hills-style wide-format XLSX/XLS (one row per employee, daily - * hours across columns). Parsed by `parseHillsTimesheet` and - * converted to the same long-format rows before insertion. + * Three formats are auto-detected from file contents (extension is just + * a hint): + * + * 1. Hills Casual Matrix (XLS/XLSX): rows show "Matrix View Report" — + * parser emits one long-format row per (employee, day). Hours use + * HH:MM, "Absent" cells are skipped, names with leading commas are + * cleaned. + * + * 2. Permanent wide format (XLSX): the existing `parseHillsTimesheet` + * parser — one row per employee, daily hours across columns. + * + * 3. Long-format CSV/XLSX: header row with employee_no, date, + * hours_worked. * * The route: - * 1. Detects the format from the file extension. - * 2. Parses to a uniform array of { employeeNo, date, hoursWorked }. - * 3. Creates one DATA_IMPORT_BATCHES record (batch_type=timesheets) - * and attaches the raw file for the audit trail. - * 4. Resolves each row's employee_no to the EMPLOYEES.id. - * 5. Inserts a TIMESHEETS row per valid input, linked to the batch. + * 1. Detects the format and parses to a uniform array of + * { employeeNo, date, hoursWorked }. + * 2. Optionally bulk-creates any missing employees (tied to siteId). + * 3. Creates one DATA_IMPORT_BATCHES record and attaches the raw + * file for the audit trail. + * 4. Inserts a TIMESHEETS row per valid input, linked to the batch. * - * Returns JSON: { batchId, totalRows, imported, skipped, errors[] }. + * Returns JSON: { batchId, totalRows, imported, skipped, errors[], + * detectedFormat, createdEmployees[] }. */ interface LongFormatRow { employeeNo: string; + name?: string; date: string; hoursWorked: number; } +interface ParseResult { + rows: LongFormatRow[]; + /** e.g. "casual-matrix", "permanent-wide", "long-format-csv" */ + detectedFormat: string; + warnings: string[]; +} + export async function POST(request: Request) { const form = await request.formData(); const file = form.get("file"); @@ -42,12 +70,13 @@ export async function POST(request: Request) { return NextResponse.json({ error: "No file provided" }, { status: 400 }); } - let longFormatRows: LongFormatRow[]; - let parseWarnings: string[] = []; + const siteId = String(form.get("siteId") ?? "").trim() || null; + const createMissingEmployees = + String(form.get("createMissingEmployees") ?? "").toLowerCase() === "true"; + + let parsed: ParseResult; try { - const result = await parseFile(file); - longFormatRows = result.rows; - parseWarnings = result.warnings; + parsed = await parseFile(file); } catch (e) { return NextResponse.json( { @@ -58,6 +87,26 @@ export async function POST(request: Request) { ); } + if (parsed.rows.length === 0) { + return NextResponse.json( + { + error: `Detected format: ${parsed.detectedFormat}, but no timesheet rows could be extracted.`, + }, + { status: 400 }, + ); + } + + // Optionally auto-create missing employees BEFORE inserting timesheets. + // We need them to exist so the timesheets have an employee.id to link to. + const createdEmployees: { employeeNo: string; name: string }[] = []; + if (createMissingEmployees && siteId) { + const created = await bulkCreateMissingEmployees( + parsed.rows, + siteId, + ); + createdEmployees.push(...created); + } + const pb = await pbAdmin(); // Create the batch with the original file attached. Timesheets stay @@ -73,23 +122,17 @@ export async function POST(request: Request) { attempts: 0, }); - // employee_no → id lookup - const allEmployees = await pb.collection("employees").getFullList({ - fields: "id,employee_no", - }); - const empByNo = new Map(); - for (const e of allEmployees) { - if (e.employee_no) empByNo.set(e.employee_no, e.id); - } + // employee_no → id lookup, rebuilt after any auto-create. + const empByNo = await loadEmployeeLookup(); let imported = 0; - const errors: { row: number; reason: string }[] = parseWarnings.map((w) => ({ + const errors: { row: number; reason: string }[] = parsed.warnings.map((w) => ({ row: 0, reason: w, })); - for (let i = 0; i < longFormatRows.length; i++) { - const row = longFormatRows[i]; + for (let i = 0; i < parsed.rows.length; i++) { + const row = parsed.rows[i]; const employeeId = empByNo.get(row.employeeNo); if (!employeeId) { errors.push({ @@ -117,52 +160,67 @@ export async function POST(request: Request) { return NextResponse.json({ batchId: batch.id, - totalRows: longFormatRows.length, + totalRows: parsed.rows.length, imported, skipped: errors.length, errors, + detectedFormat: parsed.detectedFormat, + createdEmployees, }); } -async function parseFile(file: File): Promise<{ - rows: LongFormatRow[]; - warnings: string[]; -}> { +/* ------------------------------------------------------------------ */ +/* Parsing */ +/* ------------------------------------------------------------------ */ + +async function parseFile(file: File): Promise { const name = file.name.toLowerCase(); - if (name.endsWith(".xlsx") || name.endsWith(".xls")) { + if (name.endsWith(".xlsx") || name.endsWith(".xls") || name.endsWith(".xlt")) { return parseSpreadsheet(file); } - // CSV (or anything else) - parse with Papa return parseCsv(file); } -async function parseSpreadsheet(file: File): Promise<{ - rows: LongFormatRow[]; - warnings: string[]; -}> { +async function parseSpreadsheet(file: File): Promise { const buffer = Buffer.from(await file.arrayBuffer()); const wb = XLSX.read(buffer, { type: "buffer" }); if (wb.SheetNames.length === 0) { throw new Error("Workbook has no sheets"); } - // Try every sheet - the first one that looks like a Hills wide-format - // sheet wins. If none do, fall back to long-format CSV-style parsing - // of the first sheet. + // Walk every sheet; first one we recognise as a known format wins. for (const sheetName of wb.SheetNames) { const sh = wb.Sheets[sheetName]; - const rows = XLSX.utils.sheet_to_json(sh, { header: 1, defval: null }); - const result = parseHillsTimesheet(rows as unknown[][], sheetName); - if (result.employeeRowsImported > 0) { + const rows = XLSX.utils.sheet_to_json(sh, { header: 1, defval: null }) as unknown[][]; + + if (looksLikeCasualMatrix(rows)) { + const result = parseCasualTimesheet(rows, sheetName); + if (result.rows.length > 0) { + return { + rows: result.rows.map((r) => ({ + employeeNo: r.employeeNo, + name: r.name, + date: r.date, + hoursWorked: r.hoursWorked, + })), + detectedFormat: "casual-matrix", + warnings: result.errors, + }; + } + } + + const wide = parseHillsTimesheet(rows, sheetName); + if (wide.employeeRowsImported > 0) { return { - rows: result.rows.map(toLongFormat), - warnings: result.errors, + rows: wide.rows.map(toLongFormat), + detectedFormat: "permanent-wide", + warnings: wide.errors, }; } } - // No Hills-shaped sheet found - fall back to the first sheet as - // long-format (employee_no/date/hours_worked header row). + // Neither wide parser matched — fall back to long-format header parsing + // of the first sheet. const sh = wb.Sheets[wb.SheetNames[0]]; const rows = XLSX.utils.sheet_to_json>(sh, { defval: null, @@ -177,16 +235,13 @@ async function parseSpreadsheet(file: File): Promise<{ } if (out.length === 0) { throw new Error( - `No Hills wide-format sheet found and no long-format rows in the workbook`, + "No casual-matrix, permanent-wide, or long-format rows found in the workbook", ); } - return { rows: out, warnings: [] }; + return { rows: out, detectedFormat: "long-format-xlsx", warnings: [] }; } -async function parseCsv(file: File): Promise<{ - rows: LongFormatRow[]; - warnings: string[]; -}> { +async function parseCsv(file: File): Promise { const text = await file.text(); const parsed = Papa.parse>(text, { header: true, @@ -195,7 +250,10 @@ async function parseCsv(file: File): Promise<{ }); if (parsed.errors.length > 0 && parsed.data.length === 0) { throw new Error( - `CSV could not be parsed: ${parsed.errors.slice(0, 3).map((e) => e.message).join("; ")}`, + `CSV could not be parsed: ${parsed.errors + .slice(0, 3) + .map((e) => e.message) + .join("; ")}`, ); } const out: LongFormatRow[] = []; @@ -206,7 +264,7 @@ async function parseCsv(file: File): Promise<{ if (!empNo || !date || !Number.isFinite(hours)) continue; out.push({ employeeNo: empNo.toUpperCase(), date, hoursWorked: hours }); } - return { rows: out, warnings: [] }; + return { rows: out, detectedFormat: "long-format-csv", warnings: [] }; } function toLongFormat(r: ParsedTimesheetRow): LongFormatRow { @@ -216,3 +274,82 @@ function toLongFormat(r: ParsedTimesheetRow): LongFormatRow { hoursWorked: r.hoursWorked, }; } + +/* ------------------------------------------------------------------ */ +/* Auto-create missing employees */ +/* ------------------------------------------------------------------ */ + +async function loadEmployeeLookup(): Promise> { + const pb = await pbAdmin(); + const all = await pb.collection("employees").getFullList({ + fields: "id,employee_no", + }); + const m = new Map(); + for (const e of all) { + if (e.employee_no) m.set(e.employee_no.toUpperCase(), e.id); + } + return m; +} + +/** + * Bulk-create any employee_nos from the parsed rows that don't yet + * exist in PocketBase. Each new worker is a casual tied to the supplied + * site, with sane defaults for fields we can't infer from the sheet. + * + * Returns the list of workers actually created (employee_no + name). + */ +async function bulkCreateMissingEmployees( + rows: LongFormatRow[], + siteId: string, +): Promise<{ employeeNo: string; name: string }[]> { + const pb = await pbAdmin(); + const existing = await loadEmployeeLookup(); + const seen = new Set(); + const toCreate = new Map(); // employeeNo → name + + for (const row of rows) { + const key = row.employeeNo.toUpperCase(); + if (seen.has(key)) continue; + seen.add(key); + if (existing.has(key)) continue; + if (toCreate.has(key)) continue; + toCreate.set(key, row.name ?? key); + } + + if (toCreate.size === 0) return []; + + // Look up the site so we can stamp project_code on the new workers. + let site: { project_code?: string } | null = null; + try { + site = await pb.collection("sites").getOne(siteId); + } catch { + // site was deleted between dialog open and submit — skip project_code + } + + const created: { employeeNo: string; name: string }[] = []; + for (const [employeeNo, name] of toCreate) { + try { + await pb.collection("employees").create({ + employee_no: employeeNo, + name, + worker_type: "casual", + site: siteId, + project_code: site?.project_code ?? "", + rate_per_hour: 0, + hours_per_day_standard: 9, + position: "", + }); + created.push({ employeeNo, name }); + } catch (err) { + // Swallow per-worker errors; they'll surface as "Unknown + // employee_no" on the timesheet insert and be counted under + // skipped. We don't fail the whole import. + console.warn( + `Failed to create employee ${employeeNo}:`, + err instanceof Error ? err.message : err, + ); + } + } + + return created; +} diff --git a/src/app/dashboard/employees/page.tsx b/src/app/dashboard/employees/page.tsx index 18e1da2..bd58ec6 100644 --- a/src/app/dashboard/employees/page.tsx +++ b/src/app/dashboard/employees/page.tsx @@ -99,9 +99,10 @@ function sumHoursInWindow( export default async function EmployeesPage() { const pb = await pbAdmin(); - const [employees, timesheets] = await Promise.all([ + const [employees, timesheets, sites] = await Promise.all([ pb.collection("employees").getFullList({ sort: "name", expand: "site" }), pb.collection("timesheets").getFullList(), + pb.collection("sites").getFullList({ sort: "name" }), ]); const hoursByEmployee = new Map(); @@ -165,7 +166,15 @@ export default async function EmployeesPage() { } + right={ + ({ + id: s.id, + name: s.name, + project_code: s.project_code, + }))} + /> + } /> (null); const [phase, setPhase] = React.useState("idle"); const [result, setResult] = React.useState(null); const [errorMsg, setErrorMsg] = React.useState(null); + const [siteId, setSiteId] = React.useState(""); + const [createMissingEmployees, setCreateMissingEmployees] = React.useState(true); const inputRef = React.useRef(null); const reset = React.useCallback(() => { @@ -43,17 +60,22 @@ export function ImportTimesheetsButton() { const close = () => { setOpen(false); - // Defer until the dialog animates out setTimeout(reset, 200); }; const upload = async () => { if (!file) return; + if (!siteId) { + setErrorMsg("Pick a project so we know which site these workers belong to"); + return; + } setPhase("uploading"); setErrorMsg(null); try { const form = new FormData(); form.append("file", file); + form.append("siteId", siteId); + form.append("createMissingEmployees", createMissingEmployees ? "true" : "false"); const res = await fetch("/api/timesheets/import", { method: "POST", body: form, @@ -81,6 +103,21 @@ export function ImportTimesheetsButton() { setErrorMsg(null); }; + const formatLabel = (fmt?: string) => { + switch (fmt) { + case "casual-matrix": + return "Hills casual matrix (auto-detected)"; + case "permanent-wide": + return "Permanent wide format (auto-detected)"; + case "long-format-csv": + return "Long-format CSV"; + case "long-format-xlsx": + return "Long-format XLSX"; + default: + return fmt ?? "Unknown"; + } + }; + return ( <>