feat(quotes): add repeat-request detection service
Adds findDuplicateQuotes(), which groups quotes representing the same
underlying request. The business case is mining tendering: several
engineering companies each ask us to price the SAME item list because
they're all bidding for one spot on a mine, and the office needs to
find what the earlier quote went out at.
That inverts the usual dedupe rules, so both are deliberate:
- matching is CROSS-COMPANY (two customers sending the same list is
the signal, not a false positive)
- the email-body pass is CROSS-SENDER (bidders forward the same
tender document from different addresses)
Three passes: item_content (normalised line fingerprint, order
independent), email_body, and email_message_id.
Noise filtering is tuned against live data. The n8n intake writes an
"items from email request <name>" placeholder when it can't extract
line items; without excluding those, every unextracted email matches
every other one. A minimum line length drops junk like a bare "quote"
while keeping real one-liners the office genuinely re-quotes.
The completed-quote window measures `created`, not `updated`: a bulk
backfill rewrote `updated` on all 8,964 rows to a single timestamp, so
an `updated >=` cutoff matches the whole collection.
Not wired to the UI. A live run showed the quotes collection has no
trustworthy per-quote date (94% of `created` and 95% of `updated` are
migration stamps; quote_date and email_received_at are ~empty), so the
recency window can't currently bound the scan, and full-history content
matching returns groups of up to 53 for commonly-quoted commodities.
Landing the service on its own so the matching logic is available for
the on-demand "have we quoted this before?" lookup.
Also fixes a latent bug in the test PB mock's filter parser, which
stripped outer parens even when the leading "(" didn't match the
trailing ")", corrupting `(a) || (b)` into `a) || (b`.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
b262c23bf3
commit
93cb898f1e
2 changed files with 584 additions and 8 deletions
|
|
@ -41,20 +41,20 @@ const { pbMock, clearStore } = vi.hoisted(() => {
|
|||
const trimmed = s.trim();
|
||||
if (!trimmed.startsWith("(") || !trimmed.endsWith(")")) return s;
|
||||
let depth = 0;
|
||||
let lastCloseAt = -1;
|
||||
for (let i = 0; i < trimmed.length; i++) {
|
||||
if (trimmed[i] === "(") depth++;
|
||||
else if (trimmed[i] === ")") {
|
||||
depth--;
|
||||
lastCloseAt = i;
|
||||
// The paren opened at index 0 closed here. If that's not the
|
||||
// final character, the string is something like `(a) || (b)`
|
||||
// — the leading `(` and trailing `)` are DIFFERENT pairs, so
|
||||
// stripping them would corrupt the expression into
|
||||
// `a) || (b`. Only a close at the very end means the parens
|
||||
// genuinely wrap the whole thing.
|
||||
if (depth === 0 && i < trimmed.length - 1) return s;
|
||||
}
|
||||
}
|
||||
// Wrap the whole string iff depth is 0 at the end AND the
|
||||
// opening `(` is at position 0 AND the matching close is the
|
||||
// last char.
|
||||
if (depth !== 0) return s;
|
||||
if (lastCloseAt !== trimmed.length - 1) return s;
|
||||
return trimmed.slice(1, -1);
|
||||
return depth === 0 ? trimmed.slice(1, -1) : s;
|
||||
};
|
||||
|
||||
const parseFilter = (raw: string): FilterAST => {
|
||||
|
|
@ -272,6 +272,7 @@ import {
|
|||
submitQuote,
|
||||
SubmitQuoteError,
|
||||
setQuoteNotes,
|
||||
findDuplicateQuotes,
|
||||
} from "./quotes-service";
|
||||
|
||||
beforeEach(() => {
|
||||
|
|
@ -1943,6 +1944,222 @@ describe("getQuotesArchiveData", () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe("findDuplicateQuotes", () => {
|
||||
/** Helper: create a quote plus its line items in the mock store. */
|
||||
const seedQuote = async (
|
||||
quote: Record<string, unknown>,
|
||||
items: Array<Record<string, unknown>> = [],
|
||||
) => {
|
||||
await pbMock.collection("quotes").create(quote);
|
||||
for (const item of items) {
|
||||
await pbMock.collection("quote_items").create({
|
||||
quote: quote.id,
|
||||
...item,
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
it("flags the same item list quoted for two DIFFERENT companies", async () => {
|
||||
// The core mining-tender case: two bidders on the same mine spot
|
||||
// each send us the identical list. Cross-company matching is the
|
||||
// whole point, so this MUST group.
|
||||
await seedQuote(
|
||||
{ id: "q-a", quote_number: "Q-1", status: "new", created: "2026-08-01T00:00:00Z", company: "co-blyvoor" },
|
||||
[
|
||||
{ description: "M20 x 50mm Hex Bolt", quantity: 100 },
|
||||
{ description: "M20 Flat Washer", quantity: 200 },
|
||||
],
|
||||
);
|
||||
await seedQuote(
|
||||
{ id: "q-b", quote_number: "Q-2", status: "new", created: "2026-08-05T00:00:00Z", company: "co-cementation" },
|
||||
[
|
||||
{ description: "M20 x 50mm Hex Bolt", quantity: 100 },
|
||||
{ description: "M20 Flat Washer", quantity: 200 },
|
||||
],
|
||||
);
|
||||
|
||||
const groups = await findDuplicateQuotes();
|
||||
const content = groups.filter((g) => g.kind === "item_content");
|
||||
expect(content).toHaveLength(1);
|
||||
expect(content[0].records.map((r) => r.id)).toEqual(["q-a", "q-b"]);
|
||||
// Oldest first — that's the one to open and compare prices against.
|
||||
expect(content[0].records[0].quoteNumber).toBe("Q-1");
|
||||
expect(content[0].label).toBe("2 lines · 300 units");
|
||||
});
|
||||
|
||||
it("matches regardless of line order and trivial text differences", async () => {
|
||||
await seedQuote(
|
||||
{ id: "q-a", quote_number: "Q-1", status: "new", created: "2026-08-01T00:00:00Z" },
|
||||
[
|
||||
{ description: "M20 x 50mm Hex Bolt", quantity: 10 },
|
||||
{ description: "Grease Nipple", quantity: 5 },
|
||||
],
|
||||
);
|
||||
await seedQuote(
|
||||
{ id: "q-b", quote_number: "Q-2", status: "claimed", created: "2026-08-02T00:00:00Z" },
|
||||
[
|
||||
// Reversed order + different case/punctuation/spacing.
|
||||
{ description: "grease nipple.", quantity: 5 },
|
||||
{ description: "M20 X 50MM HEX BOLT", quantity: 10 },
|
||||
],
|
||||
);
|
||||
|
||||
const groups = await findDuplicateQuotes();
|
||||
expect(groups.filter((g) => g.kind === "item_content")).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("does not flag quotes whose quantities differ", async () => {
|
||||
// Same parts but a different amount is a different request —
|
||||
// the spec numbers matter in mining supply.
|
||||
await seedQuote(
|
||||
{ id: "q-a", quote_number: "Q-1", status: "new", created: "2026-08-01T00:00:00Z" },
|
||||
[{ description: "M20 x 50mm Hex Bolt", quantity: 100 }],
|
||||
);
|
||||
await seedQuote(
|
||||
{ id: "q-b", quote_number: "Q-2", status: "new", created: "2026-08-02T00:00:00Z" },
|
||||
[{ description: "M20 x 50mm Hex Bolt", quantity: 250 }],
|
||||
);
|
||||
|
||||
const groups = await findDuplicateQuotes();
|
||||
expect(groups.filter((g) => g.kind === "item_content")).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("does not flag quotes that have no line items", async () => {
|
||||
// Otherwise every empty draft would match every other empty draft.
|
||||
await seedQuote({ id: "q-a", quote_number: "Q-1", status: "new", created: "2026-08-01T00:00:00Z" });
|
||||
await seedQuote({ id: "q-b", quote_number: "Q-2", status: "new", created: "2026-08-02T00:00:00Z" });
|
||||
|
||||
const groups = await findDuplicateQuotes();
|
||||
expect(groups).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("flags the same email body sent from DIFFERENT senders", async () => {
|
||||
// Two bidders forwarding the same tender document from the mine.
|
||||
const body =
|
||||
"Please quote on the attached schedule for the Blyvoor shaft refurbishment, closing Friday 14h00.";
|
||||
await seedQuote(
|
||||
{
|
||||
id: "q-a", quote_number: "Q-1", status: "new", created: "2026-08-01T00:00:00Z",
|
||||
email_sender: "buyer@companyone.co.za", email_body: body,
|
||||
},
|
||||
[{ description: "Widget A", quantity: 1 }],
|
||||
);
|
||||
await seedQuote(
|
||||
{
|
||||
id: "q-b", quote_number: "Q-2", status: "new", created: "2026-08-02T00:00:00Z",
|
||||
email_sender: "procurement@companytwo.co.za", email_body: body,
|
||||
},
|
||||
[{ description: "Widget B", quantity: 2 }],
|
||||
);
|
||||
|
||||
const groups = await findDuplicateQuotes();
|
||||
const byBody = groups.filter((g) => g.kind === "email_body");
|
||||
expect(byBody).toHaveLength(1);
|
||||
expect(byBody[0].records.map((r) => r.id)).toEqual(["q-a", "q-b"]);
|
||||
});
|
||||
|
||||
it("ignores very short email bodies", async () => {
|
||||
// "please quote" would otherwise match half the collection.
|
||||
await seedQuote(
|
||||
{ id: "q-a", quote_number: "Q-1", status: "new", created: "2026-08-01T00:00:00Z", email_body: "please quote" },
|
||||
[{ description: "Widget A", quantity: 1 }],
|
||||
);
|
||||
await seedQuote(
|
||||
{ id: "q-b", quote_number: "Q-2", status: "new", created: "2026-08-02T00:00:00Z", email_body: "please quote" },
|
||||
[{ description: "Widget B", quantity: 2 }],
|
||||
);
|
||||
|
||||
const groups = await findDuplicateQuotes();
|
||||
expect(groups.filter((g) => g.kind === "email_body")).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("flags a true same-Message-ID double ingest", async () => {
|
||||
await seedQuote(
|
||||
{
|
||||
id: "q-a", quote_number: "Q-1", status: "new", created: "2026-08-01T00:00:00Z",
|
||||
email_message_id: "<abc@mail.example>",
|
||||
},
|
||||
[{ description: "Widget A", quantity: 1 }],
|
||||
);
|
||||
await seedQuote(
|
||||
{
|
||||
id: "q-b", quote_number: "Q-2", status: "new", created: "2026-08-02T00:00:00Z",
|
||||
email_message_id: "<abc@mail.example>",
|
||||
},
|
||||
[{ description: "Widget B", quantity: 2 }],
|
||||
);
|
||||
|
||||
const groups = await findDuplicateQuotes();
|
||||
expect(groups.filter((g) => g.kind === "email_message_id")).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("excludes completed quotes older than the scan window", async () => {
|
||||
const old = new Date(Date.now() - 80 * 24 * 60 * 60 * 1000).toISOString();
|
||||
const recent = new Date(Date.now() - 3 * 24 * 60 * 60 * 1000).toISOString();
|
||||
await seedQuote(
|
||||
{ id: "q-old", quote_number: "Q-OLD", status: "emailed", created: old, updated: old },
|
||||
[{ description: "M20 x 50mm Hex Bolt Galvanised", quantity: 10 }],
|
||||
);
|
||||
await seedQuote(
|
||||
{ id: "q-recent", quote_number: "Q-NEW", status: "emailed", created: recent, updated: recent },
|
||||
[{ description: "M20 x 50mm Hex Bolt Galvanised", quantity: 10 }],
|
||||
);
|
||||
|
||||
// Only the recent one is in scope, so there's no pair to group.
|
||||
const groups = await findDuplicateQuotes();
|
||||
expect(groups.filter((g) => g.kind === "item_content")).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("keeps active quotes in scope regardless of age", async () => {
|
||||
// A `new` quote that's been sitting for months is still live work.
|
||||
const old = new Date(Date.now() - 200 * 24 * 60 * 60 * 1000).toISOString();
|
||||
await seedQuote(
|
||||
{ id: "q-a", quote_number: "Q-1", status: "new", created: old, updated: old },
|
||||
[{ description: "M20 x 50mm Hex Bolt Galvanised", quantity: 10 }],
|
||||
);
|
||||
await seedQuote(
|
||||
{ id: "q-b", quote_number: "Q-2", status: "new", created: old, updated: old },
|
||||
[{ description: "M20 x 50mm Hex Bolt Galvanised", quantity: 10 }],
|
||||
);
|
||||
|
||||
const groups = await findDuplicateQuotes();
|
||||
expect(groups.filter((g) => g.kind === "item_content")).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("excludes templates", async () => {
|
||||
// Every quote spawned from a template would otherwise flag against it.
|
||||
await seedQuote(
|
||||
{ id: "q-tpl", quote_number: "Q-TPL", status: "new", created: "2026-08-01T00:00:00Z", is_template: true },
|
||||
[{ description: "M20 x 50mm Hex Bolt Galvanised", quantity: 10 }],
|
||||
);
|
||||
await seedQuote(
|
||||
{ id: "q-real", quote_number: "Q-1", status: "new", created: "2026-08-02T00:00:00Z" },
|
||||
[{ description: "M20 x 50mm Hex Bolt Galvanised", quantity: 10 }],
|
||||
);
|
||||
|
||||
const groups = await findDuplicateQuotes();
|
||||
expect(groups.filter((g) => g.kind === "item_content")).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("reports the earlier quote's total value so the operator can compare pricing", async () => {
|
||||
await seedQuote(
|
||||
{ id: "q-a", quote_number: "Q-1", status: "emailed", created: "2026-08-01T00:00:00Z" },
|
||||
[{ description: "M20 x 50mm Hex Bolt Galvanised", quantity: 10, selling_price: 15.5 }],
|
||||
);
|
||||
await seedQuote(
|
||||
{ id: "q-b", quote_number: "Q-2", status: "new", created: "2026-08-02T00:00:00Z" },
|
||||
[{ description: "M20 x 50mm Hex Bolt Galvanised", quantity: 10 }],
|
||||
);
|
||||
|
||||
const groups = await findDuplicateQuotes();
|
||||
const content = groups.filter((g) => g.kind === "item_content");
|
||||
expect(content).toHaveLength(1);
|
||||
expect(content[0].records[0].totalValue).toBe(155);
|
||||
// The new one is unpriced — 0 is correct, not a bug.
|
||||
expect(content[0].records[1].totalValue).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("submitQuote", () => {
|
||||
it("flips a new quote to approval_pending when every line is priced", async () => {
|
||||
const quote = await pbMock.collection("quotes").create({
|
||||
|
|
|
|||
|
|
@ -2902,3 +2902,362 @@ export async function setIsNoQuote(
|
|||
const pb = await getAdminPocketBase();
|
||||
return pb.collection(QUOTES_COLLECTION).update<Quote>(quoteId, { is_no_quote: value });
|
||||
}
|
||||
|
||||
/* ============================================================================
|
||||
* Repeat-request detection ("have we quoted this already?")
|
||||
* ----------------------------------------------------------------------------
|
||||
* This is NOT the usual data-hygiene dedupe (that's `findDuplicateOrders`
|
||||
* in orders-service.ts, which hunts for accidental double-intakes).
|
||||
*
|
||||
* The business case is mining supply tendering: when a mine puts a spot
|
||||
* out, SEVERAL different engineering companies each ask us to quote the
|
||||
* SAME item list, because they're all bidding for that spot. The office
|
||||
* needs to spot "we already priced this exact list for Blyvoor last
|
||||
* week" the moment the second company asks, so they can go look up what
|
||||
* they charged and price the new one deliberately.
|
||||
*
|
||||
* Two consequences for the matching rules, both of which invert the
|
||||
* usual dedupe instincts:
|
||||
*
|
||||
* 1. Matches are CROSS-COMPANY on purpose. Two different customers
|
||||
* sending the same list is the signal, not a false positive.
|
||||
* 2. The email-body pass is CROSS-SENDER for the same reason — the
|
||||
* bidders are usually forwarding the same tender document from
|
||||
* the mine, so the body text is identical from different
|
||||
* addresses.
|
||||
*
|
||||
* `email_message_id` is included for completeness (it's free once the
|
||||
* records are loaded) but is expected to fire almost never: the mail
|
||||
* workflow moves each processed email to a folder it doesn't re-scan,
|
||||
* so a true same-Message-ID double-ingest is rare.
|
||||
* ========================================================================== */
|
||||
|
||||
/**
|
||||
* How far back the "completed" slice of the scan reaches. Active work
|
||||
* (new / claimed / pending approval) is always scanned regardless of
|
||||
* age; completed quotes only stay in scope this long. Anything older
|
||||
* lives on /quotes/archive and isn't worth flagging.
|
||||
*
|
||||
* Deliberately its own constant rather than reusing
|
||||
* `COMPLETED_RECENT_WINDOW_DAYS` (7, the "Completed this week" tab) or
|
||||
* `SEARCH_RECENT_WINDOW_DAYS` (90, the search cutoff) — this window is
|
||||
* a business call about how long a tender stays relevant, not a
|
||||
* performance cutoff.
|
||||
*/
|
||||
export const DUPLICATE_SCAN_COMPLETED_WINDOW_DAYS = 50;
|
||||
|
||||
export function duplicateScanCompletedCutoff(now: Date = new Date()): string {
|
||||
return new Date(
|
||||
now.getTime() - DUPLICATE_SCAN_COMPLETED_WINDOW_DAYS * 24 * 60 * 60 * 1000,
|
||||
).toISOString();
|
||||
}
|
||||
|
||||
export type QuoteDuplicateKind =
|
||||
/** Same normalised line-item set (description + qty), any customer. */
|
||||
| "item_content"
|
||||
/** Same normalised inbound email body, any sender. */
|
||||
| "email_body"
|
||||
/** Same RFC 5322 Message-ID — a true double-ingest of one email. */
|
||||
| "email_message_id";
|
||||
|
||||
export interface DuplicateQuoteRecord {
|
||||
id: string;
|
||||
quoteNumber: string;
|
||||
status: string;
|
||||
created: string;
|
||||
customerName: string | null;
|
||||
companyName: string | null;
|
||||
itemCount: number;
|
||||
totalQuantity: number;
|
||||
/** Sum of selling_price × quantity. 0 while the quote is unpriced —
|
||||
* which is normal for the `new` end of the pipeline. The operator
|
||||
* uses this to compare what the earlier quote went out at. */
|
||||
totalValue: number;
|
||||
}
|
||||
|
||||
export interface DuplicateQuoteGroup {
|
||||
kind: QuoteDuplicateKind;
|
||||
/** Stable identity for the group (the fingerprint / message-id). */
|
||||
key: string;
|
||||
/** Human-readable summary, e.g. "3 lines · 120 units". */
|
||||
label: string;
|
||||
/** Oldest first — the first record is the original quote the
|
||||
* operator most likely wants to open and compare prices against. */
|
||||
records: DuplicateQuoteRecord[];
|
||||
}
|
||||
|
||||
/** Statuses in scope regardless of age — the live workflow. */
|
||||
const DUPLICATE_SCAN_ACTIVE_STATUSES: string[] = Array.from(
|
||||
new Set([
|
||||
...QUOTE_TAB_STATUSES.new,
|
||||
...QUOTE_TAB_STATUSES.claimed,
|
||||
...QUOTE_TAB_STATUSES.pending_approval,
|
||||
]),
|
||||
);
|
||||
|
||||
/**
|
||||
* Normalise a free-text line so trivial differences (case, spacing,
|
||||
* punctuation, the odd trailing full-stop) don't stop two identical
|
||||
* requests from matching. Deliberately conservative: it does NOT stem
|
||||
* words or drop numbers, because in mining supply the numbers ARE the
|
||||
* spec ("M20 x 50mm" must not collapse into "M20 x 60mm").
|
||||
*/
|
||||
function normaliseForFingerprint(raw: string): string {
|
||||
return (raw || "")
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9]+/g, " ")
|
||||
.trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* Line descriptions that carry no product information and must never
|
||||
* contribute to a fingerprint.
|
||||
*
|
||||
* The big one is `items from email request <name>` — a placeholder the
|
||||
* n8n intake writes when the extractor couldn't pull line items out of
|
||||
* the inbound email. It is NOT written by this codebase (it arrives as
|
||||
* data), and it is common: a live sample of 1,500 quotes had 30 rows
|
||||
* reading "items from email request request for a quote asap" alone.
|
||||
* Without this guard every unextracted email matches every other
|
||||
* unextracted email and the feature reports hundreds of phantom
|
||||
* repeats — which is exactly what the first live run did.
|
||||
*/
|
||||
const PLACEHOLDER_ITEM_PATTERNS: RegExp[] = [
|
||||
/^items? from email request\b/,
|
||||
/^(quote|quotation|rfq|please quote|pricing)$/,
|
||||
/^see attach(ed|ment)?\b/,
|
||||
/^as per (attached|email|below)\b/,
|
||||
];
|
||||
|
||||
/**
|
||||
* Minimum normalised length for a line to be fingerprintable. Tuned
|
||||
* against live data: real one-liners the office genuinely re-quotes
|
||||
* ("shell rimula r4x 15w40 20l" = 26 chars, "tpkt160508r gn yg612
|
||||
* milling insert p m k" = 41) sit comfortably above this, while junk
|
||||
* like a bare "quote" (5) falls out.
|
||||
*/
|
||||
const MIN_FINGERPRINT_LINE_LENGTH = 10;
|
||||
|
||||
/**
|
||||
* Fingerprint for the item-content pass: every line reduced to
|
||||
* `qty×description` (SKU folded in when present), sorted so line ORDER
|
||||
* doesn't matter, then joined. Two quotes with the same fingerprint
|
||||
* asked for the same things in the same amounts.
|
||||
*
|
||||
* Uses `description` (the customer's original RFQ wording) rather than
|
||||
* `quote_description` (the operator's cleaned-up PDF text) — the
|
||||
* original is what's identical across two bidders forwarding the same
|
||||
* tender; the cleaned version only exists after someone has worked the
|
||||
* quote.
|
||||
*
|
||||
* Returns null when nothing survives the noise filters, which
|
||||
* correctly excludes the quote from the pass rather than letting it
|
||||
* match every other content-free quote.
|
||||
*/
|
||||
function itemContentFingerprint(items: QuoteItem[]): string | null {
|
||||
const parts = items
|
||||
.map((it) => {
|
||||
const desc = normaliseForFingerprint(it.description || "");
|
||||
const sku = normaliseForFingerprint(it.sku || "");
|
||||
// A real SKU is itself a strong identifier, so a line with one
|
||||
// stays eligible even if its prose is thin.
|
||||
if (!sku && desc.length < MIN_FINGERPRINT_LINE_LENGTH) return null;
|
||||
if (!sku && PLACEHOLDER_ITEM_PATTERNS.some((re) => re.test(desc))) return null;
|
||||
const body = sku ? `${sku}|${desc}` : desc;
|
||||
if (!body) return null;
|
||||
const qty = Math.round(Number(it.quantity) || 1);
|
||||
return `${qty}x${body}`;
|
||||
})
|
||||
.filter((p): p is string => p !== null);
|
||||
// A quote with no usable line text can't be fingerprinted — otherwise
|
||||
// every empty draft would "match" every other empty draft.
|
||||
if (parts.length === 0) return null;
|
||||
parts.sort();
|
||||
return parts.join("~");
|
||||
}
|
||||
|
||||
/**
|
||||
* Find quotes that represent the same underlying request. See the
|
||||
* block comment above for why the matching is intentionally
|
||||
* cross-company.
|
||||
*
|
||||
* Returns one group per repeated fingerprint, each holding ≥2 quotes,
|
||||
* oldest first. Groups where every record is the same quote id are
|
||||
* impossible by construction (ids are unique), but groups are still
|
||||
* filtered to ≥2 DISTINCT quotes for safety.
|
||||
*/
|
||||
export async function findDuplicateQuotes(
|
||||
pbArg?: Awaited<ReturnType<typeof getAdminPocketBase>>,
|
||||
): Promise<DuplicateQuoteGroup[]> {
|
||||
const pb = pbArg || (await getAdminPocketBase());
|
||||
pb.autoCancellation(false);
|
||||
|
||||
// Scope: live workflow at any age, plus completed within the window.
|
||||
// Templates are excluded — they're scaffolding, and every quote spawned
|
||||
// from one template would otherwise flag against the template itself.
|
||||
//
|
||||
// The window is measured on `created`, NOT `updated`, even though
|
||||
// `updated` is the closer semantic proxy for "when was this
|
||||
// completed". On the live PB a bulk backfill rewrote `updated` on
|
||||
// every row (all 8,964 quotes share a 2026-08-11 06:08 timestamp),
|
||||
// so an `updated >=` cutoff matches the entire collection and the
|
||||
// window silently does nothing — the first live run pulled 8,146
|
||||
// quotes into a scan that should have seen a few hundred.
|
||||
// `created` is the intake date and is never rewritten, so it's the
|
||||
// only trustworthy recency signal here.
|
||||
const activeFilter = DUPLICATE_SCAN_ACTIVE_STATUSES
|
||||
.map((s) => `status = "${s}"`)
|
||||
.join(" || ");
|
||||
const completedFilter = QUOTE_TAB_STATUSES.completed
|
||||
.map((s) => `status = "${s}"`)
|
||||
.join(" || ");
|
||||
const filter =
|
||||
`is_template != true && ` +
|
||||
`((${activeFilter}) || ((${completedFilter}) && created >= "${duplicateScanCompletedCutoff()}"))`;
|
||||
|
||||
const quotes = await pb.collection(QUOTES_COLLECTION).getFullList<Quote>({
|
||||
filter,
|
||||
sort: "created",
|
||||
expand: "customer,company",
|
||||
batch: 5000,
|
||||
});
|
||||
if (quotes.length === 0) return [];
|
||||
|
||||
// One bulk fetch for every in-scope quote's items, then bucket in
|
||||
// memory. The orders equivalent loops one query per record; at ~850
|
||||
// in-scope quotes that would be ~850 sequential round-trips.
|
||||
const itemsByQuote = new Map<string, QuoteItem[]>();
|
||||
const allItems = await fetchQuoteItemsForQuotesAdmin(
|
||||
pb,
|
||||
quotes.map((q) => q.id),
|
||||
);
|
||||
for (const item of allItems) {
|
||||
if (!item.quote) continue;
|
||||
const arr = itemsByQuote.get(item.quote) ?? [];
|
||||
arr.push(item);
|
||||
itemsByQuote.set(item.quote, arr);
|
||||
}
|
||||
|
||||
const records = new Map<string, DuplicateQuoteRecord>();
|
||||
const contentKeys = new Map<string, string[]>();
|
||||
const bodyKeys = new Map<string, string[]>();
|
||||
const messageIdKeys = new Map<string, string[]>();
|
||||
|
||||
for (const q of quotes) {
|
||||
const items = itemsByQuote.get(q.id) ?? [];
|
||||
const totalQuantity = items.reduce(
|
||||
(sum, it) => sum + (Math.round(Number(it.quantity) || 1)),
|
||||
0,
|
||||
);
|
||||
const totalValue = items.reduce(
|
||||
(sum, it) =>
|
||||
sum + (Number(it.selling_price) || 0) * (Number(it.quantity) || 1),
|
||||
0,
|
||||
);
|
||||
records.set(q.id, {
|
||||
id: q.id,
|
||||
quoteNumber: q.quote_number,
|
||||
status: q.status,
|
||||
created: q.created,
|
||||
customerName: q.expand?.customer?.customer ?? null,
|
||||
companyName: q.expand?.company?.company ?? null,
|
||||
itemCount: items.length,
|
||||
totalQuantity,
|
||||
totalValue,
|
||||
});
|
||||
|
||||
const contentKey = itemContentFingerprint(items);
|
||||
if (contentKey) {
|
||||
const arr = contentKeys.get(contentKey) ?? [];
|
||||
arr.push(q.id);
|
||||
contentKeys.set(contentKey, arr);
|
||||
}
|
||||
|
||||
// Email body: normalised so quoted-reply chrome and whitespace
|
||||
// differences don't break the match. Very short bodies are skipped
|
||||
// — a two-word "please quote" would match half the collection.
|
||||
const bodyKey = normaliseForFingerprint(q.email_body || "");
|
||||
if (bodyKey.length >= 40) {
|
||||
const arr = bodyKeys.get(bodyKey) ?? [];
|
||||
arr.push(q.id);
|
||||
bodyKeys.set(bodyKey, arr);
|
||||
}
|
||||
|
||||
const messageId = (q.email_message_id || "").trim();
|
||||
if (messageId) {
|
||||
const arr = messageIdKeys.get(messageId) ?? [];
|
||||
arr.push(q.id);
|
||||
messageIdKeys.set(messageId, arr);
|
||||
}
|
||||
}
|
||||
|
||||
const out: DuplicateQuoteGroup[] = [];
|
||||
const pushGroups = (
|
||||
kind: QuoteDuplicateKind,
|
||||
keyed: Map<string, string[]>,
|
||||
label: (recs: DuplicateQuoteRecord[]) => string,
|
||||
) => {
|
||||
for (const [key, ids] of keyed) {
|
||||
const unique = Array.from(new Set(ids));
|
||||
if (unique.length < 2) continue;
|
||||
const recs = unique
|
||||
.map((id) => records.get(id))
|
||||
.filter((r): r is DuplicateQuoteRecord => r !== undefined);
|
||||
if (recs.length < 2) continue;
|
||||
recs.sort((a, b) => (a.created < b.created ? -1 : a.created > b.created ? 1 : 0));
|
||||
out.push({ kind, key, label: label(recs), records: recs });
|
||||
}
|
||||
};
|
||||
|
||||
pushGroups("item_content", contentKeys, (recs) => {
|
||||
const { itemCount, totalQuantity } = recs[0];
|
||||
return `${itemCount} line${itemCount === 1 ? "" : "s"} · ${totalQuantity} unit${totalQuantity === 1 ? "" : "s"}`;
|
||||
});
|
||||
pushGroups("email_body", bodyKeys, () => "Same email request");
|
||||
pushGroups("email_message_id", messageIdKeys, () => "Same email (Message-ID)");
|
||||
|
||||
// Biggest groups first — the most-repeated tender is the one the
|
||||
// office most wants to see.
|
||||
out.sort((a, b) => b.records.length - a.records.length);
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Admin-side bulk fetch of `quote_items` for many quotes. Mirrors
|
||||
* `fetchQuoteItemsForQuotes` in pb-helpers (client PB) and
|
||||
* `fetchCandidateQuoteItems` in quotes-search-service: try PB's `IN`
|
||||
* syntax first, fall back to chunked OR-chains when it's rejected.
|
||||
*/
|
||||
async function fetchQuoteItemsForQuotesAdmin(
|
||||
pb: Awaited<ReturnType<typeof getAdminPocketBase>>,
|
||||
quoteIds: string[],
|
||||
): Promise<QuoteItem[]> {
|
||||
if (quoteIds.length === 0) return [];
|
||||
const escape = (id: string) => id.replace(/"/g, '\\"');
|
||||
const safeIds = quoteIds.map(escape);
|
||||
const fields = "id,quote,description,sku,quantity,selling_price";
|
||||
try {
|
||||
return await pb.collection(QUOTE_ITEMS_COLLECTION).getFullList<QuoteItem>({
|
||||
filter: `quote IN (${safeIds.map((id) => `"${id}"`).join(",")})`,
|
||||
fields,
|
||||
batch: 5000,
|
||||
});
|
||||
} catch {
|
||||
// PB rejected the IN filter — fall back to chunked OR-chains.
|
||||
}
|
||||
const CHUNK = 80;
|
||||
const chunks: string[][] = [];
|
||||
for (let i = 0; i < safeIds.length; i += CHUNK) {
|
||||
chunks.push(safeIds.slice(i, i + CHUNK));
|
||||
}
|
||||
const results = await Promise.all(
|
||||
chunks.map((chunk) =>
|
||||
pb.collection(QUOTE_ITEMS_COLLECTION).getFullList<QuoteItem>({
|
||||
filter: chunk.map((id) => `quote = "${id}"`).join(" || "),
|
||||
fields,
|
||||
batch: 5000,
|
||||
}),
|
||||
),
|
||||
);
|
||||
return results.flat();
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue