diff --git a/apps/agent/agent/cli/local-campaign-worker.ts b/apps/agent/agent/cli/local-campaign-worker.ts new file mode 100644 index 000000000..408bab0ae --- /dev/null +++ b/apps/agent/agent/cli/local-campaign-worker.ts @@ -0,0 +1,126 @@ +import { writeFile } from "node:fs/promises"; +import { + localCampaignCsv, + localCampaignJsonl, +} from "../lib/local-campaign-export"; +import { + assertLocalCampaignOnly, + loadCampaignFile, + loadCampaignSeeds, + runLocalCampaign, +} from "../lib/local-campaign-runner"; +import { + assertLocalCampaignPath, + defaultLocalCampaignStagingPath, + readStagedLeads, +} from "../lib/local-campaign-staging"; + +function argument(args: string[], flag: string): string | undefined { + const index = args.indexOf(flag); + return ( + args + .find((value) => value.startsWith(`${flag}=`)) + ?.slice(flag.length + 1) ?? args[index + 1] + ); +} + +function numericArgument(args: string[], flag: string): number | undefined { + const value = argument(args, flag); + if (value === undefined) return undefined; + const parsed = Number(value); + if (!Number.isFinite(parsed) || parsed <= 0) { + throw new Error(`${flag} must be a positive number.`); + } + return Math.round(parsed); +} + +async function main(): Promise { + const args = process.argv.slice(2); + assertLocalCampaignOnly(args); + const command = args[0]; + if (command === "validate") { + const campaign = await loadCampaignFile(argument(args, "--campaign") ?? ""); + console.log( + JSON.stringify( + { valid: true, campaign_id: campaign.campaign_id }, + null, + 2, + ), + ); + return; + } + if (command === "run") { + const baseCampaign = await loadCampaignFile( + argument(args, "--campaign") ?? "", + ); + const campaign = { + ...baseCampaign, + max_source_units_per_page: + numericArgument(args, "--max-source-units-per-page") ?? + baseCampaign.max_source_units_per_page, + max_source_unit_chars: + numericArgument(args, "--max-source-unit-chars") ?? + baseCampaign.max_source_unit_chars, + max_ollama_calls_per_seed: + numericArgument(args, "--max-ollama-calls-per-seed") ?? + baseCampaign.max_ollama_calls_per_seed, + max_total_ollama_calls: + numericArgument(args, "--max-total-ollama-calls") ?? + baseCampaign.max_total_ollama_calls, + per_ollama_call_timeout_ms: + numericArgument(args, "--per-ollama-call-timeout-ms") ?? + baseCampaign.per_ollama_call_timeout_ms, + }; + const seedFile = argument(args, "--seed-file"); + const seeds = seedFile + ? await loadCampaignSeeds(seedFile) + : campaign.seed_urls; + const progress = args.includes("--progress"); + const result = await runLocalCampaign(campaign, seeds, { + stagingPath: argument(args, "--staging-path"), + seedStatusPath: argument(args, "--seed-status-path"), + perSeedTimeoutMs: numericArgument(args, "--per-seed-timeout-ms"), + onProgress: progress + ? (event) => console.error(JSON.stringify(event)) + : undefined, + }); + console.log(JSON.stringify({ mode: "dry-run", ...result }, null, 2)); + return; + } + if (command === "list") { + const path = + argument(args, "--staging-path") ?? defaultLocalCampaignStagingPath(); + console.log(JSON.stringify(await readStagedLeads(path), null, 2)); + return; + } + if (command === "export") { + const format = argument(args, "--format"); + const path = + argument(args, "--staging-path") ?? defaultLocalCampaignStagingPath(); + const leads = await readStagedLeads(path); + const output = argument(args, "--out"); + if (!output) throw new Error("Export needs --out."); + const safeOutput = assertLocalCampaignPath(output); + if (format === "jsonl") { + await writeFile(safeOutput, localCampaignJsonl(leads), "utf8"); + } else if (format === "csv") { + await writeFile(safeOutput, localCampaignCsv(leads), "utf8"); + } else { + throw new Error("Export format must be csv or jsonl."); + } + console.log( + JSON.stringify( + { exported: leads.length, output: safeOutput, format }, + null, + 2, + ), + ); + return; + } + throw new Error("Use validate, run, list, or export."); +} + +main().catch((error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; +}); diff --git a/apps/agent/agent/cli/local-lead-worker.ts b/apps/agent/agent/cli/local-lead-worker.ts new file mode 100644 index 000000000..b332851d3 --- /dev/null +++ b/apps/agent/agent/cli/local-lead-worker.ts @@ -0,0 +1,72 @@ +import { + extractLocalLead, + fetchLocalLeadSource, +} from "../lib/local-lead-extraction"; +import { + defaultLocalLeadStagingPath, + stageLocalLead, +} from "../lib/local-lead-staging"; + +function valueAfter(args: string[], flag: string): string | undefined { + const index = args.indexOf(flag); + return index >= 0 ? args[index + 1] : undefined; +} + +function valueEquals(args: string[], flag: string): string | undefined { + const prefix = `${flag}=`; + return args.find((arg) => arg.startsWith(prefix))?.slice(prefix.length); +} + +function argument(args: string[], flag: string): string | undefined { + return valueEquals(args, flag) ?? valueAfter(args, flag); +} + +function assertDryRun(args: string[]): void { + const forbidden = args.filter((arg) => + ["--sync", "--sync-crm", "--production", "--neon"].includes( + arg.split("=", 1)[0] ?? "", + ), + ); + if (forbidden.length > 0) { + throw new Error( + `CRM synchronization is not implemented. Refused: ${forbidden.join(", ")}`, + ); + } +} + +async function main(): Promise { + const args = process.argv.slice(2); + assertDryRun(args); + + const text = argument(args, "--text"); + const url = argument(args, "--url"); + if ((text ? 1 : 0) + (url ? 1 : 0) !== 1) { + throw new Error("Provide exactly one of --text or --url."); + } + + const source = url + ? await fetchLocalLeadSource(url) + : { + sourceUrl: + argument(args, "--source-url") ?? "https://local.invalid/sample", + text: text as string, + }; + const lead = await extractLocalLead(source.text, source.sourceUrl); + const staged = await stageLocalLead( + lead, + argument(args, "--staging-path") ?? defaultLocalLeadStagingPath(), + ); + + console.log( + JSON.stringify( + { mode: "dry-run", extracted: lead, staging: staged }, + null, + 2, + ), + ); +} + +main().catch((error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; +}); diff --git a/apps/agent/agent/lib/local-campaign-cache.ts b/apps/agent/agent/lib/local-campaign-cache.ts new file mode 100644 index 000000000..0858988d4 --- /dev/null +++ b/apps/agent/agent/lib/local-campaign-cache.ts @@ -0,0 +1,55 @@ +import { createHash } from "node:crypto"; +import { appendFile, mkdir, readFile } from "node:fs/promises"; +import { dirname } from "node:path"; +import { fetchLocalLeadSource } from "./local-lead-extraction"; +import { defaultLocalLeadStagingPath } from "./local-lead-staging"; + +export type CachedPage = { + url: string; + content_hash: string; + text: string; + fetched_at: string; +}; + +const cachePath = () => + defaultLocalLeadStagingPath().replace(/leads\.jsonl$/, "pages.jsonl"); + +export async function readCachedPage( + url: string, + path = cachePath(), +): Promise { + let raw = ""; + try { + raw = await readFile(path, "utf8"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; + throw error; + } + for (const line of raw.split("\n")) { + if (!line.trim()) continue; + const row = JSON.parse(line) as CachedPage; + if (row.url === url) return row; + } + return null; +} + +export async function fetchCachedPage( + url: string, + fetchImpl: typeof fetch = fetch, + path = cachePath(), +): Promise<{ page: CachedPage; cached: boolean }> { + const existing = await readCachedPage(url, path); + if (existing) return { page: existing, cached: true }; + const source = await fetchLocalLeadSource(url, fetchImpl); + const page: CachedPage = { + url, + content_hash: createHash("sha256") + .update(source.text, "utf8") + .digest("hex"), + text: source.text, + fetched_at: new Date().toISOString(), + }; + await mkdir(dirname(path), { recursive: true }); + await appendFile(path, `${JSON.stringify(page)}\n`, "utf8"); + return { page, cached: false }; +} diff --git a/apps/agent/agent/lib/local-campaign-export.ts b/apps/agent/agent/lib/local-campaign-export.ts new file mode 100644 index 000000000..b13b74c1b --- /dev/null +++ b/apps/agent/agent/lib/local-campaign-export.ts @@ -0,0 +1,23 @@ +import type { LocalStagedLead } from "./local-campaign-staging"; + +type CsvValue = string | string[] | number | boolean | null | undefined; + +function csvValue(value: CsvValue): string { + const text = value === null || value === undefined ? "" : String(value); + return /[,"\n]/.test(text) ? `"${text.replaceAll('"', '""')}"` : text; +} + +export function localCampaignJsonl(leads: LocalStagedLead[]): string { + return `${leads.map((lead) => JSON.stringify(lead)).join("\n")}\n`; +} + +export function localCampaignCsv(leads: LocalStagedLead[]): string { + const headers = leads[0] ? Object.keys(leads[0]) : []; + const rows = [ + headers, + ...leads.map((lead) => + headers.map((header) => lead[header as keyof typeof lead]), + ), + ]; + return `${rows.map((row) => row.map(csvValue).join(",")).join("\n")}\n`; +} diff --git a/apps/agent/agent/lib/local-campaign-extraction.ts b/apps/agent/agent/lib/local-campaign-extraction.ts new file mode 100644 index 000000000..8541a322d --- /dev/null +++ b/apps/agent/agent/lib/local-campaign-extraction.ts @@ -0,0 +1,176 @@ +import { createHash } from "node:crypto"; +import { ZodError } from "zod"; +import { + LOCAL_CAMPAIGN_SCALAR_FIELDS, + type LocalCampaignLead, + localCampaignLeadSchema, +} from "./local-campaign-lead-schema"; +import { LOCAL_LEAD_MODEL } from "./local-lead-extraction"; +import { createLocalOllamaClient, type OllamaClient } from "./local-ollama"; + +export const LOCAL_CAMPAIGN_MODEL = LOCAL_LEAD_MODEL; + +export type LocalCampaignLeadRecord = LocalCampaignLead & { + content_hash: string; + model_name: string; + latency_ms: number; + attempt_count: number; + created_at: string; +}; + +export function parseLocalCampaignResponse( + raw: string, + sourceUrl: string, + sourceText?: string, +): LocalCampaignLead { + const trimmed = raw.trim(); + if ( + trimmed.includes("```") || + !trimmed.startsWith("{") || + !trimmed.endsWith("}") + ) { + throw new Error("Campaign model response is not plain JSON."); + } + + const value = parsePlainJsonObject(trimmed, "Campaign model response"); + return parseLocalCampaignValue(value, sourceUrl, sourceText); +} + +function parsePlainJsonObject(raw: string, label: string): unknown { + try { + return JSON.parse(raw); + } catch { + throw new Error(`${label} is invalid JSON.`); + } +} + +function parseLocalCampaignValue( + value: unknown, + sourceUrl: string, + sourceText?: string, +): LocalCampaignLead { + const lead = localCampaignLeadSchema.parse(value); + const expectedMissing = LOCAL_CAMPAIGN_SCALAR_FIELDS.filter( + (field) => lead[field] === null, + ).sort(); + const actualMissing = [...lead.missing_fields].sort(); + if (JSON.stringify(expectedMissing) !== JSON.stringify(actualMissing)) { + throw new Error("missing_fields does not match null fields."); + } + if (lead.source_url !== sourceUrl) { + throw new Error("source_url does not match the requested source."); + } + if ( + sourceText && + !sourceText + .toLocaleLowerCase() + .includes(lead.evidence_excerpt.toLocaleLowerCase()) + ) { + throw new Error("evidence_excerpt is not present in source text."); + } + + for (const field of LOCAL_CAMPAIGN_SCALAR_FIELDS) { + const fieldValue = lead[field]; + if ( + fieldValue && + !lead.evidence_excerpt + .toLocaleLowerCase() + .includes(fieldValue.toLocaleLowerCase()) + ) { + throw new Error(`${field} is not present in evidence_excerpt.`); + } + } + return lead; +} + +function needsSchemaRepair(error: unknown): boolean { + return ( + error instanceof ZodError || + (error instanceof Error && + [ + "missing_fields does not match null fields.", + "evidence_excerpt is not present in source text.", + ].includes(error.message)) + ); +} + +function repairPrompt( + sourceUrl: string, + sourceText: string, + firstResponse: string, + reason: string, +): string { + return [ + "Your previous response was rejected by validation.", + "Return exactly one raw JSON object. Do not use markdown. Do not return an array.", + "Use exactly these keys: company_name, website, contact_name, role, email, phone, location, source_url, evidence_excerpt, project_fit_reason, missing_fields.", + "All unknown scalar fields must be null.", + "project_fit_reason must always be a non-empty string.", + "evidence_excerpt must always be a non-empty exact excerpt copied from the source text below.", + "If there is not enough evidence for company_name, website, contact_name, role, email, phone, or location, use null for that field.", + "Never invent a company, website, contact name, role, email, phone, or location.", + "missing_fields must exactly list scalar fields whose values are null.", + `source_url must be exactly ${sourceUrl}.`, + `Validation failure: ${reason}`, + "Rejected response:", + firstResponse, + "Source text:", + sourceText, + ].join("\n"); +} + +export async function extractLocalCampaignLead( + text: string, + sourceUrl: string, + client: OllamaClient = createLocalOllamaClient(), + options: { timeoutMs?: number } = {}, +): Promise { + if (!text.trim()) throw new Error("Source text is empty."); + const prompt = [ + "Return exactly one raw JSON object. Do not use markdown fences. Do not return an array.", + "Use exactly these keys: company_name, website, contact_name, role, email, phone, location, source_url, evidence_excerpt, project_fit_reason, missing_fields.", + "Use null for every unknown scalar field: company_name, website, contact_name, role, email, phone, location.", + "project_fit_reason must always be a non-empty string.", + "evidence_excerpt must always be a non-empty exact excerpt copied from the source text.", + "Use missing_fields for exactly the scalar fields that are null. Do not include evidence_excerpt, project_fit_reason, source_url, or missing_fields in missing_fields.", + "Never invent a company, contact, role, email, phone, website, or location.", + "Every non-null extracted scalar value must appear in evidence_excerpt.", + "If there is not enough evidence for a lead, return null for unknown scalar fields, use an exact source excerpt as evidence_excerpt, and explain the missing evidence in project_fit_reason.", + `The source URL is ${sourceUrl}. Return that exact value as source_url.`, + "Source text:", + text, + ].join("\n"); + const result = await client.generate({ + model: LOCAL_CAMPAIGN_MODEL, + prompt, + timeoutMs: options.timeoutMs, + }); + let lead: LocalCampaignLead; + let latencyMs = result.latencyMs; + let attemptCount = 1; + try { + lead = parseLocalCampaignResponse(result.text, sourceUrl, text); + } catch (error) { + if (!needsSchemaRepair(error)) throw error; + const retry = await client.generate({ + model: LOCAL_CAMPAIGN_MODEL, + prompt: repairPrompt(sourceUrl, text, result.text, errorReason(error)), + timeoutMs: options.timeoutMs, + }); + latencyMs += retry.latencyMs; + attemptCount = 2; + lead = parseLocalCampaignResponse(retry.text, sourceUrl, text); + } + return { + ...lead, + content_hash: createHash("sha256").update(text, "utf8").digest("hex"), + model_name: LOCAL_CAMPAIGN_MODEL, + latency_ms: latencyMs, + attempt_count: attemptCount, + created_at: new Date().toISOString(), + }; +} + +function errorReason(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} diff --git a/apps/agent/agent/lib/local-campaign-lead-schema.ts b/apps/agent/agent/lib/local-campaign-lead-schema.ts new file mode 100644 index 000000000..757170050 --- /dev/null +++ b/apps/agent/agent/lib/local-campaign-lead-schema.ts @@ -0,0 +1,29 @@ +import { z } from "zod"; + +export const localCampaignLeadSchema = z + .object({ + company_name: z.string().trim().min(1).nullable(), + website: z.string().trim().url().nullable(), + contact_name: z.string().trim().min(1).nullable(), + role: z.string().trim().min(1).nullable(), + email: z.string().trim().email().nullable(), + phone: z.string().trim().min(1).nullable(), + location: z.string().trim().min(1).nullable(), + source_url: z.string().trim().url(), + evidence_excerpt: z.string().trim().min(1).max(2_000), + project_fit_reason: z.string().trim().min(1).max(1_000), + missing_fields: z.array(z.string().trim().min(1)), + }) + .strict(); + +export type LocalCampaignLead = z.infer; + +export const LOCAL_CAMPAIGN_SCALAR_FIELDS = [ + "company_name", + "website", + "contact_name", + "role", + "email", + "phone", + "location", +] as const; diff --git a/apps/agent/agent/lib/local-campaign-runner.ts b/apps/agent/agent/lib/local-campaign-runner.ts new file mode 100644 index 000000000..f61085ef8 --- /dev/null +++ b/apps/agent/agent/lib/local-campaign-runner.ts @@ -0,0 +1,599 @@ +import { createHash } from "node:crypto"; +import { appendFile, mkdir, readFile } from "node:fs/promises"; +import { dirname } from "node:path"; +import { type CachedPage, fetchCachedPage } from "./local-campaign-cache"; +import type { LocalCampaignLeadRecord } from "./local-campaign-extraction"; +import { extractLocalCampaignLead } from "./local-campaign-extraction"; +import { + type LocalCampaign, + localCampaignSchema, + localCampaignSeedSchema, +} from "./local-campaign-schema"; +import { scoreLocalCampaignLead } from "./local-campaign-scoring"; +import { + DEFAULT_LOCAL_CAMPAIGN_SOURCE_UNIT_CAPS, + extractLocalCampaignSourceUnitCandidates, + hasUsefulPageEvidence, + type LocalCampaignSourceUnit, + type LocalCampaignSourceUnitRejectionReason, +} from "./local-campaign-source-units"; +import { + assertLocalCampaignPath, + type LocalStagedLead, + normalizeDomain, + stageCampaignLead, +} from "./local-campaign-staging"; +import type { OllamaClient } from "./local-ollama"; + +export function assertLocalCampaignOnly(args: string[]): void { + const forbidden = args.filter((arg) => + ["--sync", "--sync-crm", "--production", "--neon"].includes( + arg.split("=", 1)[0] ?? "", + ), + ); + if (forbidden.length > 0) { + throw new Error( + `CRM synchronization is not implemented. Refused: ${forbidden.join(", ")}`, + ); + } +} + +export type LocalCampaignRunDependencies = { + fetchImpl?: typeof fetch; + client?: OllamaClient; + pageLoader?: ( + url: string, + fetchImpl?: typeof fetch, + ) => Promise<{ + page: CachedPage; + cached: boolean; + }>; + stager?: typeof stageCampaignLead; + stagingPath?: string; + seedStatusPath?: string; + perSeedTimeoutMs?: number; + onProgress?: (event: LocalCampaignProgressEvent) => void; +}; + +export type LocalCampaignProgressEvent = { + event: + | "seed_started" + | "fetch_started" + | "fetch_completed" + | "fetch_failed" + | "cache_hit" + | "cache_miss" + | "source_units_found" + | "source_units_skipped" + | "ollama_call_started" + | "ollama_call_completed" + | "ollama_call_failed" + | "ollama_call_timed_out" + | "staged_lead_written" + | "seed_completed" + | "seed_failed"; + url: string; + elapsed_ms: number; + reason?: string; + source_unit_count?: number; + rejected_source_unit_count?: number; + rejection_reasons?: Record; + ollama_call_count?: number; +}; + +export type LocalCampaignSeedStatus = { + url: string; + status: "completed" | "failed" | "skipped"; + failure_reason: string | null; + elapsed_ms: number; + source_unit_count: number; + rejected_source_unit_count: number; + rejection_reasons: Record; + ollama_call_count: number; + staged_count: number; + created_at: string; +}; + +export type LocalCampaignRunResult = { + stagingPath: string; + processed: number; + fetched: number; + cached: number; + source_units: number; + rejected_source_units: number; + source_unit_rejection_reasons: Record< + LocalCampaignSourceUnitRejectionReason, + number + >; + ollama_calls: number; + rejected_by_url_safety: number; + failed_fetch: number; + extracted: number; + extraction_failures: number; + staged: number; + duplicates: number; + leads: LocalStagedLead[]; + failures: Array<{ + url: string; + stage: "url_safety" | "fetch" | "extraction" | "staging"; + reason: string; + }>; + seed_statuses: LocalCampaignSeedStatus[]; +}; + +function errorReason(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +function isUrlSafetyError(error: unknown): boolean { + const reason = errorReason(error); + return [ + "Local source URLs are blocked", + "local address", + "DNS lookup failed", + "Source URL must use HTTP or HTTPS", + ].some((marker) => reason.includes(marker)); +} + +function elapsedSince(startedAt: number): number { + return Math.round(performance.now() - startedAt); +} + +function isTimeoutReason(reason: string): boolean { + return reason.toLocaleLowerCase().includes("timed out"); +} + +async function writeSeedStatus( + status: LocalCampaignSeedStatus, + path?: string, +): Promise { + if (!path) return; + const safePath = assertLocalCampaignPath(path); + await mkdir(dirname(safePath), { recursive: true }); + await appendFile(safePath, `${JSON.stringify(status)}\n`, "utf8"); +} + +async function flushSeedStatus( + seedStatuses: LocalCampaignSeedStatus[], + status: LocalCampaignSeedStatus, + path?: string, +): Promise { + seedStatuses.push(status); + await writeSeedStatus(status, path); +} + +function extractionCandidates( + units: LocalCampaignSourceUnit[], + rejectedSourceUnitCount: number, + text: string, + url: string, + maxOllamaCallsPerSeed: number, + maxSourceUnitChars: number, +): Array<{ + text: string; + sourceUrl: string; + sourceUnit: LocalCampaignSourceUnit | null; +}> { + if (units.length > 0) { + return units.slice(0, maxOllamaCallsPerSeed).map((unit) => ({ + text: unit.candidate_text, + sourceUrl: unit.parent_source_url, + sourceUnit: unit, + })); + } + if (rejectedSourceUnitCount > 0) return []; + if (!hasUsefulPageEvidence(text)) return []; + return [ + { + text: text.slice(0, maxSourceUnitChars), + sourceUrl: url, + sourceUnit: null, + }, + ]; +} + +function lacksCompanyLevelEvidence( + lead: LocalCampaignLeadRecord, + sourceUnit: LocalCampaignSourceUnit | null, +): boolean { + return sourceUnit !== null && lead.company_name === null; +} + +function cacheCounters(wasCached: boolean): { + cached: number; + fetched: number; + event: "cache_hit" | "cache_miss"; +} { + if (wasCached) return { cached: 1, fetched: 0, event: "cache_hit" }; + return { cached: 0, fetched: 1, event: "cache_miss" }; +} + +function seedCompletionFailureReason( + stagedCount: number, + candidateCount: number, +): string | null { + if (stagedCount > 0) return null; + if (candidateCount > 0) + return "No source unit produced an accepted staged lead."; + return "No extraction candidates."; +} + +function emptyRejectionReasons(): Record< + LocalCampaignSourceUnitRejectionReason, + number +> { + return { + weak_website_only: 0, + event_vendor_link: 0, + navigation_or_footer: 0, + registration_or_ticketing: 0, + venue_or_organizer: 0, + boilerplate: 0, + insufficient_company_evidence: 0, + }; +} + +function addRejectionReasons( + target: Record, + source: Record, +): void { + for (const key of Object.keys( + source, + ) as LocalCampaignSourceUnitRejectionReason[]) { + target[key] += source[key]; + } +} + +export async function loadCampaignFile(path: string): Promise { + return localCampaignSchema.parse(JSON.parse(await readFile(path, "utf8"))); +} + +export async function loadCampaignSeeds(path: string): Promise { + const raw = await readFile(path, "utf8"); + const urls: string[] = []; + if (path.toLocaleLowerCase().endsWith(".csv")) { + const lines = raw.split(/\r?\n/).filter((line) => line.trim()); + if (lines.some((line) => line.includes('"'))) { + throw new Error( + "Quoted CSV fields are not supported. Use JSONL for complex seed data.", + ); + } + const header = + lines + .shift() + ?.split(",") + .map((value) => value.trim()) ?? []; + const urlIndex = header.findIndex((value) => + ["url", "source_url"].includes(value), + ); + if (urlIndex < 0) + throw new Error("Seed CSV needs a url or source_url column."); + for (const line of lines) { + const value = line.split(",")[urlIndex]?.trim(); + if (value) urls.push(localCampaignSeedSchema.parse({ url: value }).url); + } + return urls; + } + for (const line of raw.split(/\r?\n/).filter((value) => value.trim())) { + const parsed: unknown = JSON.parse(line); + urls.push(localCampaignSeedSchema.parse(parsed).url); + } + return urls; +} + +export async function runLocalCampaign( + campaign: LocalCampaign, + seedUrls: string[] = campaign.seed_urls, + dependencies: LocalCampaignRunDependencies = {}, +): Promise { + const uniqueSeeds = [...new Set(seedUrls)]; + const domains = new Map(); + const leads: LocalStagedLead[] = []; + const failures: LocalCampaignRunResult["failures"] = []; + const seedStatuses: LocalCampaignSeedStatus[] = []; + let fetched = 0; + let cached = 0; + let sourceUnits = 0; + let rejectedSourceUnits = 0; + const sourceUnitRejectionReasons = emptyRejectionReasons(); + let ollamaCalls = 0; + let rejectedByUrlSafety = 0; + let failedFetch = 0; + let extracted = 0; + let extractionFailures = 0; + let staged = 0; + let duplicates = 0; + let processed = 0; + const maxSourceUnitsPerPage = + campaign.max_source_units_per_page ?? + DEFAULT_LOCAL_CAMPAIGN_SOURCE_UNIT_CAPS.max_source_units_per_page; + const maxSourceUnitChars = + campaign.max_source_unit_chars ?? + DEFAULT_LOCAL_CAMPAIGN_SOURCE_UNIT_CAPS.max_source_unit_chars; + const maxOllamaCallsPerSeed = campaign.max_ollama_calls_per_seed ?? 5; + const maxTotalOllamaCalls = campaign.max_total_ollama_calls ?? 50; + const perOllamaCallTimeoutMs = campaign.per_ollama_call_timeout_ms ?? 45_000; + const perSeedTimeoutMs = dependencies.perSeedTimeoutMs ?? 90_000; + const progress = dependencies.onProgress; + for (const url of uniqueSeeds) { + const seedStartedAt = performance.now(); + let seedSourceUnitCount = 0; + let seedRejectedSourceUnitCount = 0; + let seedRejectionReasons = emptyRejectionReasons(); + let seedOllamaCalls = 0; + let seedStagedCount = 0; + const emit = ( + event: LocalCampaignProgressEvent["event"], + details: Omit< + LocalCampaignProgressEvent, + "event" | "url" | "elapsed_ms" + > = {}, + ) => { + progress?.({ + event, + url, + elapsed_ms: elapsedSince(seedStartedAt), + source_unit_count: seedSourceUnitCount, + rejected_source_unit_count: seedRejectedSourceUnitCount, + rejection_reasons: seedRejectionReasons, + ollama_call_count: seedOllamaCalls, + ...details, + }); + }; + const status = ( + value: LocalCampaignSeedStatus["status"], + failureReason: string | null, + ): LocalCampaignSeedStatus => ({ + url, + status: value, + failure_reason: failureReason, + elapsed_ms: elapsedSince(seedStartedAt), + source_unit_count: seedSourceUnitCount, + rejected_source_unit_count: seedRejectedSourceUnitCount, + rejection_reasons: seedRejectionReasons, + ollama_call_count: seedOllamaCalls, + staged_count: seedStagedCount, + created_at: new Date().toISOString(), + }); + emit("seed_started"); + if ( + leads.filter((lead) => lead.status !== "duplicate").length >= + campaign.max_companies + ) + break; + const domain = normalizeDomain(new URL(url).hostname); + const pageCount = domains.get(domain) ?? 0; + if (pageCount >= campaign.max_pages_per_domain) { + const reason = "Domain page cap reached."; + emit("seed_failed", { reason }); + await flushSeedStatus( + seedStatuses, + status("skipped", reason), + dependencies.seedStatusPath, + ); + continue; + } + domains.set(domain, pageCount + 1); + processed += 1; + let loaded: Awaited< + ReturnType> + >; + try { + emit("fetch_started"); + loaded = await (dependencies.pageLoader ?? fetchCachedPage)( + url, + dependencies.fetchImpl, + ); + emit("fetch_completed"); + } catch (error) { + const stage = isUrlSafetyError(error) ? "url_safety" : "fetch"; + const reason = errorReason(error); + if (stage === "url_safety") rejectedByUrlSafety += 1; + else failedFetch += 1; + failures.push({ url, stage, reason }); + emit("fetch_failed", { reason }); + emit("seed_failed", { reason }); + await flushSeedStatus( + seedStatuses, + status("failed", reason), + dependencies.seedStatusPath, + ); + continue; + } + const cache = cacheCounters(loaded.cached); + cached += cache.cached; + fetched += cache.fetched; + emit(cache.event); + const sourceUnitCandidates = extractLocalCampaignSourceUnitCandidates( + loaded.page.text, + url, + { + max_source_units_per_page: maxSourceUnitsPerPage, + max_source_unit_chars: maxSourceUnitChars, + }, + ); + const units = sourceUnitCandidates.units; + seedRejectedSourceUnitCount = sourceUnitCandidates.rejected.length; + seedRejectionReasons = emptyRejectionReasons(); + for (const rejection of sourceUnitCandidates.rejected) { + seedRejectionReasons[rejection.rejection_reason] += 1; + } + rejectedSourceUnits += seedRejectedSourceUnitCount; + addRejectionReasons(sourceUnitRejectionReasons, seedRejectionReasons); + seedSourceUnitCount = units.length; + sourceUnits += units.length; + emit("source_units_found", { + source_unit_count: units.length, + rejected_source_unit_count: seedRejectedSourceUnitCount, + rejection_reasons: seedRejectionReasons, + }); + const candidates = extractionCandidates( + units, + seedRejectedSourceUnitCount, + loaded.page.text, + url, + maxOllamaCallsPerSeed, + maxSourceUnitChars, + ); + if (units.length === 0 && candidates.length === 0) { + const reason = "No company-level source unit or useful page evidence."; + emit("source_units_skipped", { + reason, + rejected_source_unit_count: seedRejectedSourceUnitCount, + rejection_reasons: seedRejectionReasons, + }); + emit("seed_completed", { reason }); + await flushSeedStatus( + seedStatuses, + status("completed", reason), + dependencies.seedStatusPath, + ); + continue; + } + if (units.length > candidates.length) { + emit("source_units_skipped", { + reason: "Source unit cap limited model calls.", + }); + } + let seedTimedOut = false; + for (const candidate of candidates) { + if (ollamaCalls >= maxTotalOllamaCalls) break; + if (elapsedSince(seedStartedAt) >= perSeedTimeoutMs) { + const reason = "Seed timed out."; + emit("seed_failed", { reason }); + await flushSeedStatus( + seedStatuses, + status("failed", reason), + dependencies.seedStatusPath, + ); + seedTimedOut = true; + break; + } + let lead: LocalCampaignLeadRecord; + try { + ollamaCalls += 1; + seedOllamaCalls += 1; + emit("ollama_call_started"); + lead = await extractLocalCampaignLead( + candidate.text, + candidate.sourceUrl, + dependencies.client, + { timeoutMs: perOllamaCallTimeoutMs }, + ); + emit("ollama_call_completed"); + extracted += 1; + } catch (error) { + const reason = errorReason(error); + extractionFailures += 1; + emit( + isTimeoutReason(reason) + ? "ollama_call_timed_out" + : "ollama_call_failed", + { + reason, + }, + ); + failures.push({ + url: candidate.sourceUrl, + stage: "extraction", + reason, + }); + continue; + } + if (lacksCompanyLevelEvidence(lead, candidate.sourceUnit)) { + const reason = "Source unit did not produce company-level evidence."; + extractionFailures += 1; + emit("ollama_call_failed", { reason }); + failures.push({ + url: candidate.sourceUrl, + stage: "extraction", + reason, + }); + continue; + } + const scores = scoreLocalCampaignLead(lead, campaign, { + sourceUnit: candidate.sourceUnit !== null, + extractionMethod: candidate.sourceUnit?.extraction_method ?? null, + }); + const missingRequired = campaign.required_fields.filter( + (field) => lead[field as keyof typeof lead] === null, + ); + const record: LocalStagedLead = { + ...lead, + ...scores, + campaign_id: campaign.campaign_id, + lead_id: createHash("sha256") + .update(`${campaign.campaign_id}:${lead.content_hash}`, "utf8") + .digest("hex"), + source_domain: domain, + source_unit_id: candidate.sourceUnit?.unit_id, + parent_source_url: candidate.sourceUnit?.parent_source_url, + extraction_method: candidate.sourceUnit?.extraction_method, + source_unit_rank_score: candidate.sourceUnit?.rank_score, + source_unit_ranking_reasons: candidate.sourceUnit?.ranking_reasons, + status: missingRequired.length > 0 ? "needs_review" : "new", + }; + let result: Awaited>; + try { + result = await (dependencies.stager ?? stageCampaignLead)( + record, + dependencies.stagingPath, + ); + } catch (error) { + const reason = errorReason(error); + failures.push({ + url: candidate.sourceUrl, + stage: "staging", + reason, + }); + continue; + } + seedStagedCount += 1; + emit("staged_lead_written"); + leads.push(result.staged); + if (result.duplicate) duplicates += 1; + else staged += 1; + if ( + leads.filter((lead) => lead.status !== "duplicate").length >= + campaign.max_companies + ) { + break; + } + } + if (!seedTimedOut && !seedStatuses.some((entry) => entry.url === url)) { + const reason = seedCompletionFailureReason( + seedStagedCount, + candidates.length, + ); + emit(reason ? "seed_failed" : "seed_completed", { + reason: reason ?? undefined, + }); + await flushSeedStatus( + seedStatuses, + status(reason ? "failed" : "completed", reason), + dependencies.seedStatusPath, + ); + } + if (ollamaCalls >= maxTotalOllamaCalls) break; + } + return { + stagingPath: + dependencies.stagingPath ?? "var/local-leads/campaign-leads.jsonl", + processed, + fetched, + cached, + source_units: sourceUnits, + rejected_source_units: rejectedSourceUnits, + source_unit_rejection_reasons: sourceUnitRejectionReasons, + ollama_calls: ollamaCalls, + rejected_by_url_safety: rejectedByUrlSafety, + failed_fetch: failedFetch, + extracted, + extraction_failures: extractionFailures, + staged, + duplicates, + leads, + failures, + seed_statuses: seedStatuses, + }; +} diff --git a/apps/agent/agent/lib/local-campaign-schema.ts b/apps/agent/agent/lib/local-campaign-schema.ts new file mode 100644 index 000000000..c372d4e24 --- /dev/null +++ b/apps/agent/agent/lib/local-campaign-schema.ts @@ -0,0 +1,53 @@ +import { z } from "zod"; + +export const localCampaignSchema = z + .object({ + campaign_id: z.string().trim().min(1), + project: z.string().trim().min(1), + target_customer_type: z.string().trim().min(1), + geography: z.array(z.string().trim().min(1)), + event_window: z + .object({ start: z.string().date(), end: z.string().date() }) + .strict() + .optional(), + source_types: z.array(z.enum(["seed_url", "manual_list", "search"])), + seed_urls: z.array(z.string().trim().url()), + keywords: z.array(z.string().trim().min(1)), + exclusions: z.array(z.string().trim().min(1)), + max_companies: z.number().int().positive().max(10_000), + max_pages_per_domain: z.number().int().positive().max(100), + max_source_units_per_page: z.number().int().positive().max(500).optional(), + max_source_unit_chars: z.number().int().positive().max(10_000).optional(), + max_ollama_calls_per_seed: z.number().int().positive().max(100).optional(), + max_total_ollama_calls: z.number().int().positive().max(10_000).optional(), + per_ollama_call_timeout_ms: z + .number() + .int() + .positive() + .max(120_000) + .optional(), + required_fields: z.array(z.string().trim().min(1)), + scoring_rules: z + .object({ + fit_keyword_points: z.number().int().nonnegative().max(100), + evidence_points: z.number().int().nonnegative().max(100), + contact_field_points: z.number().int().nonnegative().max(100), + }) + .strict(), + }) + .strict(); + +export type LocalCampaign = z.infer; + +export const localCampaignSeedSchema = z + .object({ + url: z.string().trim().url().optional(), + source_url: z.string().trim().url().optional(), + }) + .passthrough() + .refine((seed) => seed.url ?? seed.source_url, { + message: "Seed needs url or source_url.", + }) + .transform((seed): { url: string } => { + return { url: seed.url ?? seed.source_url ?? "" }; + }); diff --git a/apps/agent/agent/lib/local-campaign-scoring.ts b/apps/agent/agent/lib/local-campaign-scoring.ts new file mode 100644 index 000000000..6d25531a7 --- /dev/null +++ b/apps/agent/agent/lib/local-campaign-scoring.ts @@ -0,0 +1,91 @@ +import type { LocalCampaignLead } from "./local-campaign-lead-schema"; +import type { LocalCampaign } from "./local-campaign-schema"; + +export type LocalCampaignScores = { + fit_score: number; + evidence_score: number; + contact_completeness_score: number; + outreach_readiness_score: number; + review_required_reason: string; +}; + +export function scoreLocalCampaignLead( + lead: LocalCampaignLead, + campaign: LocalCampaign, + options: { sourceUnit?: boolean; extractionMethod?: string | null } = {}, +): LocalCampaignScores { + const haystack = [ + lead.company_name, + lead.website, + lead.location, + lead.project_fit_reason, + lead.evidence_excerpt, + ] + .filter(Boolean) + .join(" ") + .toLocaleLowerCase(); + const matched = campaign.keywords.filter((keyword) => + haystack.includes(keyword.toLocaleLowerCase()), + ); + const excluded = campaign.exclusions.filter((term) => + haystack.includes(term.toLocaleLowerCase()), + ); + const fitScore = Math.max( + 0, + Math.min( + 100, + matched.length * campaign.scoring_rules.fit_keyword_points - + excluded.length * campaign.scoring_rules.fit_keyword_points, + ), + ); + const sourceUnitBonus = options.sourceUnit ? 20 : 0; + const methodBonus = + options.extractionMethod === "table_row" || + options.extractionMethod === "card" + ? 10 + : 0; + const titleOnlyPenalty = + lead.company_name === null && lead.website === null ? 30 : 0; + const evidenceScore = Math.max( + 0, + Math.min( + 100, + campaign.scoring_rules.evidence_points + + (lead.evidence_excerpt.length >= 40 ? 20 : 0) + + sourceUnitBonus + + methodBonus - + titleOnlyPenalty, + ), + ); + const contactFields = [ + lead.contact_name, + lead.role, + lead.email, + lead.phone, + ].filter((value) => value !== null).length; + const contactCompletenessScore = Math.min( + 100, + contactFields * campaign.scoring_rules.contact_field_points, + ); + const outreachReadinessScore = Math.round( + (contactCompletenessScore + evidenceScore) / 2, + ); + const reasons = [ + matched.length === 0 ? "No campaign keyword appears in evidence." : null, + excluded.length > 0 + ? `Excluded term appears: ${excluded.join(", ")}.` + : null, + lead.email === null ? "Email is missing." : null, + lead.phone === null ? "Phone is missing." : null, + lead.contact_name === null ? "Contact name is missing." : null, + options.sourceUnit ? null : "Evidence is page-level.", + ].filter((reason): reason is string => reason !== null); + return { + fit_score: fitScore, + evidence_score: evidenceScore, + contact_completeness_score: contactCompletenessScore, + outreach_readiness_score: outreachReadinessScore, + review_required_reason: + reasons.join(" ") || "Evidence supports campaign fit.", + }; +} diff --git a/apps/agent/agent/lib/local-campaign-source-units.ts b/apps/agent/agent/lib/local-campaign-source-units.ts new file mode 100644 index 000000000..569f0b38e --- /dev/null +++ b/apps/agent/agent/lib/local-campaign-source-units.ts @@ -0,0 +1,460 @@ +import { createHash } from "node:crypto"; +import { normalizeDomain } from "./local-campaign-staging"; + +export type LocalCampaignSourceUnit = { + unit_id: string; + parent_source_url: string; + source_domain: string; + candidate_company_name: string | null; + candidate_website: string | null; + candidate_location: string | null; + candidate_text: string; + evidence_excerpt: string; + extraction_method: "table_row" | "list_item" | "card" | "link_context"; + content_hash: string; + rank_score: number; + ranking_reasons: string[]; +}; + +export type LocalCampaignSourceUnitRejectionReason = + | "weak_website_only" + | "event_vendor_link" + | "navigation_or_footer" + | "registration_or_ticketing" + | "venue_or_organizer" + | "boilerplate" + | "insufficient_company_evidence"; + +export type LocalCampaignRejectedSourceUnit = { + parent_source_url: string; + source_domain: string; + candidate_text: string; + candidate_website: string | null; + extraction_method: LocalCampaignSourceUnit["extraction_method"]; + rejection_reason: LocalCampaignSourceUnitRejectionReason; +}; + +export type LocalCampaignSourceUnitCaps = { + max_source_units_per_page: number; + max_source_unit_chars: number; +}; + +const COMPANY_SUFFIX = + /\b(inc|llc|ltd|limited|corp|corporation|co|company|group|systems|solutions|technologies|technology|manufacturing|industrial|industries|machinery|automation|electric|electronics|displays|fixtures|metal|plastics|tooling|logistics|labs)\b/i; +const WEBSITE_PATTERN = /\bhttps?:\/\/[^\s"'<>]+|\bwww\.[^\s"'<>]+/i; +const EMAIL_PATTERN = /\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/i; +const PHONE_PATTERN = /(?:\+?1[\s.-]?)?(?:\(?\d{3}\)?[\s.-]?)\d{3}[\s.-]?\d{4}/; +const BOILERPLATE = + /\b(cookie|privacy|terms|register now|buy tickets|subscribe|newsletter|sponsor|navigation|menu|login|sign in|venue|floor plan|organizer|copyright|all rights reserved)\b/i; +const EVENT_LEVEL = + /\b(exhibitor directory|exhibitor list|trade show|conference|expo|event|venue|registration|attendee|speaker)\b/i; +const EXHIBITOR_PROFILE = /\b(exhibitor|profile|company|listing|directory)\b/i; +const BOOTH_PATTERN = + /\b(?:booth|stand|stall|pavilion)\s*(?:no\.?|number|#)?\s*[A-Z]?\d[\w.-]*\b/i; +const LOCATION_PATTERN = + /\b(?:[A-Z][a-z]+(?:\s+[A-Z][a-z]+)*,\s*)?(?:[A-Z]{2}|USA|United States|Canada|Mexico|Germany|Taiwan|China|Thailand|Vietnam|India|Japan|Korea|United Kingdom|UK|France|Italy|Spain)\b/; +const PRODUCT_SERVICE = + /\b(display|fixture|fabrication|manufacturing|supplier|distributor|automation|machinery|equipment|hardware|server|logistics|laboratory|lab|monitoring|energy|electrical|prototype|parts|services?|products?)\b/i; +const PROMOTIONAL = + /\b(register|registration|schedule|agenda|attendee|buy tickets|book now|learn more|sponsorship opportunity|floor plan|hotel|travel)\b/i; +const REGISTRATION_TICKETING = + /\b(register|registration|tickets?|buy tickets|book now|attendee registration|visitor registration|badge|passes)\b/i; +const NAVIGATION_FOOTER = + /\b(home|about us|contact us|contact-us|privacy|terms|cookie|newsletter|subscribe|login|sign in|menu|navigation|copyright|all rights reserved|media kit|advertise)\b/i; +const SOCIAL_OR_UTILITY_DOMAIN = + /\b(?:facebook|instagram|linkedin|twitter|x|youtube|tiktok|pinterest|google|apple|wa\.me|whatsapp)\.com\b/i; +const EVENT_VENDOR = + /\b(website by|powered by|designed by|marketing \+ guidance|amplify industrial marketing|aimg|map your show|expocad|cvent|eventbrite|swoogo|swapcard|bizzabo|event platform|marketing platform)\b/i; +const VENUE_OR_ORGANIZER = + /\b(venue|organizer|organiser|convention center|conference center|exhibition center|expo center|floor plan|show management|event management)\b/i; +const BLOCK_PATTERNS = [ + { pattern: /]*>([\s\S]*?)<\/tr>/gi, method: "table_row" as const }, + { pattern: /]*>([\s\S]*?)<\/li>/gi, method: "list_item" as const }, + { + pattern: + /<(?:article|section|div)\b[^>]*(?:class|id)=["'][^"']*(?:card|exhibitor|company|profile|listing|result)[^"']*["'][^>]*>([\s\S]*?)<\/(?:article|section|div)>/gi, + method: "card" as const, + }, +]; + +export const DEFAULT_LOCAL_CAMPAIGN_SOURCE_UNIT_CAPS: LocalCampaignSourceUnitCaps = + { + max_source_units_per_page: 20, + max_source_unit_chars: 1800, + }; + +export function extractLocalCampaignSourceUnits( + text: string, + parentSourceUrl: string, + caps: LocalCampaignSourceUnitCaps = DEFAULT_LOCAL_CAMPAIGN_SOURCE_UNIT_CAPS, +): LocalCampaignSourceUnit[] { + return extractLocalCampaignSourceUnitCandidates(text, parentSourceUrl, caps) + .units; +} + +export function extractLocalCampaignSourceUnitCandidates( + text: string, + parentSourceUrl: string, + caps: LocalCampaignSourceUnitCaps = DEFAULT_LOCAL_CAMPAIGN_SOURCE_UNIT_CAPS, +): { + units: LocalCampaignSourceUnit[]; + rejected: LocalCampaignRejectedSourceUnit[]; +} { + const sourceDomain = normalizeDomain(new URL(parentSourceUrl).hostname); + const units: LocalCampaignSourceUnit[] = []; + const rejected: LocalCampaignRejectedSourceUnit[] = []; + for (const entry of BLOCK_PATTERNS) { + for (const match of text.matchAll(entry.pattern)) { + const result = buildSourceUnit( + match[1] ?? "", + parentSourceUrl, + sourceDomain, + entry.method, + caps.max_source_unit_chars, + ); + if (result.unit) units.push(result.unit); + else if (result.rejected) rejected.push(result.rejected); + } + } + for (const match of text.matchAll( + /]*href=["']([^"']+)["'][^>]*>([\s\S]*?)<\/a>/gi, + )) { + const href = normalizeWebsite(match[1] ?? "", parentSourceUrl); + const label = cleanText(match[2] ?? ""); + const block = [label, href].filter(Boolean).join(" "); + const result = buildSourceUnit( + block, + parentSourceUrl, + sourceDomain, + "link_context", + caps.max_source_unit_chars, + href, + ); + if (result.unit) units.push(result.unit); + else if (result.rejected) rejected.push(result.rejected); + } + return { + units: dedupeSourceUnits(units) + .sort((left, right) => right.rank_score - left.rank_score) + .slice(0, caps.max_source_units_per_page), + rejected, + }; +} + +export function hasUsefulPageEvidence(text: string): boolean { + const cleaned = cleanText(text); + return looksCompanyLevel(cleaned) && !isOnlyEventLevel(cleaned); +} + +export type LocalCampaignSourceUnitRanking = { + rank_score: number; + ranking_reasons: string[]; +}; + +export function rankLocalCampaignSourceUnitText( + text: string, + context: { + parent_source_url?: string; + source_domain?: string; + candidate_company_name?: string | null; + candidate_website?: string | null; + candidate_location?: string | null; + extraction_method?: LocalCampaignSourceUnit["extraction_method"]; + } = {}, +): LocalCampaignSourceUnitRanking { + const cleaned = cleanText(text); + const parentDomain = context.source_domain ?? null; + const website = + context.candidate_website ?? + cleaned.match(WEBSITE_PATTERN)?.[0]?.replace(/[),.]+$/, "") ?? + null; + const company = + context.candidate_company_name ?? candidateCompanyName(cleaned); + const location = context.candidate_location ?? candidateLocation(cleaned); + const reasons: string[] = []; + let score = 0; + + if (company) { + score += 30; + reasons.push("likely company name"); + } + if (website) { + const websiteDomain = safeDomain(website); + if (websiteDomain && parentDomain && websiteDomain !== parentDomain) { + score += 30; + reasons.push("external company website"); + } else if (EXHIBITOR_PROFILE.test(website)) { + score += 18; + reasons.push("exhibitor profile URL"); + } else { + score += 12; + reasons.push("website or profile URL"); + } + } + if ( + context.extraction_method === "link_context" && + website && + EXHIBITOR_PROFILE.test(website) + ) { + score += 12; + reasons.push("exhibitor profile link"); + } + if (BOOTH_PATTERN.test(cleaned)) { + score += 20; + reasons.push("booth or stand number"); + } + if (location || LOCATION_PATTERN.test(cleaned)) { + score += 10; + reasons.push("location evidence"); + } + if (PRODUCT_SERVICE.test(cleaned)) { + score += 15; + reasons.push("product or service description"); + } + if (/\b(exhibitor|sponsor|booth|stand|pavilion)\b/i.test(cleaned)) { + score += 10; + reasons.push("exhibitor context"); + } + if ( + cleaned.length <= 80 && + EVENT_LEVEL.test(cleaned) && + !company && + !website + ) { + score -= 30; + reasons.push("page title or event-only text"); + } + if (isOnlyEventLevel(cleaned)) { + score -= 40; + reasons.push("event or venue level evidence"); + } + if (BOILERPLATE.test(cleaned)) { + score -= 35; + reasons.push("navigation/footer/cookie boilerplate"); + } + if (PROMOTIONAL.test(cleaned) && !company) { + score -= 25; + reasons.push("registration or promotional copy"); + } + + return { + rank_score: score, + ranking_reasons: + reasons.length > 0 + ? [...new Set(reasons)] + : ["weak source unit evidence"], + }; +} + +export function dedupeSourceUnits( + units: LocalCampaignSourceUnit[], +): LocalCampaignSourceUnit[] { + const seen = new Set(); + const result: LocalCampaignSourceUnit[] = []; + for (const unit of units) { + const key = [ + unit.candidate_website ? `site:${unit.candidate_website}` : null, + unit.candidate_company_name + ? `company:${unit.candidate_company_name.toLocaleLowerCase()}` + : null, + `hash:${unit.content_hash}`, + ] + .filter(Boolean) + .join("|"); + if (seen.has(key)) continue; + seen.add(key); + result.push(unit); + } + return result; +} + +function buildSourceUnit( + raw: string, + parentSourceUrl: string, + sourceDomain: string, + extractionMethod: LocalCampaignSourceUnit["extraction_method"], + maxChars: number, + websiteHint: string | null = null, +): { + unit: LocalCampaignSourceUnit | null; + rejected: LocalCampaignRejectedSourceUnit | null; +} { + const cleaned = cleanText(raw); + const candidateText = cleaned.slice(0, maxChars); + const company = candidateCompanyName(candidateText); + const website = + websiteHint ?? + candidateText.match(WEBSITE_PATTERN)?.[0]?.replace(/[),.]+$/, "") ?? + null; + const location = candidateLocation(candidateText); + const rejectionReason = sourceUnitRejectionReason(candidateText, { + candidate_company_name: company, + candidate_website: website, + candidate_location: location, + extraction_method: extractionMethod, + }); + if (rejectionReason) { + return { + unit: null, + rejected: { + parent_source_url: parentSourceUrl, + source_domain: sourceDomain, + candidate_text: candidateText, + candidate_website: website, + extraction_method: extractionMethod, + rejection_reason: rejectionReason, + }, + }; + } + const contentHash = createHash("sha256") + .update(`${parentSourceUrl}:${candidateText}`, "utf8") + .digest("hex"); + const ranking = rankLocalCampaignSourceUnitText(candidateText, { + parent_source_url: parentSourceUrl, + source_domain: sourceDomain, + candidate_company_name: company, + candidate_website: website, + candidate_location: location, + extraction_method: extractionMethod, + }); + return { + unit: { + unit_id: contentHash.slice(0, 24), + parent_source_url: parentSourceUrl, + source_domain: sourceDomain, + candidate_company_name: company, + candidate_website: website, + candidate_location: location, + candidate_text: candidateText, + evidence_excerpt: candidateText.slice(0, 500), + extraction_method: extractionMethod, + content_hash: contentHash, + ...ranking, + }, + rejected: null, + }; +} + +export function sourceUnitRejectionReason( + text: string, + context: { + candidate_company_name?: string | null; + candidate_website?: string | null; + candidate_location?: string | null; + extraction_method?: LocalCampaignSourceUnit["extraction_method"]; + } = {}, +): LocalCampaignSourceUnitRejectionReason | null { + const cleaned = cleanText(text); + const website = context.candidate_website ?? null; + const company = + context.candidate_company_name ?? candidateCompanyName(cleaned); + const location = context.candidate_location ?? candidateLocation(cleaned); + const hasWebsite = website !== null || WEBSITE_PATTERN.test(cleaned); + const hasBooth = BOOTH_PATTERN.test(cleaned); + const hasProduct = PRODUCT_SERVICE.test(cleaned); + const hasExhibitorContext = + /\b(exhibitor|booth|stand|pavilion)\b/i.test(cleaned) || + (context.extraction_method === "link_context" && + website !== null && + EXHIBITOR_PROFILE.test(website)); + + if (cleaned.length < 20) return "insufficient_company_evidence"; + if (EVENT_VENDOR.test(cleaned) || (website && EVENT_VENDOR.test(website))) { + return "event_vendor_link"; + } + if (website && SOCIAL_OR_UTILITY_DOMAIN.test(website)) { + return "navigation_or_footer"; + } + if (REGISTRATION_TICKETING.test(cleaned)) return "registration_or_ticketing"; + if (VENUE_OR_ORGANIZER.test(cleaned) && !company) return "venue_or_organizer"; + if (BOILERPLATE.test(cleaned) || NAVIGATION_FOOTER.test(cleaned)) { + return "boilerplate"; + } + if (isOnlyEventLevel(cleaned)) return "venue_or_organizer"; + if (!looksCompanyLevel(cleaned)) return "insufficient_company_evidence"; + if (!company) + return hasWebsite ? "weak_website_only" : "insufficient_company_evidence"; + if ( + !(hasWebsite || location || hasBooth || hasProduct || hasExhibitorContext) + ) { + return "insufficient_company_evidence"; + } + return null; +} + +function looksCompanyLevel(text: string): boolean { + if (text.length < 20 || BOILERPLATE.test(text)) return false; + return ( + COMPANY_SUFFIX.test(text) || + WEBSITE_PATTERN.test(text) || + EMAIL_PATTERN.test(text) || + PHONE_PATTERN.test(text) + ); +} + +function isOnlyEventLevel(text: string): boolean { + return ( + EVENT_LEVEL.test(text) && + !COMPANY_SUFFIX.test(text) && + !WEBSITE_PATTERN.test(text) + ); +} + +function candidateCompanyName(text: string): string | null { + const segments = text + .split(/\s{2,}|[|•]/) + .map((value) => value.trim()) + .filter(Boolean); + return ( + segments.find( + (segment) => COMPANY_SUFFIX.test(segment) && segment.length <= 120, + ) ?? null + ); +} + +function candidateLocation(text: string): string | null { + const match = text.match( + /\b(?:[A-Z][a-z]+(?:\s+[A-Z][a-z]+)*,\s*)?[A-Z]{2}\b/, + ); + return match?.[0] ?? null; +} + +function normalizeWebsite( + value: string, + parentSourceUrl: string, +): string | null { + if (!value || value.startsWith("#") || value.startsWith("mailto:")) + return null; + try { + return new URL(value, parentSourceUrl).toString(); + } catch { + return null; + } +} + +function safeDomain(value: string): string | null { + try { + return normalizeDomain(new URL(value).hostname); + } catch { + try { + return normalizeDomain(new URL(`https://${value}`).hostname); + } catch { + return null; + } + } +} + +function cleanText(value: string): string { + return value + .replace(//gi, " ") + .replace(//gi, " ") + .replace(/<[^>]+>/g, " ") + .replace(/ /g, " ") + .replace(/&/g, "&") + .replace(/</g, "<") + .replace(/>/g, ">") + .replace(/'/g, "'") + .replace(/"/g, '"') + .replace(/\s+/g, " ") + .trim(); +} diff --git a/apps/agent/agent/lib/local-campaign-staging.ts b/apps/agent/agent/lib/local-campaign-staging.ts new file mode 100644 index 000000000..c62b769d9 --- /dev/null +++ b/apps/agent/agent/lib/local-campaign-staging.ts @@ -0,0 +1,118 @@ +import { appendFile, mkdir, readFile } from "node:fs/promises"; +import { basename, dirname, isAbsolute, relative, resolve } from "node:path"; +import type { LocalCampaignLeadRecord } from "./local-campaign-extraction"; +import type { LocalCampaignScores } from "./local-campaign-scoring"; + +export type LocalCampaignStatus = + | "new" + | "duplicate" + | "needs_review" + | "approved" + | "rejected"; + +export type LocalStagedLead = LocalCampaignLeadRecord & + LocalCampaignScores & { + campaign_id: string; + lead_id: string; + source_domain: string; + source_unit_id?: string; + parent_source_url?: string; + extraction_method?: string; + source_unit_rank_score?: number; + source_unit_ranking_reasons?: string[]; + status: LocalCampaignStatus; + }; + +const AGENT_ROOT = resolve(dirname(new URL(import.meta.url).pathname), "../.."); +const DEFAULT_PATH = "var/local-leads/campaign-leads.jsonl"; + +export function defaultLocalCampaignStagingPath(): string { + return resolve(AGENT_ROOT, DEFAULT_PATH); +} + +export function assertLocalCampaignPath(path: string): string { + const root = resolve(AGENT_ROOT, "var/local-leads"); + const resolved = resolve(AGENT_ROOT, path); + const relativePath = relative(root, resolved); + if ( + !relativePath || + relativePath.startsWith("..") || + isAbsolute(relativePath) + ) { + throw new Error("Campaign path must stay under var/local-leads."); + } + if (basename(resolved).startsWith(".env") || resolved.includes("/.git/")) { + throw new Error("Campaign path cannot target configuration or Git files."); + } + return resolved; +} + +export async function readStagedLeads( + path = defaultLocalCampaignStagingPath(), +): Promise { + const safePath = assertLocalCampaignPath(path); + let raw = ""; + try { + raw = await readFile(safePath, "utf8"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return []; + throw error; + } + return raw + .split("\n") + .filter((line) => line.trim()) + .map((line) => JSON.parse(line) as LocalStagedLead); +} + +function dedupeKeys(lead: LocalStagedLead): string[] { + return [ + `domain:${normalizeDomain(lead.source_domain)}`, + lead.company_name ? `company:${normalizeText(lead.company_name)}` : null, + lead.email ? `email:${normalizeText(lead.email)}` : null, + lead.phone ? `phone:${normalizePhone(lead.phone)}` : null, + `hash:${lead.content_hash}`, + ].filter((key): key is string => key !== null); +} + +export function normalizeDomain(value: string): string { + return value + .toLocaleLowerCase() + .replace(/^www\./, "") + .replace(/\.$/, ""); +} + +function normalizeText(value: string): string { + return value + .toLocaleLowerCase() + .replace(/[^a-z0-9]+/g, " ") + .trim(); +} + +function normalizePhone(value: string): string { + return value.replace(/\D/g, ""); +} + +export function dedupeAgainst( + lead: LocalStagedLead, + existing: LocalStagedLead[], +): LocalStagedLead | null { + const keys = new Set(dedupeKeys(lead)); + return ( + existing.find((candidate) => + dedupeKeys(candidate).some((key) => keys.has(key)), + ) ?? null + ); +} + +export async function stageCampaignLead( + lead: LocalStagedLead, + path = defaultLocalCampaignStagingPath(), +): Promise<{ path: string; staged: LocalStagedLead; duplicate: boolean }> { + const safePath = assertLocalCampaignPath(path); + const existing = await readStagedLeads(safePath); + const duplicate = dedupeAgainst(lead, existing); + const staged = duplicate ? { ...lead, status: "duplicate" as const } : lead; + await mkdir(dirname(safePath), { recursive: true }); + await appendFile(safePath, `${JSON.stringify(staged)}\n`, "utf8"); + return { path: safePath, staged, duplicate: duplicate !== null }; +} diff --git a/apps/agent/agent/lib/local-lead-extraction.ts b/apps/agent/agent/lib/local-lead-extraction.ts new file mode 100644 index 000000000..cc02757ac --- /dev/null +++ b/apps/agent/agent/lib/local-lead-extraction.ts @@ -0,0 +1,301 @@ +import { createHash } from "node:crypto"; +import { lookup as resolveHostname } from "node:dns/promises"; +import { type LocalLead, localLeadSchema } from "./local-lead-schema"; +import { createLocalOllamaClient, type OllamaClient } from "./local-ollama"; + +export const LOCAL_LEAD_MODEL = "qwen2.5:14b"; +const DEFAULT_FETCH_TIMEOUT_MS = 15_000; +const DEFAULT_MAX_SOURCE_BYTES = 512_000; +const DEFAULT_DNS_TIMEOUT_MS = 5_000; + +export type DnsLookup = ( + hostname: string, + options: { all: true; verbatim: true }, +) => Promise>; + +export type LocalLeadRecord = LocalLead & { + model_name: string; + latency_ms: number; + content_hash: string; + parser_status: "accepted"; + attempt_count: number; + created_at: string; +}; + +export function parseLocalLeadResponse( + raw: string, + sourceUrl: string, +): LocalLead { + const trimmed = raw.trim(); + if ( + trimmed.includes("```") || + !trimmed.startsWith("{") || + !trimmed.endsWith("}") + ) { + throw new Error("Model response is not a plain JSON object."); + } + + let value: unknown; + try { + value = JSON.parse(trimmed); + } catch { + throw new Error("Model response is invalid JSON."); + } + + const lead = localLeadSchema.parse(value); + const nullFields = [ + "company_name", + "website", + "contact_name", + "role", + "email", + "phone", + ] as const; + const expectedMissing = nullFields.filter((field) => lead[field] === null); + const actualMissing = [...lead.missing_fields].sort(); + const sortedExpected = [...expectedMissing].sort(); + if (JSON.stringify(actualMissing) !== JSON.stringify(sortedExpected)) { + throw new Error("missing_fields does not match null fields."); + } + + if (lead.source_url !== sourceUrl) { + throw new Error("source_url does not match the requested source."); + } + + for (const field of [ + "company_name", + "contact_name", + "email", + "phone", + ] as const) { + const valueForEvidence = lead[field]; + if ( + valueForEvidence && + !lead.evidence_excerpt + .toLocaleLowerCase() + .includes(valueForEvidence.toLocaleLowerCase()) + ) { + throw new Error(`${field} is not present in evidence_excerpt.`); + } + } + + return lead; +} + +export async function extractLocalLead( + text: string, + sourceUrl: string, + client: OllamaClient = createLocalOllamaClient(), +): Promise { + if (!text.trim()) throw new Error("Source text is empty."); + + const prompt = [ + "Return exactly one plain JSON object. Do not use markdown fences.", + "Use these keys exactly: company_name, website, contact_name, role, email, phone, source_url, evidence_excerpt, confidence_reason, missing_fields.", + "Use null for every unknown scalar field.", + "Use missing_fields for exactly the scalar fields that are null.", + "Never invent an email, phone number, or contact name.", + "Every non-null company_name, contact_name, email, and phone must appear verbatim in evidence_excerpt.", + `The source URL is ${sourceUrl}. Return that exact value as source_url.`, + "Page text:", + text, + ].join("\n"); + + const result = await client.generate({ model: LOCAL_LEAD_MODEL, prompt }); + const lead = parseLocalLeadResponse(result.text, sourceUrl); + const contentHash = createHash("sha256").update(text, "utf8").digest("hex"); + + return { + ...lead, + model_name: LOCAL_LEAD_MODEL, + latency_ms: result.latencyMs, + content_hash: contentHash, + parser_status: "accepted", + attempt_count: 1, + created_at: new Date().toISOString(), + }; +} + +export async function fetchLocalLeadSource( + url: string, + fetchImpl: typeof fetch = fetch, + options: { + timeoutMs?: number; + maxBytes?: number; + dnsTimeoutMs?: number; + lookup?: DnsLookup; + } = {}, +): Promise<{ sourceUrl: string; text: string }> { + const parsed = new URL(url); + if (!["http:", "https:"].includes(parsed.protocol)) { + throw new Error("Source URL must use HTTP or HTTPS."); + } + if (isBlockedHostname(parsed.hostname)) { + throw new Error("Local source URLs are blocked."); + } + await assertPublicHostname( + parsed.hostname, + options.lookup ?? resolveHostname, + options.dnsTimeoutMs ?? DEFAULT_DNS_TIMEOUT_MS, + ); + + const controller = new AbortController(); + const timeoutMs = options.timeoutMs ?? DEFAULT_FETCH_TIMEOUT_MS; + let timeout: ReturnType | undefined; + const timeoutPromise = new Promise((_resolve, reject) => { + timeout = setTimeout(() => { + controller.abort(); + reject(new Error("Source fetch timed out.")); + }, timeoutMs); + }); + let response: Response; + try { + response = await Promise.race([ + fetchImpl(url, { + signal: controller.signal, + headers: { accept: "text/html,text/plain;q=0.9" }, + }), + timeoutPromise, + ]); + } catch (error) { + if (controller.signal.aborted) throw new Error("Source fetch timed out."); + throw error; + } finally { + if (timeout) clearTimeout(timeout); + } + if (!response.ok) throw new Error(`Source returned HTTP ${response.status}.`); + const maxBytes = options.maxBytes ?? DEFAULT_MAX_SOURCE_BYTES; + const contentLength = Number(response.headers.get("content-length")); + if (Number.isFinite(contentLength) && contentLength > maxBytes) { + throw new Error( + `Source exceeds the maximum content size of ${maxBytes} bytes.`, + ); + } + if (!response.body) throw new Error("Source returned no body."); + + const reader = response.body.getReader(); + const chunks: Uint8Array[] = []; + let size = 0; + try { + while (true) { + const next = await reader.read(); + if (next.done) break; + size += next.value.byteLength; + if (size > maxBytes) { + await reader.cancel(); + throw new Error( + `Source exceeds the maximum content size of ${maxBytes} bytes.`, + ); + } + chunks.push(next.value); + } + } finally { + reader.releaseLock(); + } + + const bytes = new Uint8Array(size); + let offset = 0; + for (const chunk of chunks) { + bytes.set(chunk, offset); + offset += chunk.byteLength; + } + return { sourceUrl: url, text: new TextDecoder().decode(bytes) }; +} + +async function assertPublicHostname( + hostname: string, + lookup: DnsLookup, + timeoutMs: number, +): Promise { + try { + let timeout: ReturnType | undefined; + const timeoutPromise = new Promise((_resolve, reject) => { + timeout = setTimeout( + () => reject(new Error("DNS lookup timed out.")), + timeoutMs, + ); + }); + const addresses = await Promise.race([ + lookup(hostname, { all: true, verbatim: true }), + timeoutPromise, + ]).finally(() => { + if (timeout) clearTimeout(timeout); + }); + if ( + addresses.length === 0 || + addresses.some((entry) => isBlockedHostname(entry.address)) + ) { + throw new Error("Source hostname resolves to a local address."); + } + } catch (error) { + if ( + error instanceof Error && + error.message === "Source hostname resolves to a local address." + ) { + throw error; + } + throw new Error("Source hostname DNS lookup failed."); + } +} + +function isBlockedHostname(hostname: string): boolean { + const host = hostname.toLocaleLowerCase().replace(/\.$/, ""); + const internalSuffixes = [ + ".localhost", + ".local", + ".internal", + ".lan", + ".home.arpa", + ]; + if ( + ["localhost", "internal", "intranet"].includes(host) || + internalSuffixes.some((suffix) => host.endsWith(suffix)) + ) { + return true; + } + + const parts = host.split(".").map(Number); + if (parts.length === 4 && parts.every(Number.isInteger)) { + const first = parts[0] ?? -1; + const second = parts[1] ?? -1; + return ( + first === 0 || + first === 10 || + first === 127 || + (first === 169 && second === 254) || + (first === 172 && second >= 16 && second <= 31) || + (first === 192 && second === 168) || + (first === 192 && second === 0) || + (first === 198 && second >= 18 && second <= 19) || + (first === 198 && second === 51) || + (first === 203 && second === 0) || + (first >= 224 && first <= 255) || + (first === 100 && second >= 64 && second <= 127) + ); + } + + if (host.includes(":")) { + return ( + host === "::" || + host.startsWith("ff") || + host.startsWith("2001:db8:") || + host.startsWith("::ffff:0.") || + host.startsWith("::ffff:100.") || + host.startsWith("::ffff:127.") || + host.startsWith("::ffff:169.254.") || + host.startsWith("::ffff:172.") || + host.startsWith("::ffff:192.0.") || + host === "::1" || + host.startsWith("fc") || + host.startsWith("fd") || + host.startsWith("fe8") || + host.startsWith("fe9") || + host.startsWith("fea") || + host.startsWith("feb") || + host.startsWith("::ffff:10.") || + host.startsWith("::ffff:192.168.") + ); + } + + return false; +} diff --git a/apps/agent/agent/lib/local-lead-schema.ts b/apps/agent/agent/lib/local-lead-schema.ts new file mode 100644 index 000000000..e6935b5f9 --- /dev/null +++ b/apps/agent/agent/lib/local-lead-schema.ts @@ -0,0 +1,31 @@ +import { z } from "zod"; + +export const localLeadSchema = z + .object({ + company_name: z.string().trim().min(1).nullable(), + website: z.string().trim().min(1).nullable(), + contact_name: z.string().trim().min(1).nullable(), + role: z.string().trim().min(1).nullable(), + email: z.string().trim().email().nullable(), + phone: z.string().trim().min(1).nullable(), + source_url: z.string().trim().url().nullable(), + evidence_excerpt: z.string().trim().min(1), + confidence_reason: z.string().trim().min(1), + missing_fields: z.array(z.string().trim().min(1)), + }) + .strict(); + +export type LocalLead = z.infer; + +export const LOCAL_LEAD_FIELDS = [ + "company_name", + "website", + "contact_name", + "role", + "email", + "phone", + "source_url", + "evidence_excerpt", + "confidence_reason", + "missing_fields", +] as const; diff --git a/apps/agent/agent/lib/local-lead-staging.ts b/apps/agent/agent/lib/local-lead-staging.ts new file mode 100644 index 000000000..f3db92b5b --- /dev/null +++ b/apps/agent/agent/lib/local-lead-staging.ts @@ -0,0 +1,55 @@ +import { appendFile, mkdir, readFile } from "node:fs/promises"; +import { basename, dirname, isAbsolute, relative, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import type { LocalLeadRecord } from "./local-lead-extraction"; + +const DEFAULT_STAGING_PATH = "var/local-leads/leads.jsonl"; +const LOCAL_AGENT_ROOT = resolve( + dirname(fileURLToPath(import.meta.url)), + "../..", +); + +export function defaultLocalLeadStagingPath(): string { + return resolve(LOCAL_AGENT_ROOT, DEFAULT_STAGING_PATH); +} + +export function assertLocalLeadStagingPath(path: string): string { + const resolved = resolve(LOCAL_AGENT_ROOT, path); + const allowed = resolve(LOCAL_AGENT_ROOT, "var/local-leads"); + const relativePath = relative(allowed, resolved); + if ( + !relativePath || + relativePath.startsWith("..") || + isAbsolute(relativePath) + ) { + throw new Error("Staging path must stay under var/local-leads."); + } + if (basename(resolved).startsWith(".env") || resolved.includes("/.git/")) { + throw new Error("Staging path cannot target configuration or Git files."); + } + return resolved; +} + +export async function stageLocalLead( + lead: LocalLeadRecord, + path = defaultLocalLeadStagingPath(), +): Promise<{ path: string; staged: boolean }> { + const safePath = assertLocalLeadStagingPath(path); + let existing = ""; + try { + existing = await readFile(safePath, "utf8"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + + for (const line of existing.split("\n")) { + if (!line.trim()) continue; + const row = JSON.parse(line) as { content_hash?: unknown }; + if (row.content_hash === lead.content_hash) + return { path: safePath, staged: false }; + } + + await mkdir(dirname(safePath), { recursive: true }); + await appendFile(safePath, `${JSON.stringify(lead)}\n`, "utf8"); + return { path: safePath, staged: true }; +} diff --git a/apps/agent/agent/lib/local-ollama.ts b/apps/agent/agent/lib/local-ollama.ts new file mode 100644 index 000000000..8cd37d2d6 --- /dev/null +++ b/apps/agent/agent/lib/local-ollama.ts @@ -0,0 +1,73 @@ +const OLLAMA_URL = "http://127.0.0.1:11434/api/generate"; +const DEFAULT_TIMEOUT_MS = 120_000; + +export type OllamaGeneration = { + text: string; + latencyMs: number; +}; + +export type OllamaClient = { + generate(input: { + model: string; + prompt: string; + timeoutMs?: number; + }): Promise; +}; + +export function createLocalOllamaClient( + fetchImpl: typeof fetch = fetch, + endpoint = OLLAMA_URL, +): OllamaClient { + return { + async generate({ model, prompt, timeoutMs = DEFAULT_TIMEOUT_MS }) { + const startedAt = performance.now(); + const controller = new AbortController(); + let timeout: ReturnType | undefined; + const timeoutPromise = new Promise((_resolve, reject) => { + timeout = setTimeout(() => { + controller.abort(); + reject(new Error("Ollama request timed out.")); + }, timeoutMs); + }); + let response: Response; + try { + response = await Promise.race([ + fetchImpl(endpoint, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model, + prompt, + format: "json", + stream: false, + options: { temperature: 0 }, + }), + signal: controller.signal, + }), + timeoutPromise, + ]); + } catch (error) { + if (controller.signal.aborted) { + throw new Error("Ollama request timed out."); + } + throw error; + } finally { + if (timeout) clearTimeout(timeout); + } + + if (!response.ok) { + throw new Error(`Ollama returned HTTP ${response.status}.`); + } + + const body = (await response.json()) as { response?: unknown }; + if (typeof body.response !== "string") { + throw new Error("Ollama returned no text response."); + } + + return { + text: body.response, + latencyMs: Math.round(performance.now() - startedAt), + }; + }, + }; +} diff --git a/apps/agent/campaigns/README.md b/apps/agent/campaigns/README.md new file mode 100644 index 000000000..27e8c8454 --- /dev/null +++ b/apps/agent/campaigns/README.md @@ -0,0 +1,9 @@ +# Local campaign files + +Campaign files use JSON. Seed files use JSONL with one object per line. + +Simple CSV files also work when they contain a `url` or `source_url` column. +Quoted CSV fields are rejected. JSONL is the preferred format for complex seed data. + +The campaign runner stores pages, leads, and exports under `var/local-leads`. +It never writes CRM or Neon data. diff --git a/apps/agent/campaigns/partwall-2027.json b/apps/agent/campaigns/partwall-2027.json new file mode 100644 index 000000000..93cab49a5 --- /dev/null +++ b/apps/agent/campaigns/partwall-2027.json @@ -0,0 +1,29 @@ +{ + "campaign_id": "partwall-trade-show-2027", + "project": "PartWall", + "target_customer_type": "trade show exhibitors and industrial display buyers", + "geography": ["United States", "Canada"], + "event_window": { + "start": "2027-03-01", + "end": "2027-06-30" + }, + "source_types": ["seed_url", "manual_list"], + "seed_urls": [], + "keywords": [ + "exhibitor", + "trade show", + "booth", + "display", + "industrial", + "manufacturer" + ], + "exclusions": ["job board", "press release only", "consumer retail"], + "max_companies": 25, + "max_pages_per_domain": 3, + "required_fields": ["company_name", "website", "evidence_excerpt"], + "scoring_rules": { + "fit_keyword_points": 15, + "evidence_points": 40, + "contact_field_points": 15 + } +} diff --git a/apps/agent/package.json b/apps/agent/package.json index 2360a75ad..5c6ee54fd 100644 --- a/apps/agent/package.json +++ b/apps/agent/package.json @@ -9,6 +9,8 @@ "backfill:images": "bun scripts/backfill-brand-images.ts", "dev": "eve dev", "dev:headless": "eve dev --no-ui", + "local-campaign:dry-run": "bun agent/cli/local-campaign-worker.ts", + "local-lead:dry-run": "bun agent/cli/local-lead-worker.ts", "dispatch": "curl -fsS -X POST \"${AGENT_URL:-http://127.0.0.1:2000}/eve/v1/dev/schedules/dispatch\"", "build": "eve build", "start": "bun scripts/start.ts", diff --git a/apps/agent/test/local-campaign.spec.ts b/apps/agent/test/local-campaign.spec.ts new file mode 100644 index 000000000..648a69f9c --- /dev/null +++ b/apps/agent/test/local-campaign.spec.ts @@ -0,0 +1,924 @@ +import { describe, expect, it } from "bun:test"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fetchCachedPage } from "../agent/lib/local-campaign-cache"; +import { + localCampaignCsv, + localCampaignJsonl, +} from "../agent/lib/local-campaign-export"; +import { + extractLocalCampaignLead, + parseLocalCampaignResponse, +} from "../agent/lib/local-campaign-extraction"; +import { + assertLocalCampaignOnly, + loadCampaignSeeds, + runLocalCampaign, +} from "../agent/lib/local-campaign-runner"; +import { localCampaignSchema } from "../agent/lib/local-campaign-schema"; +import { scoreLocalCampaignLead } from "../agent/lib/local-campaign-scoring"; +import { + extractLocalCampaignSourceUnitCandidates, + extractLocalCampaignSourceUnits, + hasUsefulPageEvidence, + rankLocalCampaignSourceUnitText, + sourceUnitRejectionReason, +} from "../agent/lib/local-campaign-source-units"; +import { + assertLocalCampaignPath, + dedupeAgainst, + normalizeDomain, +} from "../agent/lib/local-campaign-staging"; +import { createLocalOllamaClient } from "../agent/lib/local-ollama"; + +const campaign = localCampaignSchema.parse({ + campaign_id: "test-campaign", + project: "PartWall", + target_customer_type: "exhibitor", + geography: ["United States"], + source_types: ["seed_url", "manual_list"], + seed_urls: ["https://example.com/one"], + keywords: ["exhibitor", "display"], + exclusions: ["consumer retail"], + max_companies: 2, + max_pages_per_domain: 1, + required_fields: ["company_name", "website", "evidence_excerpt"], + scoring_rules: { + fit_keyword_points: 20, + evidence_points: 40, + contact_field_points: 15, + }, +}); + +const leadJson = JSON.stringify({ + company_name: "Acme Displays", + website: "https://acme.example", + contact_name: null, + role: null, + email: null, + phone: null, + location: "Chicago", + source_url: "https://example.com/one", + evidence_excerpt: + "Acme Displays, https://acme.example, is an exhibitor display company in Chicago.", + project_fit_reason: "The exhibitor needs display equipment.", + missing_fields: ["contact_name", "role", "email", "phone"], +}); + +describe("local campaign runner", () => { + it("validates campaign definitions", () => { + expect(localCampaignSchema.safeParse(campaign).success).toBe(true); + }); + + it("loads CSV and JSONL seed lists", async () => { + const directory = await mkdtemp(join(tmpdir(), "local-campaign-")); + try { + const csv = join(directory, "seeds.csv"); + const jsonl = join(directory, "seeds.jsonl"); + await Bun.write(csv, "url\nhttps://example.com/a\n"); + await Bun.write(jsonl, '{"source_url":"https://example.com/b"}\n'); + expect(await loadCampaignSeeds(csv)).toEqual(["https://example.com/a"]); + expect(await loadCampaignSeeds(jsonl)).toEqual(["https://example.com/b"]); + } finally { + await rm(directory, { recursive: true, force: true }); + } + }); + + it("rejects quoted CSV fields and recommends JSONL", async () => { + const directory = await mkdtemp(join(tmpdir(), "local-campaign-csv-")); + try { + const csv = join(directory, "quoted.csv"); + await Bun.write(csv, 'url\n"https://example.com/a,b"\n'); + await expect(loadCampaignSeeds(csv)).rejects.toThrow("Use JSONL"); + } finally { + await rm(directory, { recursive: true, force: true }); + } + }); + + it("reuses cached pages and respects domain caps", async () => { + let loads = 0; + const result = await runLocalCampaign( + campaign, + ["https://example.com/one", "https://example.com/two"], + { + pageLoader: async () => { + loads += 1; + return { + cached: loads === 1, + page: { + url: "https://example.com/one", + content_hash: "hash", + text: "Acme Displays is an exhibitor display company in Chicago.", + fetched_at: new Date().toISOString(), + }, + }; + }, + client: { generate: async () => ({ text: leadJson, latencyMs: 4 }) }, + stager: async (lead, path) => ({ + path: path ?? "local", + staged: lead, + duplicate: false, + }), + }, + ); + expect(loads).toBe(1); + expect(result.cached).toBe(1); + }); + + it("continues after an invalid extraction and reports the failure", async () => { + const secondUrl = "https://example.org/two"; + let calls = 0; + const result = await runLocalCampaign( + campaign, + ["https://example.com/one", secondUrl], + { + pageLoader: async (url) => ({ + cached: false, + page: { + url, + content_hash: url, + text: "Acme Displays, https://acme.example, is an exhibitor display company in Chicago.", + fetched_at: new Date().toISOString(), + }, + }), + client: { + generate: async () => ({ + text: + calls++ < 2 + ? '{"invalid":true}' + : leadJson.replace("https://example.com/one", secondUrl), + latencyMs: 4, + }), + }, + stager: async (lead, path) => ({ + path: path ?? "local", + staged: lead, + duplicate: false, + }), + }, + ); + expect(result.processed).toBe(2); + expect(result.extracted).toBe(1); + expect(result.extraction_failures).toBe(1); + expect(result.staged).toBe(1); + expect(result.failures[0]?.stage).toBe("extraction"); + }); + + it("caches fetched page content by URL", async () => { + const directory = await mkdtemp(join(tmpdir(), "local-campaign-cache-")); + try { + const path = join(directory, "pages.jsonl"); + let calls = 0; + const fetchImpl: typeof fetch = async () => { + calls += 1; + return new Response("cached page", { status: 200 }); + }; + const first = await fetchCachedPage( + "https://example.com/page", + fetchImpl, + path, + ); + const second = await fetchCachedPage( + "https://example.com/page", + fetchImpl, + path, + ); + expect(first.cached).toBe(false); + expect(second.cached).toBe(true); + expect(calls).toBe(1); + expect(second.page.content_hash).toBe(first.page.content_hash); + } finally { + await rm(directory, { recursive: true, force: true }); + } + }); + + it("rejects unsupported model output and preserves evidence rules", () => { + expect(() => + parseLocalCampaignResponse( + `\`\`\`json\n${leadJson}\n\`\`\``, + "https://example.com/one", + ), + ).toThrow(); + expect(() => + parseLocalCampaignResponse( + leadJson.replace("Acme Displays", "Invented Company"), + "https://example.com/one", + ), + ).toThrow("company_name is not present"); + }); + + it("requires project_fit_reason to be a string", () => { + const value = JSON.parse(leadJson) as Record; + value.project_fit_reason = null; + expect(() => + parseLocalCampaignResponse( + JSON.stringify(value), + "https://example.com/one", + ), + ).toThrow("project_fit_reason"); + }); + + it("requires non-empty exact source evidence", () => { + const emptyEvidence = JSON.parse(leadJson) as Record; + emptyEvidence.evidence_excerpt = ""; + expect(() => + parseLocalCampaignResponse( + JSON.stringify(emptyEvidence), + "https://example.com/one", + "Acme Displays source text.", + ), + ).toThrow("evidence_excerpt"); + const wrongEvidence = JSON.parse(leadJson) as Record; + wrongEvidence.evidence_excerpt = + "Acme Displays, https://acme.example, is an exhibitor display company in Chicago."; + expect(() => + parseLocalCampaignResponse( + JSON.stringify(wrongEvidence), + "https://example.com/one", + "Different source text.", + ), + ).toThrow("evidence_excerpt is not present"); + }); + + it("retries schema-invalid model output once and accepts a repaired response", async () => { + const sourceText = + "Acme Displays, https://acme.example, is an exhibitor display company in Chicago."; + let calls = 0; + const result = await extractLocalCampaignLead( + sourceText, + "https://example.com/one", + { + generate: async () => { + calls += 1; + return { + text: + calls === 1 + ? JSON.stringify({ + ...JSON.parse(leadJson), + evidence_excerpt: "", + project_fit_reason: null, + }) + : leadJson, + latencyMs: 5, + }; + }, + }, + ); + expect(result.attempt_count).toBe(2); + expect(result.latency_ms).toBe(10); + }); + + it("rejects after one failed repair attempt with the Zod issue", async () => { + let calls = 0; + await expect( + extractLocalCampaignLead( + "Acme Displays, https://acme.example, is an exhibitor display company in Chicago.", + "https://example.com/one", + { + generate: async () => { + calls += 1; + return { + text: JSON.stringify({ + ...JSON.parse(leadJson), + project_fit_reason: null, + }), + latencyMs: 5, + }; + }, + }, + ), + ).rejects.toThrow("project_fit_reason"); + expect(calls).toBe(2); + }); + + it("scores from keywords, evidence, and fields", () => { + const lead = parseLocalCampaignResponse( + leadJson, + "https://example.com/one", + ); + const scores = scoreLocalCampaignLead(lead, campaign); + expect(scores.fit_score).toBeGreaterThan(0); + expect(scores.evidence_score).toBe(60); + expect(scores.contact_completeness_score).toBe(0); + expect(scores.review_required_reason).toContain("Email is missing."); + }); + + it("extracts company source units from table rows", () => { + const units = extractLocalCampaignSourceUnits( + "
Acme Displays LLChttps://acme.exampleChicago, IL
", + "https://example.com/exhibitors", + ); + expect(units).toHaveLength(1); + expect(units[0]?.extraction_method).toBe("table_row"); + expect(units[0]?.candidate_company_name).toContain("Acme Displays LLC"); + expect(units[0]?.candidate_website).toBe("https://acme.example"); + }); + + it("extracts company source units from cards and list items", () => { + const units = extractLocalCampaignSourceUnits( + [ + "

Fixture Systems Inc

Website

Display fixtures in Dallas, TX

", + "
  • Panel Technology Ltd https://panel.example exhibit walls
", + ].join(""), + "https://example.com/exhibitors", + ); + expect(units.map((unit) => unit.extraction_method)).toContain("card"); + expect(units.map((unit) => unit.extraction_method)).toContain("list_item"); + }); + + it("extracts company source units from links with surrounding text", () => { + const units = extractLocalCampaignSourceUnits( + "Booth Systems Corp", + "https://example.com/exhibitors", + ); + expect(units).toHaveLength(1); + expect(units[0]?.extraction_method).toBe("link_context"); + }); + + it("rejects title or venue only evidence and boilerplate", () => { + expect( + hasUsefulPageEvidence("Automate 2027 exhibitor directory at JI Expo"), + ).toBe(false); + expect( + extractLocalCampaignSourceUnits( + "
  • Cookie settings privacy navigation menu
    • ", + "https://example.com/exhibitors", + ), + ).toHaveLength(0); + }); + + it("caps and trims source units", () => { + const html = [ + "Acme Displays LLChttps://acme.exampleLong text ".concat( + "x".repeat(200), + "", + ), + "Beta Fixtures Inchttps://beta.example", + ].join(""); + const units = extractLocalCampaignSourceUnits( + html, + "https://example.com/exhibitors", + { max_source_units_per_page: 1, max_source_unit_chars: 80 }, + ); + expect(units).toHaveLength(1); + expect(units[0]?.candidate_text.length).toBeLessThanOrEqual(80); + }); + + it("dedupes source units before Ollama", () => { + const result = extractLocalCampaignSourceUnits( + [ + "Acme Displays LLChttps://acme.example", + "
  • Acme Displays LLC https://acme.example
    • ", + ].join(""), + "https://example.com/exhibitors", + ); + expect(result).toHaveLength(1); + }); + + it("ranks company units with websites high", () => { + const ranking = rankLocalCampaignSourceUnitText( + "Acme Displays LLC https://acme.example custom trade show display fixtures", + { source_domain: "event.example" }, + ); + expect(ranking.rank_score).toBeGreaterThanOrEqual(70); + expect(ranking.ranking_reasons).toContain("likely company name"); + expect(ranking.ranking_reasons).toContain("external company website"); + }); + + it("ranks company units with booth evidence high", () => { + const ranking = rankLocalCampaignSourceUnitText( + "Beta Fixtures Inc booth 1240 industrial display products Austin, TX", + { source_domain: "event.example" }, + ); + expect(ranking.rank_score).toBeGreaterThanOrEqual(70); + expect(ranking.ranking_reasons).toContain("booth or stand number"); + }); + + it("ranks event titles low", () => { + const ranking = rankLocalCampaignSourceUnitText( + "PartWall 2027 exhibitor directory trade show registration", + { source_domain: "event.example" }, + ); + expect(ranking.rank_score).toBeLessThan(0); + expect(ranking.ranking_reasons).toContain("page title or event-only text"); + }); + + it("ranks venue-only evidence low", () => { + const ranking = rankLocalCampaignSourceUnitText( + "JI Expo venue event floor plan attendee registration", + { source_domain: "event.example" }, + ); + expect(ranking.rank_score).toBeLessThan(0); + expect(ranking.ranking_reasons).toContain("event or venue level evidence"); + }); + + it("ranks organizer boilerplate low", () => { + const ranking = rankLocalCampaignSourceUnitText( + "Organizer newsletter privacy terms copyright all rights reserved", + { source_domain: "event.example" }, + ); + expect(ranking.rank_score).toBeLessThan(0); + expect(ranking.ranking_reasons).toContain( + "navigation/footer/cookie boilerplate", + ); + }); + + it("sends top-ranked source units to Ollama first", async () => { + const calls: string[] = []; + const result = await runLocalCampaign( + { ...campaign, max_ollama_calls_per_seed: 1, max_companies: 5 }, + ["https://example.com/exhibitors"], + { + pageLoader: async (url) => ({ + cached: false, + page: { + url, + content_hash: "hash", + text: [ + "Beta Fixtures Incgeneral supplier listing", + "Acme Displays LLChttps://acme.exampleBooth 1240custom trade show display fixtures", + ].join(""), + fetched_at: new Date().toISOString(), + }, + }), + client: { + generate: async ({ prompt }) => { + calls.push(prompt); + return { + text: leadJson.replace( + "https://example.com/one", + "https://example.com/exhibitors", + ), + latencyMs: 4, + }; + }, + }, + stager: async (lead, path) => ({ + path: path ?? "local", + staged: lead, + duplicate: false, + }), + }, + ); + expect(result.ollama_calls).toBe(1); + expect(calls[0]).toContain("Acme Displays LLC"); + expect(calls[0]).not.toContain("Beta Fixtures Inc"); + }); + + it("records source unit ranking reasons on staged leads", async () => { + const sourceText = + "Acme Displays LLC https://acme.example Booth 1240 custom display fixtures Chicago, IL"; + const sourceLead = JSON.stringify({ + ...JSON.parse(leadJson), + source_url: "https://example.com/exhibitors", + evidence_excerpt: sourceText, + }); + const result = await runLocalCampaign( + { ...campaign, max_ollama_calls_per_seed: 1, max_companies: 5 }, + ["https://example.com/exhibitors"], + { + pageLoader: async (url) => ({ + cached: false, + page: { + url, + content_hash: "hash", + text: `${sourceText}`, + fetched_at: new Date().toISOString(), + }, + }), + client: { + generate: async () => ({ + text: sourceLead, + latencyMs: 4, + }), + }, + stager: async (lead, path) => ({ + path: path ?? "local", + staged: lead, + duplicate: false, + }), + }, + ); + expect(result.leads[0]?.source_unit_rank_score).toBeGreaterThan(0); + expect(result.leads[0]?.source_unit_ranking_reasons).toContain( + "external company website", + ); + }); +}); + +describe("local campaign source unit filtering", () => { + it("rejects AIMG-style website-only event vendor links", () => { + const candidates = extractLocalCampaignSourceUnitCandidates( + "Website by Amplify Industrial Marketing + Guidance.", + "https://www.automateshow.com/exhibitors", + ); + expect(candidates.units).toHaveLength(0); + expect(candidates.rejected[0]?.rejection_reason).toBe("event_vendor_link"); + }); + + it("rejects registration and ticketing source units", () => { + expect( + sourceUnitRejectionReason( + "Register now for attendee badges and buy tickets for the expo", + ), + ).toBe("registration_or_ticketing"); + }); + + it("rejects footer social privacy and contact links", () => { + const candidates = extractLocalCampaignSourceUnitCandidates( + [ + "Facebook", + "Privacy Policy", + "Contact Us", + ].join(""), + "https://example.com/exhibitors", + ); + expect(candidates.units).toHaveLength(0); + expect(candidates.rejected.map((unit) => unit.rejection_reason)).toContain( + "navigation_or_footer", + ); + expect(candidates.rejected.map((unit) => unit.rejection_reason)).toContain( + "boilerplate", + ); + }); + + it("rejects organizer or venue-only source units", () => { + expect( + sourceUnitRejectionReason( + "Show organizer convention center floor plan event management", + ), + ).toBe("venue_or_organizer"); + }); + + it("keeps company source units with website and product text", () => { + const candidates = extractLocalCampaignSourceUnitCandidates( + "Acme Displays LLChttps://acme.examplecustom display fixtures and booth walls", + "https://example.com/exhibitors", + ); + expect(candidates.units).toHaveLength(1); + expect(candidates.rejected).toHaveLength(0); + }); + + it("keeps company source units with booth or profile evidence", () => { + const candidates = extractLocalCampaignSourceUnitCandidates( + "Beta Fixtures Inc booth 1240", + "https://example.com/exhibitors", + ); + expect(candidates.units).toHaveLength(1); + expect(candidates.units[0]?.ranking_reasons).toContain( + "booth or stand number", + ); + }); + + it("records rejected source unit reasons in run diagnostics", async () => { + const result = await runLocalCampaign( + { ...campaign, max_ollama_calls_per_seed: 1, max_companies: 5 }, + ["https://example.com/exhibitors"], + { + pageLoader: async (url) => ({ + cached: false, + page: { + url, + content_hash: "hash", + text: "Website by Amplify Industrial Marketing + Guidance.", + fetched_at: new Date().toISOString(), + }, + }), + client: { + generate: async () => { + throw new Error("Ollama should not be called."); + }, + }, + }, + ); + expect(result.ollama_calls).toBe(0); + expect(result.rejected_source_units).toBe(1); + expect(result.source_unit_rejection_reasons.event_vendor_link).toBe(1); + expect(result.seed_statuses[0]?.rejection_reasons.event_vendor_link).toBe( + 1, + ); + }); +}); + +describe("local campaign runner extraction flow", () => { + it("avoids whole-page model calls when source units exist", async () => { + const calls: string[] = []; + const result = await runLocalCampaign( + { ...campaign, max_ollama_calls_per_seed: 2, max_companies: 5 }, + ["https://example.com/exhibitors"], + { + pageLoader: async (url) => ({ + cached: false, + page: { + url, + content_hash: "hash", + text: [ + "Acme Displays LLChttps://acme.exampleChicago, IL", + "Beta Fixtures Inchttps://beta.exampleAustin, TX", + ].join(""), + fetched_at: new Date().toISOString(), + }, + }), + client: { + generate: async ({ prompt }) => { + calls.push(prompt); + const sourceUrl = "https://example.com/exhibitors"; + return { + text: leadJson.replace("https://example.com/one", sourceUrl), + latencyMs: 4, + }; + }, + }, + stager: async (lead, path) => ({ + path: path ?? "local", + staged: lead, + duplicate: false, + }), + }, + ); + expect(result.source_units).toBe(2); + expect(result.ollama_calls).toBe(2); + expect(calls.every((prompt) => !prompt.includes(""))).toBe(true); + }); + + it("rejects source unit output without company-level evidence", async () => { + const thinLead = JSON.stringify({ + company_name: null, + website: null, + contact_name: null, + role: null, + email: null, + phone: null, + location: null, + source_url: "https://example.com/exhibitors", + evidence_excerpt: "Members Company Application", + project_fit_reason: + "The source text does not identify a specific company or website.", + missing_fields: [ + "company_name", + "website", + "contact_name", + "role", + "email", + "phone", + "location", + ], + }); + const result = await runLocalCampaign( + { ...campaign, max_ollama_calls_per_seed: 1, max_companies: 5 }, + ["https://example.com/exhibitors"], + { + pageLoader: async (url) => ({ + cached: false, + page: { + url, + content_hash: "hash", + text: "
  • Members Company Application https://application.example
    • ", + fetched_at: new Date().toISOString(), + }, + }), + client: { generate: async () => ({ text: thinLead, latencyMs: 4 }) }, + stager: async (lead, path) => ({ + path: path ?? "local", + staged: lead, + duplicate: false, + }), + }, + ); + expect(result.staged).toBe(0); + expect(result.failures[0]?.reason).toBe( + "Source unit did not produce company-level evidence.", + ); + }); + + it("rejects source unit output that only has a website", async () => { + const websiteOnlyLead = JSON.stringify({ + ...JSON.parse(leadJson), + company_name: null, + website: "https://acme.example", + location: null, + source_url: "https://example.com/exhibitors", + evidence_excerpt: "Acme Displays LLC https://acme.example", + missing_fields: [ + "company_name", + "contact_name", + "role", + "email", + "phone", + "location", + ], + }); + const result = await runLocalCampaign( + { ...campaign, max_ollama_calls_per_seed: 1, max_companies: 5 }, + ["https://example.com/exhibitors"], + { + pageLoader: async (url) => ({ + cached: false, + page: { + url, + content_hash: "hash", + text: "
  • Acme Displays LLC https://acme.example custom display fixtures
    • ", + fetched_at: new Date().toISOString(), + }, + }), + client: { + generate: async () => ({ + text: websiteOnlyLead, + latencyMs: 4, + }), + }, + stager: async (lead, path) => ({ + path: path ?? "local", + staged: lead, + duplicate: false, + }), + }, + ); + expect(result.staged).toBe(0); + expect(result.failures[0]?.reason).toBe( + "Source unit did not produce company-level evidence.", + ); + }); + + it("falls back to page-level extraction when no source units are found", async () => { + let calls = 0; + const result = await runLocalCampaign( + campaign, + ["https://example.com/one"], + { + pageLoader: async (url) => ({ + cached: false, + page: { + url, + content_hash: "hash", + text: "Acme Displays, https://acme.example, is an exhibitor display company in Chicago.", + fetched_at: new Date().toISOString(), + }, + }), + client: { + generate: async () => { + calls += 1; + return { text: leadJson, latencyMs: 4 }; + }, + }, + stager: async (lead, path) => ({ + path: path ?? "local", + staged: lead, + duplicate: false, + }), + }, + ); + expect(calls).toBe(1); + expect(result.source_units).toBe(0); + expect(result.ollama_calls).toBe(1); + }); + + it("flushes per-seed failure status", async () => { + const directory = await mkdtemp(join(tmpdir(), "local-campaign-status-")); + const seedStatusPath = `var/local-leads/${directory.split("/").pop()}-status.jsonl`; + try { + const result = await runLocalCampaign( + campaign, + ["https://example.com/one"], + { + pageLoader: async () => { + throw new Error("Source fetch timed out."); + }, + seedStatusPath, + }, + ); + expect(result.seed_statuses).toHaveLength(1); + expect(result.seed_statuses[0]?.failure_reason).toBe( + "Source fetch timed out.", + ); + expect( + await readFile( + new URL(`../${seedStatusPath}`, import.meta.url), + "utf8", + ), + ).toContain("Source fetch timed out."); + } finally { + await rm(directory, { recursive: true, force: true }); + await rm(new URL(`../${seedStatusPath}`, import.meta.url), { + force: true, + }); + } + }); + + it("stops a seed when the per-seed timeout is reached", async () => { + const result = await runLocalCampaign( + { ...campaign, max_ollama_calls_per_seed: 2 }, + ["https://example.com/one"], + { + perSeedTimeoutMs: -1, + pageLoader: async (url) => ({ + cached: false, + page: { + url, + content_hash: "hash", + text: "Acme Displays LLC is an exhibitor display company in Chicago.", + fetched_at: new Date().toISOString(), + }, + }), + client: { + generate: async () => { + await new Promise((resolve) => setTimeout(resolve, 2)); + return { text: leadJson, latencyMs: 2 }; + }, + }, + stager: async (lead, path) => ({ + path: path ?? "local", + staged: lead, + duplicate: false, + }), + }, + ); + expect(result.seed_statuses[0]?.failure_reason).toBe("Seed timed out."); + expect(result.ollama_calls).toBe(0); + }); + + it("scores company-level source unit evidence above page-level evidence", () => { + const lead = parseLocalCampaignResponse( + leadJson, + "https://example.com/one", + ); + const pageScore = scoreLocalCampaignLead(lead, campaign); + const unitScore = scoreLocalCampaignLead(lead, campaign, { + sourceUnit: true, + extractionMethod: "table_row", + }); + expect(unitScore.evidence_score).toBeGreaterThan(pageScore.evidence_score); + }); + + it("deduplicates normalized domain and exports JSONL and CSV", async () => { + expect(normalizeDomain("WWW.Example.com.")).toBe("example.com"); + const lead = { + ...parseLocalCampaignResponse(leadJson, "https://example.com/one"), + ...scoreLocalCampaignLead( + parseLocalCampaignResponse(leadJson, "https://example.com/one"), + campaign, + ), + campaign_id: campaign.campaign_id, + lead_id: "lead", + source_domain: "example.com", + status: "new" as const, + }; + expect(dedupeAgainst(lead, [lead])).not.toBeNull(); + const jsonl = localCampaignJsonl([lead]); + const csv = localCampaignCsv([lead]); + expect(JSON.parse(jsonl).lead_id).toBe("lead"); + expect(csv).toContain("campaign_id"); + }); + + it("refuses CRM synchronization flags before work", () => { + expect(() => assertLocalCampaignOnly(["run", "--sync-crm"])).toThrow(); + }); + + it("keeps staging and export paths under local lead storage", () => { + expect(assertLocalCampaignPath("var/local-leads/export.csv")).toEndWith( + "/apps/agent/var/local-leads/export.csv", + ); + expect(() => assertLocalCampaignPath("../../outside.csv")).toThrow( + "var/local-leads", + ); + }); + + it("keeps local campaign modules free of CRM writer imports", async () => { + const files = [ + "agent/lib/local-campaign-cache.ts", + "agent/lib/local-campaign-export.ts", + "agent/lib/local-campaign-extraction.ts", + "agent/lib/local-campaign-lead-schema.ts", + "agent/lib/local-campaign-runner.ts", + "agent/lib/local-campaign-schema.ts", + "agent/lib/local-campaign-scoring.ts", + "agent/lib/local-campaign-source-units.ts", + "agent/lib/local-campaign-staging.ts", + "agent/cli/local-campaign-worker.ts", + ]; + const forbidden = + /from\s+["'](?:@crm\/db|.*prisma.*|.*neon.*|.*database.*|.*outreach.*|.*email.*)["']/i; + for (const file of files) { + const text = await readFile( + new URL(`../${file}`, import.meta.url), + "utf8", + ); + expect(text).not.toMatch(forbidden); + } + }); + + it("times out stalled Ollama requests", async () => { + const client = createLocalOllamaClient( + async (_input, init) => + new Promise((_resolve, reject) => { + init?.signal?.addEventListener("abort", () => { + reject(new Error("aborted")); + }); + }), + ); + await expect( + client.generate({ model: "qwen2.5:14b", prompt: "{}", timeoutMs: 1 }), + ).rejects.toThrow("timed out"); + }); +}); diff --git a/apps/agent/test/local-lead-extraction.spec.ts b/apps/agent/test/local-lead-extraction.spec.ts new file mode 100644 index 000000000..2cb9fafb7 --- /dev/null +++ b/apps/agent/test/local-lead-extraction.spec.ts @@ -0,0 +1,162 @@ +import { describe, expect, it } from "bun:test"; +import { + extractLocalLead, + fetchLocalLeadSource, + parseLocalLeadResponse, +} from "../agent/lib/local-lead-extraction"; + +const sourceUrl = "https://example.com/acme"; +const valid = JSON.stringify({ + company_name: "Acme Field Systems", + website: "https://acmefield.example", + contact_name: "Jordan Lee", + role: "Head of Partnerships", + email: null, + phone: null, + source_url: sourceUrl, + evidence_excerpt: "Acme Field Systems. Jordan Lee — Head of Partnerships.", + confidence_reason: "The page states the company, contact, and role.", + missing_fields: ["email", "phone"], +}); + +describe("local lead extraction", () => { + it("accepts valid strict JSON", () => { + expect(parseLocalLeadResponse(valid, sourceUrl).contact_name).toBe( + "Jordan Lee", + ); + }); + + it("requires null for missing email and phone", () => { + const lead = parseLocalLeadResponse(valid, sourceUrl); + expect(lead.email).toBeNull(); + expect(lead.phone).toBeNull(); + }); + + it("rejects an invented-looking email without evidence", () => { + const value = JSON.parse(valid) as Record; + value.email = "jordan@example.com"; + value.missing_fields = ["phone"]; + expect(() => + parseLocalLeadResponse(JSON.stringify(value), sourceUrl), + ).toThrow("email is not present in evidence_excerpt"); + }); + + it("rejects markdown-wrapped JSON", () => { + expect(() => + parseLocalLeadResponse(`\`\`\`json\n${valid}\n\`\`\``, sourceUrl), + ).toThrow("plain JSON object"); + }); + + it("rejects invalid JSON", () => { + expect(() => parseLocalLeadResponse("{not-json}", sourceUrl)).toThrow( + "invalid JSON", + ); + }); + + it("requires an evidence excerpt", () => { + const value = JSON.parse(valid) as Record; + value.evidence_excerpt = ""; + expect(() => + parseLocalLeadResponse(JSON.stringify(value), sourceUrl), + ).toThrow(); + }); + + it("blocks private and internal source URLs before fetching", async () => { + const fetchImpl = async () => { + throw new Error("fetch must not run"); + }; + for (const url of [ + "http://127.0.0.1/private", + "http://192.168.1.10/private", + "http://service.internal/private", + "http://printer.local/private", + ]) { + await expect(fetchLocalLeadSource(url, fetchImpl)).rejects.toThrow( + "Local source URLs are blocked", + ); + } + }); + + it("enforces the source byte limit", async () => { + const response = new Response("small", { + status: 200, + headers: { "content-length": "20" }, + }); + expect( + fetchLocalLeadSource("https://example.com", async () => response, { + maxBytes: 4, + lookup: async () => [{ address: "93.184.216.34", family: 4 }], + }), + ).rejects.toThrow("maximum content size"); + }); + + it("rejects hostnames that resolve to local addresses", async () => { + const fetchImpl: typeof fetch = async () => { + throw new Error("fetch must not run"); + }; + for (const address of ["127.0.0.1", "10.0.0.4", "fd00::4"]) { + await expect( + fetchLocalLeadSource("https://public.example", fetchImpl, { + lookup: async () => [ + { address, family: address.includes(":") ? 6 : 4 }, + ], + }), + ).rejects.toThrow("local address"); + } + }); + + it("allows a hostname that resolves to a public address", async () => { + const source = await fetchLocalLeadSource( + "https://public.example", + async () => new Response("public page", { status: 200 }), + { lookup: async () => [{ address: "93.184.216.34", family: 4 }] }, + ); + expect(source.text).toBe("public page"); + }); + + it("times out stalled source fetches", async () => { + await expect( + fetchLocalLeadSource( + "https://public.example", + async (_input, init) => + new Promise((_resolve, reject) => { + init?.signal?.addEventListener("abort", () => { + reject(new Error("aborted")); + }); + }), + { + lookup: async () => [{ address: "93.184.216.34", family: 4 }], + timeoutMs: 1, + }, + ), + ).rejects.toThrow("timed out"); + }); + + it("rejects DNS lookup failure", async () => { + await expect( + fetchLocalLeadSource("https://public.example", fetch, { + lookup: async () => { + throw new Error("not found"); + }, + }), + ).rejects.toThrow("DNS lookup failed"); + }); + + it("rejects stalled DNS lookup", async () => { + await expect( + fetchLocalLeadSource("https://public.example", fetch, { + dnsTimeoutMs: 1, + lookup: async () => new Promise(() => {}), + }), + ).rejects.toThrow("DNS lookup failed"); + }); + + it("does not require DATABASE_URL or CRM modules", async () => { + const result = await extractLocalLead( + "Acme Field Systems. Jordan Lee — Head of Partnerships.", + sourceUrl, + { generate: async () => ({ text: valid, latencyMs: 5 }) }, + ); + expect(result.parser_status).toBe("accepted"); + }); +}); diff --git a/apps/agent/test/local-lead-staging.spec.ts b/apps/agent/test/local-lead-staging.spec.ts new file mode 100644 index 000000000..745fdfbfc --- /dev/null +++ b/apps/agent/test/local-lead-staging.spec.ts @@ -0,0 +1,56 @@ +import { describe, expect, it } from "bun:test"; +import { randomUUID } from "node:crypto"; +import { rm } from "node:fs/promises"; +import { extractLocalLead } from "../agent/lib/local-lead-extraction"; +import { + defaultLocalLeadStagingPath, + stageLocalLead, +} from "../agent/lib/local-lead-staging"; + +const response = JSON.stringify({ + company_name: "Acme Field Systems", + website: "https://acmefield.example", + contact_name: "Jordan Lee", + role: "Head of Partnerships", + email: null, + phone: null, + source_url: "https://example.com/acme", + evidence_excerpt: "Acme Field Systems. Jordan Lee — Head of Partnerships.", + confidence_reason: "The page states the company, contact, and role.", + missing_fields: ["email", "phone"], +}); + +describe("local lead staging", () => { + it("anchors the default path inside apps/agent", () => { + expect(defaultLocalLeadStagingPath()).toEndWith( + "/apps/agent/var/local-leads/leads.jsonl", + ); + }); + + it("deduplicates by content hash", async () => { + const path = `var/local-leads/test-${randomUUID()}.jsonl`; + let stagedPath: string | undefined; + try { + const lead = await extractLocalLead( + "Acme Field Systems. Jordan Lee — Head of Partnerships.", + "https://example.com/acme", + { generate: async () => ({ text: response, latencyMs: 5 }) }, + ); + const first = await stageLocalLead(lead, path); + stagedPath = first.path; + expect(first.staged).toBe(true); + expect((await stageLocalLead(lead, path)).staged).toBe(false); + } finally { + if (stagedPath) await rm(stagedPath, { force: true }); + } + }); + + it("rejects paths outside local staging", async () => { + const lead = await extractLocalLead( + "Acme Field Systems. Jordan Lee — Head of Partnerships.", + "https://example.com/acme", + { generate: async () => ({ text: response, latencyMs: 5 }) }, + ); + expect(stageLocalLead(lead, ".env")).rejects.toThrow("var/local-leads"); + }); +}); diff --git a/apps/api/src/app.module.ts b/apps/api/src/app.module.ts index 82a345df2..872a972cf 100644 --- a/apps/api/src/app.module.ts +++ b/apps/api/src/app.module.ts @@ -22,6 +22,7 @@ import { EnrichmentModule } from "./enrichment/enrichment.module"; import { FieldsModule } from "./fields/fields.module"; import { GoogleModule } from "./google/google.module"; import { HealthModule } from "./health/health.module"; +import { IngestModule } from "./ingest/ingest.module"; import { LoggingModule } from "./logging/logging.module"; import { logAuthRoute } from "./logging/request-logger.middleware"; import { MailboxModule } from "./mailbox/mailbox.module"; @@ -75,6 +76,7 @@ import { WorkspaceModule } from "./workspace/workspace.module"; SsoModule, SlackModule, BackfillModule, + IngestModule, TelemetryModule, TrackingModule, ArchiveModule, diff --git a/apps/api/src/generated/server.ts b/apps/api/src/generated/server.ts index b77da4001..9cf683813 100644 --- a/apps/api/src/generated/server.ts +++ b/apps/api/src/generated/server.ts @@ -26,6 +26,7 @@ import { enrichmentQueueInput } from "@crm/validation/enrichment-queue"; import { fieldListInput, fieldListOutput, fieldByKeyInput, serializedFieldOutput, fieldEntityInput, fieldFiltersOutput, fieldIdInput, fieldCoverageOutput, fieldCreateInput, fieldUpdateArgs, fieldReorderInput, fieldReorderOutput, fieldDeleteOutput, fieldBackfillOutput } from "../fields/fields.contracts"; import { googleConnectionStatusOutput, setAutoCreateInput, suppressDomainInput, suppressDomainOutput, threadInput, emailThreadOutput, calendarEventInput, calendarEventOutput } from "../google/google.contracts"; import { purgeSyncedDataOutput, revokeAccessOutput, microsoftConnectionStatusOutput, setOutlookAutoCreateInput } from "../microsoft/microsoft.contracts"; +import { ingestSignalInput, ingestSignalOutput, signalInboxInput, signalInboxOutput, signalSourceRecordInput, signalCompanyCandidatesOutput, resolveSignalCompanyInput, resolveSignalCompanyOutput, qualifySignalInput, qualifySignalOutput, promoteSignalInput, promoteSignalOutput } from "../ingest/ingest.contracts"; import { savedViewListInput, savedViewListOutput, savedViewCreateInput, savedViewOutput, savedViewUpdateArgs, savedViewIdInput, savedViewDeleteOutput } from "../saved-views/saved-views.contracts"; import { agentModelOutput, modelCatalogOutput, setAgentModelInput, researchKeyOutput, setResearchKeyInput, archiveRetentionOutput, setArchiveRetentionDaysInput } from "../settings/settings.contracts"; import { slackStatusOutput, slackMatchesOutput, slackChannelsInput, slackChannelsOutput, slackJoinChannelInput, slackJoinChannelOutput, slackRefreshPeopleOutput, slackCreateChannelInput, slackCreateChannelOutput, slackDisconnectOutput } from "../slack/slack.contracts"; @@ -558,6 +559,32 @@ const appRouter = t.router({ .output(calendarEventOutput) .query(async () => "PLACEHOLDER_DO_NOT_REMOVE" as any) }), + ingest: t.router({ + signal: publicProcedure + .input(ingestSignalInput) + .output(ingestSignalOutput) + .mutation(async () => "PLACEHOLDER_DO_NOT_REMOVE" as any), + inbox: publicProcedure + .input(signalInboxInput) + .output(signalInboxOutput) + .query(async () => "PLACEHOLDER_DO_NOT_REMOVE" as any), + companyCandidates: publicProcedure + .input(signalSourceRecordInput) + .output(signalCompanyCandidatesOutput) + .query(async () => "PLACEHOLDER_DO_NOT_REMOVE" as any), + resolveCompany: publicProcedure + .input(resolveSignalCompanyInput) + .output(resolveSignalCompanyOutput) + .mutation(async () => "PLACEHOLDER_DO_NOT_REMOVE" as any), + qualify: publicProcedure + .input(qualifySignalInput) + .output(qualifySignalOutput) + .mutation(async () => "PLACEHOLDER_DO_NOT_REMOVE" as any), + promote: publicProcedure + .input(promoteSignalInput) + .output(promoteSignalOutput) + .mutation(async () => "PLACEHOLDER_DO_NOT_REMOVE" as any) + }), microsoft: t.router({ status: publicProcedure .output(microsoftConnectionStatusOutput) diff --git a/apps/api/src/ingest/ingest.contracts.ts b/apps/api/src/ingest/ingest.contracts.ts new file mode 100644 index 000000000..29b55a1d8 --- /dev/null +++ b/apps/api/src/ingest/ingest.contracts.ts @@ -0,0 +1,186 @@ +import { z } from "zod"; + +export type SignalPayloadValue = + | string + | number + | boolean + | null + | SignalPayloadValue[] + | { [key: string]: SignalPayloadValue }; + +const signalPayloadValue: z.ZodType = z.lazy(() => + z.union([ + z.string(), + z.number().finite(), + z.boolean(), + z.null(), + z.array(signalPayloadValue), + z.record(z.string(), signalPayloadValue), + ]), +); + +export const signalPayload = z.record(z.string(), signalPayloadValue); +export type SignalPayload = z.infer; + +export const ingestSignalInput = z.object({ + project: z.string().trim().min(1).max(96), + source: z.string().trim().min(1).max(96), + sourceType: z.string().trim().min(1).max(160), + sourceId: z.string().trim().min(1).max(320), + sourceUrl: z.string().url().nullable().optional(), + observedAt: z.string().datetime({ offset: true }).nullable().optional(), + entity: z.string().trim().min(1).max(320).nullable().optional(), + signalScore: z.number().min(0).max(100).nullable().optional(), + tags: z.array(z.string().trim().min(1).max(80)).max(50).default([]), + payload: signalPayload.default({}), +}); + +export const ingestSignalOutput = z.object({ + status: z.literal("accepted"), + sourceRecordId: z.string(), + project: z.string(), + deduplicated: z.boolean(), + promoted: z.literal(false), +}); + +export const signalInboxInput = z.object({ + project: z.string().trim().min(1).max(96).optional(), + source: z.string().trim().min(1).max(96).optional(), + minScore: z.number().min(0).max(100).optional(), + status: z.enum(["all", "unresolved", "mapped"]).default("unresolved"), + limit: z.number().int().min(1).max(200).default(50), +}); + +export const signalInboxItem = z.object({ + sourceRecordId: z.string(), + project: z.string(), + source: z.string(), + sourceType: z.string(), + sourceId: z.string(), + sourceUrl: z.string().nullable(), + observedAt: z.string(), + entity: z.string().nullable(), + signalScore: z.number().nullable(), + company: z.string().nullable(), + subject: z.string().nullable(), + stageKey: z.string().nullable(), + priority: z.string().nullable(), + demandTrigger: z.string().nullable(), + nextAction: z.string().nullable(), + mapped: z.boolean(), + payload: signalPayload, +}); + +export const signalInboxOutput = z.object({ + rows: z.array(signalInboxItem), + count: z.number().int().nonnegative(), +}); + +export const signalSourceRecordInput = z.object({ + sourceRecordId: z.string().trim().min(1), +}); + +export const companyCandidateOutput = z.object({ + companyId: z.string(), + name: z.string(), + domain: z.string().nullable(), + businessUnitId: z.string().nullable(), + score: z.number().min(0).max(110), + reasons: z.array(z.string()), +}); + +export const signalCompanyCandidatesOutput = z.object({ + sourceRecordId: z.string(), + project: z.string(), + entity: z.string().nullable(), + domain: z.string().nullable(), + mappedCompanyId: z.string().nullable(), + candidates: z.array(companyCandidateOutput), +}); + +export const resolveSignalCompanyInput = signalSourceRecordInput.extend({ + companyId: z.string().trim().min(1).nullable().optional(), + companyName: z.string().trim().min(1).max(320).nullable().optional(), + domain: z.string().trim().max(320).nullable().optional(), + createIfMissing: z.boolean().default(false), + queueResearch: z.boolean().default(true), +}); + +export const resolveSignalCompanyOutput = z.object({ + sourceRecordId: z.string(), + canonicalCompanyId: z.string(), + companyId: z.string(), + companyName: z.string(), + matchMethod: z.string(), + created: z.boolean(), + researchQueued: z.boolean(), +}); + +export const signalScoreComponents = z.object({ + icpMatch: z.number().min(0).max(25), + commercialTrigger: z.number().min(0).max(20), + projectRelevance: z.number().min(0).max(20), + companyValue: z.number().min(0).max(10), + location: z.number().min(0).max(10), + decisionMaker: z.number().min(0).max(10), + recency: z.number().min(0).max(5), +}); + +export const signalClassification = z.enum([ + "signal", + "research", + "qualified", + "priority", +]); + +export const qualifySignalInput = signalSourceRecordInput.extend({ + components: signalScoreComponents.optional(), + evidence: z.record(z.string(), z.string().max(2000)).default({}), + notes: z.string().trim().max(4000).nullable().optional(), +}); + +export const qualifySignalOutput = z.object({ + sourceRecordId: z.string(), + score: z.number().min(0).max(100), + classification: signalClassification, + method: z.enum(["components", "legacy-score", "unscored"]), + companyResolved: z.boolean(), + canonicalOpportunityEligible: z.boolean(), + visibleDealEligible: z.boolean(), +}); + +export const promoteSignalInput = signalSourceRecordInput.extend({ + createDeal: z.boolean().default(false), + ownerId: z.string().trim().min(1).nullable().optional(), + amountUsd: z.number().nonnegative().nullable().optional(), +}); + +export const promoteSignalOutput = z.object({ + sourceRecordId: z.string(), + score: z.number().min(0).max(100), + classification: signalClassification, + canonicalOpportunityId: z.string(), + dealId: z.string().nullable(), + createdCanonicalOpportunity: z.boolean(), + createdDeal: z.boolean(), +}); + +export type IngestSignalInput = z.infer; +export type IngestSignalOutput = z.infer; +export type SignalInboxInput = z.infer; +export type SignalInboxOutput = z.infer; +export type SignalCompanyCandidatesOutput = z.infer< + typeof signalCompanyCandidatesOutput +>; +export type ResolveSignalCompanyInput = z.infer< + typeof resolveSignalCompanyInput +>; +export type ResolveSignalCompanyOutput = z.infer< + typeof resolveSignalCompanyOutput +>; +export type SignalScoreComponents = z.infer; +export type SignalClassification = z.infer; +export type QualifySignalInput = z.infer; +export type QualifySignalOutput = z.infer; +export type PromoteSignalInput = z.infer; +export type PromoteSignalOutput = z.infer; diff --git a/apps/api/src/ingest/ingest.module.ts b/apps/api/src/ingest/ingest.module.ts new file mode 100644 index 000000000..74af8829c --- /dev/null +++ b/apps/api/src/ingest/ingest.module.ts @@ -0,0 +1,13 @@ +import { Module } from "@nestjs/common"; +import { CompaniesModule } from "../companies/companies.module"; +import { DealsModule } from "../deals/deals.module"; +import { IngestRouter } from "./ingest.router"; +import { IngestService } from "./ingest.service"; +import { SignalQualificationService } from "./signal-qualification.service"; + +@Module({ + imports: [CompaniesModule, DealsModule], + providers: [IngestService, SignalQualificationService, IngestRouter], + exports: [IngestService, SignalQualificationService], +}) +export class IngestModule {} diff --git a/apps/api/src/ingest/ingest.router.ts b/apps/api/src/ingest/ingest.router.ts new file mode 100644 index 000000000..6352246b6 --- /dev/null +++ b/apps/api/src/ingest/ingest.router.ts @@ -0,0 +1,97 @@ +import { Inject } from "@nestjs/common"; +import { Input, Mutation, Query, Router, UseMiddlewares } from "nestjs-trpc"; +import type { z } from "zod"; +import { AuthMiddleware } from "../trpc/middlewares/auth.middleware"; +import { restMeta } from "../trpc/openapi"; +import { + ingestSignalInput, + ingestSignalOutput, + promoteSignalInput, + promoteSignalOutput, + qualifySignalInput, + qualifySignalOutput, + resolveSignalCompanyInput, + resolveSignalCompanyOutput, + signalCompanyCandidatesOutput, + signalInboxInput, + signalInboxOutput, + signalSourceRecordInput, +} from "./ingest.contracts"; +import { IngestService } from "./ingest.service"; +import { SignalQualificationService } from "./signal-qualification.service"; + +@Router({ alias: "ingest" }) +@UseMiddlewares(AuthMiddleware) +export class IngestRouter { + constructor( + @Inject(IngestService) private readonly ingest: IngestService, + @Inject(SignalQualificationService) + private readonly qualification: SignalQualificationService, + ) {} + + @Mutation({ + input: ingestSignalInput, + output: ingestSignalOutput, + meta: restMeta("POST", "/ingest/signal", ["Ingest"]), + }) + async signal(@Input() input: z.infer) { + return this.ingest.signal(input); + } + + @Query({ + input: signalInboxInput, + output: signalInboxOutput, + meta: restMeta("GET", "/ingest/signals", ["Ingest"]), + }) + async inbox(@Input() input: z.infer) { + return this.ingest.inbox(input); + } + + @Query({ + input: signalSourceRecordInput, + output: signalCompanyCandidatesOutput, + meta: restMeta( + "GET", + "/ingest/signals/{sourceRecordId}/company-candidates", + ["Ingest"], + ), + }) + async companyCandidates(@Input("sourceRecordId") sourceRecordId: string) { + return this.ingest.companyCandidates(sourceRecordId); + } + + @Mutation({ + input: resolveSignalCompanyInput, + output: resolveSignalCompanyOutput, + meta: restMeta("POST", "/ingest/signals/{sourceRecordId}/resolve-company", [ + "Ingest", + ]), + }) + async resolveCompany( + @Input() input: z.infer, + ) { + return this.ingest.resolveCompany(input); + } + + @Mutation({ + input: qualifySignalInput, + output: qualifySignalOutput, + meta: restMeta("POST", "/ingest/signals/{sourceRecordId}/qualify", [ + "Ingest", + ]), + }) + async qualify(@Input() input: z.infer) { + return this.qualification.qualify(input); + } + + @Mutation({ + input: promoteSignalInput, + output: promoteSignalOutput, + meta: restMeta("POST", "/ingest/signals/{sourceRecordId}/promote", [ + "Ingest", + ]), + }) + async promote(@Input() input: z.infer) { + return this.qualification.promote(input); + } +} diff --git a/apps/api/src/ingest/ingest.service.ts b/apps/api/src/ingest/ingest.service.ts new file mode 100644 index 000000000..52f025042 --- /dev/null +++ b/apps/api/src/ingest/ingest.service.ts @@ -0,0 +1,415 @@ +import { randomUUID } from "node:crypto"; +import type { Db } from "@crm/db"; +import { + BadRequestException, + Injectable, + NotFoundException, +} from "@nestjs/common"; +import { z } from "zod"; +import { CompaniesService } from "../companies/companies.service"; +import { normalizeDomain } from "../companies/domain"; +import { InjectDatabase } from "../database/database.constants"; +import type { + IngestSignalInput, + IngestSignalOutput, + ResolveSignalCompanyInput, + ResolveSignalCompanyOutput, + SignalCompanyCandidatesOutput, + SignalInboxInput, + SignalInboxOutput, + SignalPayload, +} from "./ingest.contracts"; +import { signalPayload } from "./ingest.contracts"; + +type BusinessUnitRow = { id: string; enabled: boolean }; +type SourceRecordRow = { id: string }; +type SignalRow = { + id: string; + businessUnitId: string; + project: string; + sourceSystem: string; + sourceType: string; + sourceId: string; + payload: SignalPayload; +}; +type CandidateRow = { + id: string; + name: string; + domain: string | null; + businessUnitId: string | null; +}; +type InboxRow = { + sourceRecordId: string; + project: string; + source: string; + sourceType: string; + sourceId: string; + sourceUrl: string | null; + observedAt: Date; + entity: string | null; + signalScore: number | null; + company: string | null; + subject: string | null; + stageKey: string | null; + priority: string | null; + demandTrigger: string | null; + nextAction: string | null; + mapped: boolean; + payload: SignalPayload; +}; + +@Injectable() +export class IngestService { + constructor( + @InjectDatabase() private readonly db: Db, + private readonly companies: CompaniesService, + ) {} + + async signal(input: IngestSignalInput): Promise { + const [businessUnit] = await this.db.$queryRaw` + SELECT id, enabled FROM business_unit WHERE key = ${input.project} LIMIT 1 + `; + if (!businessUnit) + throw new BadRequestException(`Unknown business unit: ${input.project}.`); + if (!businessUnit.enabled) + throw new BadRequestException( + `Business unit is disabled: ${input.project}.`, + ); + + const [existing] = await this.db.$queryRaw` + SELECT id FROM source_record + WHERE "sourceSystem" = ${input.source} + AND "sourceType" = ${input.sourceType} + AND "sourceId" = ${input.sourceId} + LIMIT 1 + `; + + const observedAt = input.observedAt + ? new Date(input.observedAt) + : new Date(); + const payload = JSON.stringify({ + ...input.payload, + project: input.project, + source: input.source, + source_type: input.sourceType, + source_id: input.sourceId, + entity: input.entity ?? null, + signal_score: input.signalScore ?? null, + tags: input.tags, + ingested_at: new Date().toISOString(), + }); + const sourceRecordId = existing?.id ?? randomUUID(); + + const [saved] = await this.db.$queryRaw` + INSERT INTO source_record ( + id, "businessUnitId", "sourceSystem", "sourceType", "sourceId", + "sourceUrl", "observedAt", payload, "createdAt" + ) + VALUES ( + ${sourceRecordId}, ${businessUnit.id}, ${input.source}, ${input.sourceType}, + ${input.sourceId}, ${input.sourceUrl ?? null}, ${observedAt}, ${payload}::jsonb, + CURRENT_TIMESTAMP + ) + ON CONFLICT ("sourceSystem", "sourceType", "sourceId") DO UPDATE SET + "businessUnitId" = EXCLUDED."businessUnitId", + "sourceUrl" = EXCLUDED."sourceUrl", + "observedAt" = EXCLUDED."observedAt", + payload = EXCLUDED.payload + RETURNING id + `; + if (!saved) + throw new Error("Signal ingest did not return a source record."); + return { + status: "accepted", + sourceRecordId: saved.id, + project: input.project, + deduplicated: Boolean(existing), + promoted: false, + }; + } + + async inbox(input: SignalInboxInput): Promise { + const rows = await this.db.$queryRaw` + SELECT + sr.id AS "sourceRecordId", bu.key AS project, + sr."sourceSystem" AS source, sr."sourceType" AS "sourceType", + sr."sourceId" AS "sourceId", sr."sourceUrl" AS "sourceUrl", + sr."observedAt" AS "observedAt", + COALESCE(sr.payload->>'entity', sr.payload->>'company') AS entity, + COALESCE(NULLIF(sr.payload->>'signal_score','')::float, + NULLIF(sr.payload->'metadata'->>'fit_score','')::float) AS "signalScore", + sr.payload->>'company' AS company, + sr.payload->>'subject' AS subject, + sr.payload->>'stage_key' AS "stageKey", + sr.payload->>'priority' AS priority, + sr.payload->>'demand_trigger' AS "demandTrigger", + sr.payload->>'next_action' AS "nextAction", + EXISTS ( + SELECT 1 FROM record_mapping rm + WHERE rm."sourceSystem" = sr."sourceSystem" + AND rm."sourceType" = sr."sourceType" + AND rm."sourceId" = sr."sourceId" + AND rm."canonicalType" = 'company' AND rm.status = 'active' + ) AS mapped, + sr.payload AS payload + FROM source_record sr + JOIN business_unit bu ON bu.id = sr."businessUnitId" + WHERE (${input.project ?? null}::text IS NULL OR bu.key = ${input.project ?? null}) + AND (${input.minScore ?? null}::float IS NULL OR COALESCE( + NULLIF(sr.payload->>'signal_score','')::float, + NULLIF(sr.payload->'metadata'->>'fit_score','')::float, 0 + ) >= ${input.minScore ?? null}) + AND ( + ${input.status} = 'all' + OR (${input.status} = 'mapped' AND EXISTS ( + SELECT 1 FROM record_mapping rm WHERE rm."sourceSystem" = sr."sourceSystem" + AND rm."sourceType" = sr."sourceType" AND rm."sourceId" = sr."sourceId" + AND rm."canonicalType" = 'company' AND rm.status = 'active')) + OR (${input.status} = 'unresolved' AND NOT EXISTS ( + SELECT 1 FROM record_mapping rm WHERE rm."sourceSystem" = sr."sourceSystem" + AND rm."sourceType" = sr."sourceType" AND rm."sourceId" = sr."sourceId" + AND rm."canonicalType" = 'company' AND rm.status = 'active')) + ) + ORDER BY COALESCE(NULLIF(sr.payload->>'signal_score','')::float, + NULLIF(sr.payload->'metadata'->>'fit_score','')::float, 0) DESC, + sr."observedAt" DESC + LIMIT ${input.limit} + `; + return { + rows: rows.map((row) => ({ + ...row, + observedAt: row.observedAt.toISOString(), + })), + count: rows.length, + }; + } + + async companyCandidates( + sourceRecordId: string, + ): Promise { + const signal = await this.loadSignal(sourceRecordId); + const entity = this.signalEntity(signal.payload); + const domain = this.signalDomain(signal.payload); + const [mapped] = await this.db.$queryRaw< + Array<{ applicationId: string | null }> + >` + SELECT "applicationId" FROM record_mapping + WHERE "sourceSystem" = ${signal.sourceSystem} + AND "sourceType" = ${signal.sourceType} + AND "sourceId" = ${signal.sourceId} + AND "canonicalType" = 'company' AND status = 'active' LIMIT 1 + `; + const rows = + entity || domain + ? await this.db.$queryRaw` + SELECT id, name, domain, "businessUnitId" AS "businessUnitId" + FROM company + WHERE "archivedAt" IS NULL AND ( + (${entity}::text IS NOT NULL AND lower(name) LIKE '%' || lower(${entity}) || '%') + OR (${domain}::text IS NOT NULL AND domain = ${domain}) + ) + LIMIT 20 + ` + : []; + const candidates = rows + .map((row) => + this.scoreCandidate(row, entity, domain, signal.businessUnitId), + ) + .sort((a, b) => b.score - a.score || a.name.localeCompare(b.name)); + return { + sourceRecordId, + project: signal.project, + entity, + domain, + mappedCompanyId: mapped?.applicationId ?? null, + candidates, + }; + } + + async resolveCompany( + input: ResolveSignalCompanyInput, + ): Promise { + const signal = await this.loadSignal(input.sourceRecordId); + const entity = input.companyName ?? this.signalEntity(signal.payload); + const requestedDomain = normalizeDomain( + input.domain ?? this.signalDomain(signal.payload), + ); + let matchMethod = "manual"; + let created = false; + let company: CandidateRow | undefined; + + if (input.companyId) { + [company] = await this.db.$queryRaw` + SELECT id, name, domain, "businessUnitId" AS "businessUnitId" + FROM company WHERE id = ${input.companyId} AND "archivedAt" IS NULL LIMIT 1 + `; + if (!company) + throw new NotFoundException(`No company with id ${input.companyId}.`); + } + if (!company && requestedDomain) { + [company] = await this.db.$queryRaw` + SELECT id, name, domain, "businessUnitId" AS "businessUnitId" + FROM company WHERE domain = ${requestedDomain} AND "archivedAt" IS NULL LIMIT 1 + `; + if (company) matchMethod = "domain"; + } + if (!company && entity) { + [company] = await this.db.$queryRaw` + SELECT id, name, domain, "businessUnitId" AS "businessUnitId" + FROM company WHERE lower(name) = lower(${entity}) AND "archivedAt" IS NULL LIMIT 1 + `; + if (company) matchMethod = "name-exact"; + } + if (!company && !input.createIfMissing) { + throw new BadRequestException( + "No exact company match. Review the company-candidates endpoint or set createIfMissing=true after verification.", + ); + } + if (!company) { + if (!entity) + throw new BadRequestException( + "A company name is required to create a company.", + ); + const made = await this.companies.create({ + name: entity, + domain: requestedDomain ?? undefined, + }); + await this.db.$queryRaw` + UPDATE company SET "businessUnitId" = ${signal.businessUnitId} + WHERE id = ${made.id} AND "businessUnitId" IS NULL + `; + company = { + id: made.id, + name: made.name, + domain: made.domain, + businessUnitId: signal.businessUnitId, + }; + matchMethod = "created"; + created = true; + } + + const normalizedName = company.name.trim().toLowerCase(); + const [canonical] = await this.db.$queryRaw>` + INSERT INTO canonical_company ( + id, "businessUnitId", name, "normalizedName", domain, website, fields, + "createdAt", "updatedAt" + ) + VALUES ( + ${randomUUID()}, ${signal.businessUnitId}, ${company.name}, ${normalizedName}, + ${company.domain}, ${company.domain ? `https://${company.domain}` : null}, + ${JSON.stringify({ source_signal_id: signal.id })}::jsonb, + CURRENT_TIMESTAMP, CURRENT_TIMESTAMP + ) + ON CONFLICT ("businessUnitId", "normalizedName") DO UPDATE SET + domain = COALESCE(canonical_company.domain, EXCLUDED.domain), + website = COALESCE(canonical_company.website, EXCLUDED.website), + "updatedAt" = CURRENT_TIMESTAMP + RETURNING id + `; + if (!canonical) throw new Error("Canonical company resolution failed."); + + await this.db.$queryRaw` + INSERT INTO record_mapping ( + id, "sourceSystem", "sourceType", "sourceId", "canonicalType", "canonicalId", + application, "applicationId", "matchMethod", status, "createdAt", "updatedAt" + ) + VALUES ( + ${randomUUID()}, ${signal.sourceSystem}, ${signal.sourceType}, ${signal.sourceId}, + 'company', ${canonical.id}, 'comp-ai-core', ${company.id}, ${matchMethod}, 'active', + CURRENT_TIMESTAMP, CURRENT_TIMESTAMP + ) + ON CONFLICT ("sourceSystem", "sourceType", "sourceId", "canonicalType") DO UPDATE SET + "canonicalId" = EXCLUDED."canonicalId", + application = EXCLUDED.application, + "applicationId" = EXCLUDED."applicationId", + "matchMethod" = EXCLUDED."matchMethod", + status = 'active', "updatedAt" = CURRENT_TIMESTAMP + `; + + let researchQueued = false; + if (input.queueResearch) + researchQueued = (await this.companies.enrich(company.id)).queued; + return { + sourceRecordId: input.sourceRecordId, + canonicalCompanyId: canonical.id, + companyId: company.id, + companyName: company.name, + matchMethod, + created, + researchQueued, + }; + } + + private async loadSignal(sourceRecordId: string): Promise { + const [row] = await this.db.$queryRaw` + SELECT sr.id, sr."businessUnitId" AS "businessUnitId", bu.key AS project, + sr."sourceSystem" AS "sourceSystem", sr."sourceType" AS "sourceType", + sr."sourceId" AS "sourceId", sr.payload + FROM source_record sr JOIN business_unit bu ON bu.id = sr."businessUnitId" + WHERE sr.id = ${sourceRecordId} LIMIT 1 + `; + if (!row) + throw new NotFoundException( + `No source signal with id ${sourceRecordId}.`, + ); + return { ...row, payload: signalPayload.parse(row.payload) }; + } + + private signalEntity(payload: SignalPayload): string | null { + for (const key of ["entity", "company"] as const) { + const value = z.string().trim().min(1).safeParse(payload[key]); + if (value.success) return value.data; + } + return null; + } + + private signalDomain(payload: SignalPayload): string | null { + for (const key of ["domain", "website"] as const) { + const value = z.string().safeParse(payload[key]); + if (!value.success) continue; + const domain = normalizeDomain(value.data); + if (domain) return domain; + } + return null; + } + + private scoreCandidate( + row: CandidateRow, + entity: string | null, + domain: string | null, + businessUnitId: string, + ) { + let score = 0; + const reasons: string[] = []; + if (domain && row.domain === domain) { + score += 100; + reasons.push("exact-domain"); + } + if (entity) { + const wanted = entity.toLowerCase(); + const actual = row.name.toLowerCase(); + if (actual === wanted) { + score += 95; + reasons.push("exact-name"); + } else if (actual.startsWith(wanted) || wanted.startsWith(actual)) { + score += 80; + reasons.push("name-prefix"); + } else if (actual.includes(wanted) || wanted.includes(actual)) { + score += 70; + reasons.push("name-contains"); + } + } + if (row.businessUnitId === businessUnitId) { + score += 10; + reasons.push("same-business-unit"); + } + return { + companyId: row.id, + name: row.name, + domain: row.domain, + businessUnitId: row.businessUnitId, + score: Math.min(110, score), + reasons, + }; + } +} diff --git a/apps/api/src/ingest/signal-qualification.service.ts b/apps/api/src/ingest/signal-qualification.service.ts new file mode 100644 index 000000000..c00241c4f --- /dev/null +++ b/apps/api/src/ingest/signal-qualification.service.ts @@ -0,0 +1,308 @@ +import { randomUUID } from "node:crypto"; +import type { Db } from "@crm/db"; +import { + BadRequestException, + Injectable, + NotFoundException, +} from "@nestjs/common"; +import { z } from "zod"; +import { InjectDatabase } from "../database/database.constants"; +import { DealsService } from "../deals/deals.service"; +import type { + PromoteSignalInput, + PromoteSignalOutput, + QualifySignalInput, + QualifySignalOutput, + SignalClassification, + SignalScoreComponents, +} from "./ingest.contracts"; +import { + type SignalPayload, + type SignalPayloadValue, + signalPayload, +} from "./ingest.contracts"; + +type SignalRow = { + id: string; + businessUnitId: string; + sourceSystem: string; + sourceType: string; + sourceId: string; + payload: SignalPayload; +}; + +type CompanyMapping = { + canonicalId: string; + applicationId: string | null; +}; + +type OpportunityMapping = { + canonicalId: string; + applicationId: string | null; +}; + +@Injectable() +export class SignalQualificationService { + constructor( + @InjectDatabase() private readonly db: Db, + private readonly deals: DealsService, + ) {} + + async qualify(input: QualifySignalInput): Promise { + const signal = await this.loadSignal(input.sourceRecordId); + const scored = this.score(signal.payload, input.components); + const company = await this.companyMapping(signal); + + const qualification = { + score: scored.score, + classification: scored.classification, + method: scored.method, + components: input.components ?? null, + evidence: input.evidence, + notes: input.notes ?? null, + qualified_at: new Date().toISOString(), + }; + + await this.db.$queryRaw` + UPDATE source_record + SET payload = payload || ${JSON.stringify({ + signal_score: scored.score, + qualification, + })}::jsonb + WHERE id = ${signal.id} + `; + + return { + sourceRecordId: signal.id, + score: scored.score, + classification: scored.classification, + method: scored.method, + companyResolved: Boolean(company?.applicationId), + canonicalOpportunityEligible: scored.score >= 70 && Boolean(company), + visibleDealEligible: + scored.score >= 85 && Boolean(company?.applicationId), + }; + } + + async promote(input: PromoteSignalInput): Promise { + const signal = await this.loadSignal(input.sourceRecordId); + const scored = this.score(signal.payload); + if (scored.score < 70) { + throw new BadRequestException( + `Signal score ${scored.score} is below the canonical opportunity threshold of 70.`, + ); + } + + const company = await this.companyMapping(signal); + if (!company) { + throw new BadRequestException( + "Resolve the signal to a company before promoting it.", + ); + } + + const payload = signal.payload; + const name = + this.text(payload.subject) ?? + this.text(payload.message) ?? + "Qualified opportunity"; + const amount = input.amountUsd ?? this.number(payload.estimated_value_usd); + const stage = scored.score >= 85 ? "priority" : "qualified"; + + let createdCanonicalOpportunity = false; + let canonicalOpportunityId: string; + const existing = await this.opportunityMapping(signal); + if (existing) { + canonicalOpportunityId = existing.canonicalId; + await this.db.$queryRaw` + UPDATE canonical_opportunity + SET "companyId" = ${company.canonicalId}, + name = ${name}, stage = ${stage}, amount = ${amount}, + fields = COALESCE(fields, '{}'::jsonb) || ${JSON.stringify({ + source_signal_id: signal.id, + signal_score: scored.score, + classification: scored.classification, + })}::jsonb, + "updatedAt" = CURRENT_TIMESTAMP + WHERE id = ${canonicalOpportunityId} + `; + } else { + canonicalOpportunityId = randomUUID(); + await this.db.$queryRaw` + INSERT INTO canonical_opportunity ( + id, "businessUnitId", "companyId", "pipelineId", name, stage, + amount, fields, "createdAt", "updatedAt" + ) + VALUES ( + ${canonicalOpportunityId}, ${signal.businessUnitId}, ${company.canonicalId}, + NULL, ${name}, ${stage}, ${amount}, ${JSON.stringify({ + source_signal_id: signal.id, + signal_score: scored.score, + classification: scored.classification, + })}::jsonb, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP + ) + `; + createdCanonicalOpportunity = true; + } + + let dealId = existing?.applicationId ?? null; + let createdDeal = false; + if (input.createDeal) { + if (scored.score < 85) { + throw new BadRequestException( + `Signal score ${scored.score} is below the visible deal threshold of 85.`, + ); + } + if (!company.applicationId) { + throw new BadRequestException( + "The resolved company is not linked to a visible Comp company.", + ); + } + if (!dealId) { + if (!input.ownerId) { + throw new BadRequestException( + "ownerId is required when creating a visible deal.", + ); + } + const deal = await this.deals.create({ + name, + companyId: company.applicationId, + ownerId: input.ownerId, + stage: "DEMO_BOOKED", + amountCents: amount == null ? null : Math.round(amount * 100), + currency: "USD", + }); + dealId = deal.id; + createdDeal = true; + await this.db.$queryRaw` + UPDATE deal SET "businessUnitId" = ${signal.businessUnitId} + WHERE id = ${deal.id} + `; + } + } + + await this.db.$queryRaw` + INSERT INTO record_mapping ( + id, "sourceSystem", "sourceType", "sourceId", "canonicalType", "canonicalId", + application, "applicationId", "matchMethod", status, "createdAt", "updatedAt" + ) + VALUES ( + ${randomUUID()}, ${signal.sourceSystem}, ${signal.sourceType}, ${signal.sourceId}, + 'opportunity', ${canonicalOpportunityId}, 'comp-ai-crm', ${dealId}, + 'signal-score', 'active', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP + ) + ON CONFLICT ("sourceSystem", "sourceType", "sourceId", "canonicalType") DO UPDATE SET + "canonicalId" = EXCLUDED."canonicalId", + application = EXCLUDED.application, + "applicationId" = COALESCE(EXCLUDED."applicationId", record_mapping."applicationId"), + "matchMethod" = EXCLUDED."matchMethod", status = 'active', + "updatedAt" = CURRENT_TIMESTAMP + `; + + return { + sourceRecordId: signal.id, + score: scored.score, + classification: scored.classification, + canonicalOpportunityId, + dealId, + createdCanonicalOpportunity, + createdDeal, + }; + } + + private score(payload: SignalPayload, components?: SignalScoreComponents) { + if (components) { + const score = Math.round( + components.icpMatch + + components.commercialTrigger + + components.projectRelevance + + components.companyValue + + components.location + + components.decisionMaker + + components.recency, + ); + return { + score, + classification: this.classification(score), + method: "components" as const, + }; + } + const explicit = this.number(payload.signal_score); + const metadata = this.object(payload.metadata); + const legacy = explicit ?? this.number(metadata?.fit_score); + if (legacy != null) { + const score = Math.max(0, Math.min(100, Math.round(legacy))); + return { + score, + classification: this.classification(score), + method: "legacy-score" as const, + }; + } + return { + score: 0, + classification: "signal" as const, + method: "unscored" as const, + }; + } + + private classification(score: number): SignalClassification { + if (score >= 85) return "priority"; + if (score >= 70) return "qualified"; + if (score >= 50) return "research"; + return "signal"; + } + + private async loadSignal(id: string): Promise { + const [row] = await this.db.$queryRaw` + SELECT id, "businessUnitId" AS "businessUnitId", "sourceSystem" AS "sourceSystem", + "sourceType" AS "sourceType", "sourceId" AS "sourceId", payload + FROM source_record WHERE id = ${id} LIMIT 1 + `; + if (!row) throw new NotFoundException(`No source signal with id ${id}.`); + return { ...row, payload: signalPayload.parse(row.payload) }; + } + + private async companyMapping( + signal: SignalRow, + ): Promise { + const [row] = await this.db.$queryRaw` + SELECT "canonicalId" AS "canonicalId", "applicationId" AS "applicationId" + FROM record_mapping + WHERE "sourceSystem" = ${signal.sourceSystem} AND "sourceType" = ${signal.sourceType} + AND "sourceId" = ${signal.sourceId} AND "canonicalType" = 'company' + AND status = 'active' LIMIT 1 + `; + return row ?? null; + } + + private async opportunityMapping( + signal: SignalRow, + ): Promise { + const [row] = await this.db.$queryRaw` + SELECT "canonicalId" AS "canonicalId", "applicationId" AS "applicationId" + FROM record_mapping + WHERE "sourceSystem" = ${signal.sourceSystem} AND "sourceType" = ${signal.sourceType} + AND "sourceId" = ${signal.sourceId} AND "canonicalType" = 'opportunity' + AND status = 'active' LIMIT 1 + `; + return row ?? null; + } + + private object(value: SignalPayloadValue | undefined): SignalPayload | null { + const parsed = signalPayload.safeParse(value); + return parsed.success ? parsed.data : null; + } + + private text(value: SignalPayloadValue | undefined): string | null { + const parsed = z.string().trim().min(1).safeParse(value); + return parsed.success ? parsed.data : null; + } + + private number(value: SignalPayloadValue | undefined): number | null { + const numeric = z.number().finite().safeParse(value); + if (numeric.success) return numeric.data; + const text = z.string().trim().min(1).safeParse(value); + if (!text.success) return null; + const parsed = Number(text.data); + return Number.isFinite(parsed) ? parsed : null; + } +} diff --git a/apps/app/app/(app)/[slug]/intelligence/intelligence-inbox.tsx b/apps/app/app/(app)/[slug]/intelligence/intelligence-inbox.tsx new file mode 100644 index 000000000..413e5d7fe --- /dev/null +++ b/apps/app/app/(app)/[slug]/intelligence/intelligence-inbox.tsx @@ -0,0 +1,650 @@ +"use client"; + +import Launch from "@carbon/icons-react/es/Launch"; +import Renew from "@carbon/icons-react/es/Renew"; +import Search from "@carbon/icons-react/es/Search"; +import { Badge } from "@crm/ui/components/badge"; +import { Button } from "@crm/ui/components/button"; +import { Icon } from "@crm/ui/components/icon"; +import { Input } from "@crm/ui/components/input"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@crm/ui/components/select"; +import { + Sheet, + SheetContent, + SheetDescription, + SheetHeader, + SheetTitle, +} from "@crm/ui/components/sheet"; +import { Spinner } from "@crm/ui/components/spinner"; +import { useMutation, useQuery } from "@tanstack/react-query"; +import { useMemo, useState } from "react"; +import { toast } from "sonner"; +import { LocalRelativeTime } from "@/components/local-date-time"; +import { useTRPC } from "@/lib/trpc/client"; +import type { RouterOutputs } from "@/lib/trpc/types"; + +type SignalRow = RouterOutputs["ingest"]["inbox"]["rows"][number]; +type InboxStatus = "all" | "unresolved" | "mapped"; + +type ScoreComponents = { + icpMatch: number; + commercialTrigger: number; + projectRelevance: number; + companyValue: number; + location: number; + decisionMaker: number; + recency: number; +}; + +const EMPTY_SCORE: ScoreComponents = { + icpMatch: 0, + commercialTrigger: 0, + projectRelevance: 0, + companyValue: 0, + location: 0, + decisionMaker: 0, + recency: 0, +}; + +export function IntelligenceInbox({ + initialInput, +}: { + initialInput: { status: "unresolved"; limit: number }; +}) { + const trpc = useTRPC(); + const [project, setProject] = useState("all"); + const [status, setStatus] = useState(initialInput.status); + const [minScore, setMinScore] = useState("0"); + const [selected, setSelected] = useState(null); + + const input = useMemo( + () => ({ + status, + limit: initialInput.limit, + project: project === "all" ? undefined : project, + minScore: Number(minScore) > 0 ? Number(minScore) : undefined, + }), + [status, initialInput.limit, project, minScore], + ); + + const signals = useQuery({ + ...trpc.ingest.inbox.queryOptions(input), + placeholderData: (previous) => previous, + }); + + const rows = signals.data?.rows ?? []; + const projects = useMemo( + () => [...new Set(rows.map((row) => row.project))].sort(), + [rows], + ); + + return ( + <> +
    +
    + + {signals.data?.count ?? 0} signals in this view +
    +
    + + + + +
    +
    + +
    + + + + + + + + + + + + + + {rows.map((row) => ( + + + + + + + + + + ))} + {rows.length === 0 ? ( + + + + ) : null} + +
    SignalProjectSourceScoreCompanyObservedAction
    +
    + {row.subject ?? row.entity ?? row.sourceId} +
    + {row.demandTrigger ? ( +
    + {row.demandTrigger} +
    + ) : null} +     		+ {row.project} + + {row.source} + + + +
    + {row.company ?? row.entity ?? "Unresolved"} +
    +
    + + {row.mapped ? "Resolved" : "Needs match"} + +
    +     		+ + + +
    + No signals match this view. +
    +
    + + setSelected(null)} + onChanged={async () => { + await signals.refetch(); + }} + /> + + ); +} + +function SignalReviewSheet({ + row, + onClose, + onChanged, +}: { + row: SignalRow | null; + onClose: () => void; + onChanged: () => Promise; +}) { + const trpc = useTRPC(); + const [score, setScore] = useState(EMPTY_SCORE); + const [ownerId, setOwnerId] = useState("unassigned"); + + const candidates = useQuery({ + ...trpc.ingest.companyCandidates.queryOptions({ + sourceRecordId: row?.sourceRecordId ?? "", + }), + enabled: Boolean(row), + }); + const users = useQuery(trpc.users.list.queryOptions()); + + const refresh = async () => { + await Promise.all([onChanged(), candidates.refetch()]); + }; + + const resolve = useMutation( + trpc.ingest.resolveCompany.mutationOptions({ + onSuccess: async (result) => { + toast.success( + `${result.companyName} resolved${result.researchQueued ? " and queued for research" : ""}.`, + ); + await refresh(); + }, + onError: (error) => toast.error(error.message), + }), + ); + const qualify = useMutation( + trpc.ingest.qualify.mutationOptions({ + onSuccess: async (result) => { + toast.success( + `Signal scored ${result.score}/100 — ${result.classification}.`, + ); + await refresh(); + }, + onError: (error) => toast.error(error.message), + }), + ); + const promote = useMutation( + trpc.ingest.promote.mutationOptions({ + onSuccess: async (result) => { + toast.success( + result.createdDeal + ? "Opportunity and visible deal created." + : "Canonical opportunity created.", + ); + await refresh(); + }, + onError: (error) => toast.error(error.message), + }), + ); + const research = useMutation( + trpc.companies.enrich.mutationOptions({ + onSuccess: (result) => + toast.success( + result.queued + ? "Company research queued." + : "Research is already queued.", + ), + onError: (error) => toast.error(error.message), + }), + ); + + if (!row) return null; + const mappedCompanyId = candidates.data?.mappedCompanyId ?? null; + const pending = + resolve.isPending || + qualify.isPending || + promote.isPending || + research.isPending; + + return ( + { + if (!open) onClose(); + }} + > + + + + {row.subject ?? row.entity ?? "Signal review"} + + + {row.project} · {row.source} · {row.sourceType} + + + +
    +
    +
    +
    +
    + Current score +
    +
    + +
    +
    + + {row.mapped ? "Company resolved" : "Company unresolved"} + +
    + {row.demandTrigger ? ( + + ) : null} + {row.nextAction ? ( + + ) : null} + {row.sourceUrl ? ( + + ) : null} +
    + +
    +
    +

    1. Resolve company

    +

    + Match this signal to one existing account before research or + opportunity promotion. +

    +
    + {candidates.isLoading ? : null} + {mappedCompanyId ? ( +
    + Mapped to Comp company + +
    + ) : ( +
    + {(candidates.data?.candidates ?? []) + .slice(0, 5) + .map((candidate) => ( +
    +
    +
    + {candidate.name} +
    +
    + {candidate.domain ?? "No domain"} · match{" "} + {candidate.score} +
    +
    + +
    + ))} + {(candidates.data?.candidates ?? []).length === 0 ? ( +
    + No existing company candidate found. Create only after + verifying the entity name. +
    + ) : null} + +
    + )} +
    + +
    +
    +

    2. Qualify signal

    +

    + Use the transparent 100-point commercial score. +

    +
    +
    + + setScore((s) => ({ ...s, icpMatch: value })) + } + /> + + setScore((s) => ({ ...s, commercialTrigger: value })) + } + /> + + setScore((s) => ({ ...s, projectRelevance: value })) + } + /> + + setScore((s) => ({ ...s, companyValue: value })) + } + /> + + setScore((s) => ({ ...s, location: value })) + } + /> + + setScore((s) => ({ ...s, decisionMaker: value })) + } + /> + + setScore((s) => ({ ...s, recency: value })) + } + /> +
    +
    + Total +
    + {Object.values(score).reduce( + (sum, value) => sum + value, + 0, + )}{" "} + / 100 +
    +
    +
    +
    +
    + + {row.signalScore != null ? ( + + ) : null} +
    +
    + +
    +
    +

    3. Promote

    +

    + 70+ can become a canonical opportunity. A visible deal requires + 85+, a resolved company, and an owner. +

    +
    +
    + +
    +
    + + +
    +
    +
    +     +     + ); +} + +function ScoreBadge({ score }: { score: number | null }) { + if (score == null) return Unscored; + if (score >= 85) return {score} · Priority; + if (score >= 70) + return {score} · Qualified; + if (score >= 50) return {score} · Research; + return {score} · Signal; +} + +function Info({ label, value }: { label: string; value: string }) { + return ( +
    +
    {label}
    +
    {value}
    +
    + ); +} + +function ScoreInput({ + label, + value, + max, + onChange, +}: { + label: string; + value: number; + max: number; + onChange: (value: number) => void; +}) { + const inputId = `score-${label.toLowerCase().replaceAll(" ", "-")}`; + return ( +
    + + {label} / {max} + + + onChange(Math.max(0, Math.min(max, Number(event.target.value) || 0))) + } + aria-label={label} + /> +
    + ); +} diff --git a/apps/app/app/(app)/[slug]/intelligence/page.tsx b/apps/app/app/(app)/[slug]/intelligence/page.tsx new file mode 100644 index 000000000..3085f4308 --- /dev/null +++ b/apps/app/app/(app)/[slug]/intelligence/page.tsx @@ -0,0 +1,62 @@ +import type { Metadata } from "next"; +import { Suspense } from "react"; +import { + PageShell, + PageShellContent, + PageShellDescription, + PageShellHeader, + PageShellHeading, + PageShellLoading, + PageShellTitle, +} from "@/components/page-shell"; +import { requireSession } from "@/lib/session"; +import { HydrateClient } from "@/lib/trpc/hydrate"; +import { getServerQueryClient, getServerTrpc } from "@/lib/trpc/server"; +import { IntelligenceInbox } from "./intelligence-inbox"; + +export const metadata: Metadata = { + title: "Intelligence", +}; + +const DEFAULT_INBOX = { + status: "unresolved" as const, + limit: 100, +}; + +export default function IntelligencePage() { + return ( + + + + Intelligence + + Review incoming commercial signals before they become CRM records or + opportunities. + + + + + + }> + + + + + ); +} + +async function Intelligence() { + await requireSession(); + const trpc = getServerTrpc(); + const queryClient = getServerQueryClient(); + await Promise.all([ + queryClient.prefetchQuery(trpc.ingest.inbox.queryOptions(DEFAULT_INBOX)), + queryClient.prefetchQuery(trpc.users.list.queryOptions()), + ]); + + return ( + + + + ); +} diff --git a/apps/app/components/app-icon-rail.tsx b/apps/app/components/app-icon-rail.tsx index ad3440e00..f5434c7cf 100644 --- a/apps/app/components/app-icon-rail.tsx +++ b/apps/app/components/app-icon-rail.tsx @@ -4,6 +4,7 @@ import Building from "@carbon/icons-react/es/Building"; import Close from "@carbon/icons-react/es/Close"; import Dashboard from "@carbon/icons-react/es/Dashboard"; import Partnership from "@carbon/icons-react/es/Partnership"; +import Search from "@carbon/icons-react/es/Search"; import Settings from "@carbon/icons-react/es/Settings"; import UserMultiple from "@carbon/icons-react/es/UserMultiple"; import { Button } from "@crm/ui/components/button"; @@ -49,6 +50,12 @@ const ITEMS: RailItem[] = [ match: "prefix", related: ["/agents"], }, + { + title: "Intelligence", + href: "/intelligence", + icon: Search, + match: "prefix", + }, { title: "Companies", href: "/companies", icon: Building, match: "prefix" }, { title: "Contacts", diff --git a/apps/app/components/crm/section-prefetch.ts b/apps/app/components/crm/section-prefetch.ts index 8636fa5ff..9dd736fd9 100644 --- a/apps/app/components/crm/section-prefetch.ts +++ b/apps/app/components/crm/section-prefetch.ts @@ -7,7 +7,13 @@ import { contactsSearchParams } from "@/app/(app)/[slug]/contacts/contacts-searc import { dealsSearchParams } from "@/app/(app)/[slug]/deals/deals-search-params"; import { useTRPC } from "@/lib/trpc/client"; -export type Section = "/" | "/companies" | "/contacts" | "/deals" | "/settings"; +export type Section = + | "/" + | "/intelligence" + | "/companies" + | "/contacts" + | "/deals" + | "/settings"; export function usePrefetchSection(): (section: string) => void { const trpc = useTRPC(); @@ -21,6 +27,14 @@ export function usePrefetchSection(): (section: string) => void { trpc.dashboard.summary.queryOptions({ scope: "me" }), ); return; + case "/intelligence": + void queryClient.prefetchQuery( + trpc.ingest.inbox.queryOptions({ + status: "unresolved", + limit: 100, + }), + ); + return; case "/companies": void queryClient.prefetchQuery( trpc.companies.list.queryOptions( diff --git a/packages/db/prisma/migrations/20260902184536_sequencenow_crm_foundation/migration.sql b/packages/db/prisma/migrations/20260902184536_sequencenow_crm_foundation/migration.sql new file mode 100644 index 000000000..3427a8483 --- /dev/null +++ b/packages/db/prisma/migrations/20260902184536_sequencenow_crm_foundation/migration.sql @@ -0,0 +1,318 @@ +-- AlterTable +ALTER TABLE "agentTask" ADD COLUMN "businessUnitId" TEXT; + +-- AlterTable +ALTER TABLE "company" ADD COLUMN "businessUnitId" TEXT; + +-- AlterTable +ALTER TABLE "deal" ADD COLUMN "businessUnitId" TEXT; + +-- CreateTable +CREATE TABLE "business_unit" ( + "id" TEXT NOT NULL, + "key" TEXT NOT NULL, + "name" TEXT NOT NULL, + "description" TEXT, + "enabled" BOOLEAN NOT NULL DEFAULT true, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "business_unit_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "canonical_company" ( + "id" TEXT NOT NULL, + "businessUnitId" TEXT NOT NULL, + "name" TEXT NOT NULL, + "normalizedName" TEXT NOT NULL, + "domain" TEXT, + "phone" TEXT, + "website" TEXT, + "fields" JSONB, + "verifiedAt" TIMESTAMP(3), + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "canonical_company_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "person" ( + "id" TEXT NOT NULL, + "businessUnitId" TEXT NOT NULL, + "firstName" TEXT NOT NULL, + "lastName" TEXT, + "normalizedName" TEXT NOT NULL, + "email" TEXT, + "phone" TEXT, + "title" TEXT, + "fields" JSONB, + "verifiedAt" TIMESTAMP(3), + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "person_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "company_person" ( + "companyId" TEXT NOT NULL, + "personId" TEXT NOT NULL, + "role" TEXT, + "isPrimary" BOOLEAN NOT NULL DEFAULT false, + + CONSTRAINT "company_person_pkey" PRIMARY KEY ("companyId","personId") +); + +-- CreateTable +CREATE TABLE "pipeline" ( + "id" TEXT NOT NULL, + "businessUnitId" TEXT NOT NULL, + "name" TEXT NOT NULL, + "stages" JSONB NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "pipeline_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "canonical_opportunity" ( + "id" TEXT NOT NULL, + "businessUnitId" TEXT NOT NULL, + "companyId" TEXT, + "pipelineId" TEXT, + "name" TEXT NOT NULL, + "stage" TEXT NOT NULL, + "amount" DECIMAL(14,2), + "fields" JSONB, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "canonical_opportunity_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "source_record" ( + "id" TEXT NOT NULL, + "businessUnitId" TEXT NOT NULL, + "sourceSystem" TEXT NOT NULL, + "sourceType" TEXT NOT NULL, + "sourceId" TEXT NOT NULL, + "sourceUrl" TEXT, + "observedAt" TIMESTAMP(3), + "payload" JSONB, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "companyId" TEXT, + "personId" TEXT, + "opportunityId" TEXT, + + CONSTRAINT "source_record_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "record_mapping" ( + "id" TEXT NOT NULL, + "sourceSystem" TEXT NOT NULL, + "sourceType" TEXT NOT NULL, + "sourceId" TEXT NOT NULL, + "canonicalType" TEXT NOT NULL, + "canonicalId" TEXT NOT NULL, + "application" TEXT, + "applicationId" TEXT, + "matchMethod" TEXT NOT NULL, + "status" TEXT NOT NULL DEFAULT 'active', + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "record_mapping_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "campaign" ( + "id" TEXT NOT NULL, + "businessUnitId" TEXT NOT NULL, + "name" TEXT NOT NULL, + "status" TEXT NOT NULL DEFAULT 'draft', + "segment" JSONB, + "template" TEXT, + "sendingEnabled" BOOLEAN NOT NULL DEFAULT false, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "campaign_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "campaign_member" ( + "id" TEXT NOT NULL, + "campaignId" TEXT NOT NULL, + "companyId" TEXT, + "personId" TEXT, + "state" TEXT NOT NULL DEFAULT 'queued', + "draft" TEXT, + "sentAt" TIMESTAMP(3), + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "campaign_member_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "outreach_approval" ( + "id" TEXT NOT NULL, + "campaignId" TEXT NOT NULL, + "memberId" TEXT, + "state" TEXT NOT NULL DEFAULT 'pending', + "approvedBy" TEXT, + "approvedAt" TIMESTAMP(3), + "rejectionReason" TEXT, + "immutableHash" TEXT NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "outreach_approval_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "suppression_entry" ( + "id" TEXT NOT NULL, + "value" TEXT NOT NULL, + "normalized" TEXT NOT NULL, + "kind" TEXT NOT NULL, + "reason" TEXT, + "source" TEXT, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "suppression_entry_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "agent_run_context" ( + "id" TEXT NOT NULL, + "businessUnitId" TEXT NOT NULL, + "agentRunId" TEXT NOT NULL, + "budget" INTEGER NOT NULL DEFAULT 0, + "fitScores" JSONB, + "reasons" JSONB, + "evidenceCount" INTEGER NOT NULL DEFAULT 0, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "finishedAt" TIMESTAMP(3), + + CONSTRAINT "agent_run_context_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE UNIQUE INDEX "business_unit_key_key" ON "business_unit"("key"); + +-- CreateIndex +CREATE INDEX "business_unit_enabled_key_idx" ON "business_unit"("enabled", "key"); + +-- CreateIndex +CREATE INDEX "canonical_company_domain_idx" ON "canonical_company"("domain"); + +-- CreateIndex +CREATE UNIQUE INDEX "canonical_company_businessUnitId_normalizedName_key" ON "canonical_company"("businessUnitId", "normalizedName"); + +-- CreateIndex +CREATE INDEX "person_email_idx" ON "person"("email"); + +-- CreateIndex +CREATE INDEX "person_phone_idx" ON "person"("phone"); + +-- CreateIndex +CREATE UNIQUE INDEX "pipeline_businessUnitId_name_key" ON "pipeline"("businessUnitId", "name"); + +-- CreateIndex +CREATE INDEX "canonical_opportunity_businessUnitId_stage_idx" ON "canonical_opportunity"("businessUnitId", "stage"); + +-- CreateIndex +CREATE INDEX "source_record_businessUnitId_sourceType_idx" ON "source_record"("businessUnitId", "sourceType"); + +-- CreateIndex +CREATE UNIQUE INDEX "source_record_sourceSystem_sourceType_sourceId_key" ON "source_record"("sourceSystem", "sourceType", "sourceId"); + +-- CreateIndex +CREATE INDEX "record_mapping_canonicalType_canonicalId_idx" ON "record_mapping"("canonicalType", "canonicalId"); + +-- CreateIndex +CREATE UNIQUE INDEX "record_mapping_sourceSystem_sourceType_sourceId_canonicalTy_key" ON "record_mapping"("sourceSystem", "sourceType", "sourceId", "canonicalType"); + +-- CreateIndex +CREATE INDEX "campaign_businessUnitId_status_idx" ON "campaign"("businessUnitId", "status"); + +-- CreateIndex +CREATE INDEX "campaign_member_state_idx" ON "campaign_member"("state"); + +-- CreateIndex +CREATE UNIQUE INDEX "campaign_member_campaignId_companyId_personId_key" ON "campaign_member"("campaignId", "companyId", "personId"); + +-- CreateIndex +CREATE INDEX "outreach_approval_campaignId_state_idx" ON "outreach_approval"("campaignId", "state"); + +-- CreateIndex +CREATE UNIQUE INDEX "suppression_entry_normalized_kind_key" ON "suppression_entry"("normalized", "kind"); + +-- CreateIndex +CREATE INDEX "agent_run_context_businessUnitId_createdAt_idx" ON "agent_run_context"("businessUnitId", "createdAt"); + +-- CreateIndex +CREATE UNIQUE INDEX "agent_run_context_agentRunId_key" ON "agent_run_context"("agentRunId"); + +-- AddForeignKey +ALTER TABLE "company" ADD CONSTRAINT "company_businessUnitId_fkey" FOREIGN KEY ("businessUnitId") REFERENCES "business_unit"("id") ON DELETE SET NULL ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "agentTask" ADD CONSTRAINT "agentTask_businessUnitId_fkey" FOREIGN KEY ("businessUnitId") REFERENCES "business_unit"("id") ON DELETE SET NULL ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "deal" ADD CONSTRAINT "deal_businessUnitId_fkey" FOREIGN KEY ("businessUnitId") REFERENCES "business_unit"("id") ON DELETE SET NULL ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "canonical_company" ADD CONSTRAINT "canonical_company_businessUnitId_fkey" FOREIGN KEY ("businessUnitId") REFERENCES "business_unit"("id") ON DELETE RESTRICT ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "person" ADD CONSTRAINT "person_businessUnitId_fkey" FOREIGN KEY ("businessUnitId") REFERENCES "business_unit"("id") ON DELETE RESTRICT ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "company_person" ADD CONSTRAINT "company_person_companyId_fkey" FOREIGN KEY ("companyId") REFERENCES "canonical_company"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "company_person" ADD CONSTRAINT "company_person_personId_fkey" FOREIGN KEY ("personId") REFERENCES "person"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "pipeline" ADD CONSTRAINT "pipeline_businessUnitId_fkey" FOREIGN KEY ("businessUnitId") REFERENCES "business_unit"("id") ON DELETE RESTRICT ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "canonical_opportunity" ADD CONSTRAINT "canonical_opportunity_businessUnitId_fkey" FOREIGN KEY ("businessUnitId") REFERENCES "business_unit"("id") ON DELETE RESTRICT ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "canonical_opportunity" ADD CONSTRAINT "canonical_opportunity_companyId_fkey" FOREIGN KEY ("companyId") REFERENCES "canonical_company"("id") ON DELETE SET NULL ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "canonical_opportunity" ADD CONSTRAINT "canonical_opportunity_pipelineId_fkey" FOREIGN KEY ("pipelineId") REFERENCES "pipeline"("id") ON DELETE SET NULL ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "source_record" ADD CONSTRAINT "source_record_businessUnitId_fkey" FOREIGN KEY ("businessUnitId") REFERENCES "business_unit"("id") ON DELETE RESTRICT ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "source_record" ADD CONSTRAINT "source_record_companyId_fkey" FOREIGN KEY ("companyId") REFERENCES "canonical_company"("id") ON DELETE SET NULL ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "source_record" ADD CONSTRAINT "source_record_personId_fkey" FOREIGN KEY ("personId") REFERENCES "person"("id") ON DELETE SET NULL ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "source_record" ADD CONSTRAINT "source_record_opportunityId_fkey" FOREIGN KEY ("opportunityId") REFERENCES "canonical_opportunity"("id") ON DELETE SET NULL ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "campaign" ADD CONSTRAINT "campaign_businessUnitId_fkey" FOREIGN KEY ("businessUnitId") REFERENCES "business_unit"("id") ON DELETE RESTRICT ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "campaign_member" ADD CONSTRAINT "campaign_member_campaignId_fkey" FOREIGN KEY ("campaignId") REFERENCES "campaign"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "outreach_approval" ADD CONSTRAINT "outreach_approval_campaignId_fkey" FOREIGN KEY ("campaignId") REFERENCES "campaign"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "agent_run_context" ADD CONSTRAINT "agent_run_context_businessUnitId_fkey" FOREIGN KEY ("businessUnitId") REFERENCES "business_unit"("id") ON DELETE RESTRICT ON UPDATE CASCADE; diff --git a/packages/db/prisma/schema.prisma b/packages/db/prisma/schema.prisma index 82bc1f368..d315ed9ea 100644 --- a/packages/db/prisma/schema.prisma +++ b/packages/db/prisma/schema.prisma @@ -318,6 +318,9 @@ model Company { source RecordSource @default(MANUAL) + businessUnitId String? + businessUnit BusinessUnit? @relation("CompCompanyBusinessUnit", fields: [businessUnitId], references: [id], onDelete: SetNull) + lastActivityAt DateTime? archivedAt DateTime? @@ -487,6 +490,9 @@ model AgentTask { subject String? + businessUnitId String? + businessUnit BusinessUnit? @relation("CompAgentTaskBusinessUnit", fields: [businessUnitId], references: [id], onDelete: SetNull) + createdAt DateTime @default(now()) @@index([dueAt, leasedUntil]) @@ -923,6 +929,9 @@ model Deal { ownerId String owner User @relation("DealOwner", fields: [ownerId], references: [id]) + businessUnitId String? + businessUnit BusinessUnit? @relation("CompDealBusinessUnit", fields: [businessUnitId], references: [id], onDelete: SetNull) + stage DealStage @default(DEMO_BOOKED) stageChangedAt DateTime @default(now()) amount Decimal? @db.Decimal(14, 2) @@ -1583,3 +1592,249 @@ model Apikey { @@index([key]) @@map("apikey") } + +// Sequence Now foundation. These models are isolated from Comp AI's core +// records so canonical data and source provenance remain portable. +model BusinessUnit { + id String @id @default(uuid()) + key String @unique + name String + description String? + enabled Boolean @default(true) + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + compCompanies Company[] @relation("CompCompanyBusinessUnit") + compDeals Deal[] @relation("CompDealBusinessUnit") + compAgentTasks AgentTask[] @relation("CompAgentTaskBusinessUnit") + companies CanonicalCompany[] + people CanonicalPerson[] + opportunities CanonicalOpportunity[] + pipelines Pipeline[] + campaigns Campaign[] + agentRuns AgentRunContext[] + sourceRecords SourceRecord[] + + @@index([enabled, key]) + @@map("business_unit") +} + +model CanonicalCompany { + id String @id @default(uuid()) + businessUnitId String + businessUnit BusinessUnit @relation(fields: [businessUnitId], references: [id], onDelete: Restrict) + name String + normalizedName String + domain String? + phone String? + website String? + fields Json? + verifiedAt DateTime? + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + people CompanyPerson[] + opportunities CanonicalOpportunity[] + sources SourceRecord[] + + @@unique([businessUnitId, normalizedName]) + @@index([domain]) + @@map("canonical_company") +} + +model CanonicalPerson { + id String @id @default(uuid()) + businessUnitId String + businessUnit BusinessUnit @relation(fields: [businessUnitId], references: [id], onDelete: Restrict) + firstName String + lastName String? + normalizedName String + email String? + phone String? + title String? + fields Json? + verifiedAt DateTime? + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + companies CompanyPerson[] + sources SourceRecord[] + + @@index([email]) + @@index([phone]) + @@map("person") +} + +model CompanyPerson { + companyId String + personId String + company CanonicalCompany @relation(fields: [companyId], references: [id], onDelete: Cascade) + person CanonicalPerson @relation(fields: [personId], references: [id], onDelete: Cascade) + role String? + isPrimary Boolean @default(false) + + @@id([companyId, personId]) + @@map("company_person") +} + +model Pipeline { + id String @id @default(uuid()) + businessUnitId String + businessUnit BusinessUnit @relation(fields: [businessUnitId], references: [id], onDelete: Restrict) + name String + stages Json + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + opportunities CanonicalOpportunity[] + + @@unique([businessUnitId, name]) + @@map("pipeline") +} + +model CanonicalOpportunity { + id String @id @default(uuid()) + businessUnitId String + businessUnit BusinessUnit @relation(fields: [businessUnitId], references: [id], onDelete: Restrict) + companyId String? + company CanonicalCompany? @relation(fields: [companyId], references: [id], onDelete: SetNull) + pipelineId String? + pipeline Pipeline? @relation(fields: [pipelineId], references: [id], onDelete: SetNull) + name String + stage String + amount Decimal? @db.Decimal(14, 2) + fields Json? + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + sources SourceRecord[] + + @@index([businessUnitId, stage]) + @@map("canonical_opportunity") +} + +model SourceRecord { + id String @id @default(uuid()) + businessUnitId String + businessUnit BusinessUnit @relation(fields: [businessUnitId], references: [id], onDelete: Restrict) + sourceSystem String + sourceType String + sourceId String + sourceUrl String? + observedAt DateTime? + payload Json? + createdAt DateTime @default(now()) + + companyId String? + company CanonicalCompany? @relation(fields: [companyId], references: [id], onDelete: SetNull) + personId String? + person CanonicalPerson? @relation(fields: [personId], references: [id], onDelete: SetNull) + opportunityId String? + opportunity CanonicalOpportunity? @relation(fields: [opportunityId], references: [id], onDelete: SetNull) + + @@unique([sourceSystem, sourceType, sourceId]) + @@index([businessUnitId, sourceType]) + @@map("source_record") +} + +model RecordMapping { + id String @id @default(uuid()) + sourceSystem String + sourceType String + sourceId String + canonicalType String + canonicalId String + application String? + applicationId String? + matchMethod String + status String @default("active") + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + @@unique([sourceSystem, sourceType, sourceId, canonicalType]) + @@index([canonicalType, canonicalId]) + @@map("record_mapping") +} + +model Campaign { + id String @id @default(uuid()) + businessUnitId String + businessUnit BusinessUnit @relation(fields: [businessUnitId], references: [id], onDelete: Restrict) + name String + status String @default("draft") + segment Json? + template String? + sendingEnabled Boolean @default(false) + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + members CampaignMember[] + approvals OutreachApproval[] + + @@index([businessUnitId, status]) + @@map("campaign") +} + +model CampaignMember { + id String @id @default(uuid()) + campaignId String + campaign Campaign @relation(fields: [campaignId], references: [id], onDelete: Cascade) + companyId String? + personId String? + state String @default("queued") + draft String? + sentAt DateTime? + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + @@unique([campaignId, companyId, personId]) + @@index([state]) + @@map("campaign_member") +} + +model OutreachApproval { + id String @id @default(uuid()) + campaignId String + campaign Campaign @relation(fields: [campaignId], references: [id], onDelete: Cascade) + memberId String? + state String @default("pending") + approvedBy String? + approvedAt DateTime? + rejectionReason String? + immutableHash String + createdAt DateTime @default(now()) + + @@index([campaignId, state]) + @@map("outreach_approval") +} + +model SuppressionEntry { + id String @id @default(uuid()) + value String + normalized String + kind String + reason String? + source String? + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + @@unique([normalized, kind]) + @@map("suppression_entry") +} + +model AgentRunContext { + id String @id @default(uuid()) + businessUnitId String + businessUnit BusinessUnit @relation(fields: [businessUnitId], references: [id], onDelete: Restrict) + agentRunId String + budget Int @default(0) + fitScores Json? + reasons Json? + evidenceCount Int @default(0) + createdAt DateTime @default(now()) + finishedAt DateTime? + + @@unique([agentRunId]) + @@index([businessUnitId, createdAt]) + @@map("agent_run_context") +}