From 1561073c78a4c0e693a91cad2501e522320d1855 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 7 Aug 2026 15:38:18 +0000 Subject: [PATCH 01/33] chore: release release --- .github/.release-please-manifest.json | 2 +- CHANGELOG.md | 7 +++++++ package.json | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/.release-please-manifest.json b/.github/.release-please-manifest.json index cddbaefb9..c95106184 100644 --- a/.github/.release-please-manifest.json +++ b/.github/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "1.1.0" + ".": "1.2.0" } diff --git a/CHANGELOG.md b/CHANGELOG.md index 0fa2ed919..e2dea4a35 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [1.2.0](https://github.com/trycompai/crm/compare/v1.1.0...v1.2.0) (2026-08-07) + + +### Features + +* **api:** add microsoft sign-in and outlook mailbox sync ([#73](https://github.com/trycompai/crm/issues/73)) ([2a0062f](https://github.com/trycompai/crm/commit/2a0062fb76ffdaa5bbbb3848a5573b8b53cd0036)) + ## [1.1.0](https://github.com/trycompai/crm/compare/v1.0.0...v1.1.0) (2026-08-06) diff --git a/package.json b/package.json index 58f50e8b1..bd61fd43b 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "crm", "private": true, "license": "MIT", - "version": "1.1.0", + "version": "1.2.0", "scripts": { "prepare": "git rev-parse --git-dir >/dev/null 2>&1 && git config core.hooksPath .githooks || true", "build": "turbo run build", From efd44f04d4ce8066d7d1355be30141aa4d310e28 Mon Sep 17 00:00:00 2001 From: lumenstech <124342884+lumenstech@users.noreply.github.com> Date: Thu, 3 Sep 2026 21:09:52 -0400 Subject: [PATCH 02/33] feat(api): add signal ingest contracts --- apps/api/src/ingest/ingest.contracts.ts | 27 +++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 apps/api/src/ingest/ingest.contracts.ts diff --git a/apps/api/src/ingest/ingest.contracts.ts b/apps/api/src/ingest/ingest.contracts.ts new file mode 100644 index 000000000..d1dc31397 --- /dev/null +++ b/apps/api/src/ingest/ingest.contracts.ts @@ -0,0 +1,27 @@ +import { z } from "zod"; + +const signalPayload = z.record(z.string(), z.unknown()); + +export const ingestSignalInput = z.object({ + project: z.string().trim().min(1).max(96), + source: z.string().trim().min(1).max(96), + sourceType: z.string().trim().min(1).max(160), + sourceId: z.string().trim().min(1).max(320), + sourceUrl: z.string().url().nullable().optional(), + observedAt: z.string().datetime({ offset: true }).nullable().optional(), + entity: z.string().trim().min(1).max(320).nullable().optional(), + signalScore: z.number().min(0).max(100).nullable().optional(), + tags: z.array(z.string().trim().min(1).max(80)).max(50).default([]), + payload: signalPayload.default({}), +}); + +export const ingestSignalOutput = z.object({ + status: z.literal("accepted"), + sourceRecordId: z.string(), + project: z.string(), + deduplicated: z.boolean(), + promoted: z.literal(false), +}); + +export type IngestSignalInput = z.infer; +export type IngestSignalOutput = z.infer; From 3616bbaf01d0a088aef4106e1517025a3718f733 Mon Sep 17 00:00:00 2001 From: lumenstech <124342884+lumenstech@users.noreply.github.com> Date: Thu, 3 Sep 2026 21:14:58 -0400 Subject: [PATCH 03/33] feat(api): add idempotent signal ingest service --- apps/api/src/ingest/ingest.service.ts | 119 ++++++++++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 apps/api/src/ingest/ingest.service.ts diff --git a/apps/api/src/ingest/ingest.service.ts b/apps/api/src/ingest/ingest.service.ts new file mode 100644 index 000000000..2aa233f01 --- /dev/null +++ b/apps/api/src/ingest/ingest.service.ts @@ -0,0 +1,119 @@ +import type { Db } from "@crm/db"; +import { BadRequestException, Injectable, Logger } from "@nestjs/common"; +import { randomUUID } from "node:crypto"; +import { InjectDatabase } from "../database/database.constants"; +import type { IngestSignalInput, IngestSignalOutput } from "./ingest.contracts"; + +type BusinessUnitRow = { + id: string; + enabled: boolean; +}; + +type SourceRecordRow = { + id: string; +}; + +@Injectable() +export class IngestService { + private readonly logger = new Logger(IngestService.name); + + constructor(@InjectDatabase() private readonly db: Db) {} + + async signal(input: IngestSignalInput): Promise { + const [businessUnit] = await this.db.$queryRaw` + SELECT id, enabled + FROM business_unit + WHERE key = ${input.project} + LIMIT 1 + `; + + if (!businessUnit) { + throw new BadRequestException( + `Unknown business unit: ${input.project}. Register it before ingesting signals.`, + ); + } + + if (!businessUnit.enabled) { + throw new BadRequestException( + `Business unit is disabled: ${input.project}.`, + ); + } + + const [existing] = await this.db.$queryRaw` + SELECT id + FROM source_record + WHERE "sourceSystem" = ${input.source} + AND "sourceType" = ${input.sourceType} + AND "sourceId" = ${input.sourceId} + LIMIT 1 + `; + + const observedAt = input.observedAt ? new Date(input.observedAt) : new Date(); + const payload = JSON.stringify({ + ...input.payload, + project: input.project, + source: input.source, + source_type: input.sourceType, + source_id: input.sourceId, + entity: input.entity ?? null, + signal_score: input.signalScore ?? null, + tags: input.tags, + ingested_at: new Date().toISOString(), + }); + const sourceRecordId = existing?.id ?? randomUUID(); + + const [saved] = await this.db.$queryRaw` + INSERT INTO source_record ( + id, + "businessUnitId", + "sourceSystem", + "sourceType", + "sourceId", + "sourceUrl", + "observedAt", + payload, + "createdAt" + ) + VALUES ( + ${sourceRecordId}, + ${businessUnit.id}, + ${input.source}, + ${input.sourceType}, + ${input.sourceId}, + ${input.sourceUrl ?? null}, + ${observedAt}, + ${payload}::jsonb, + CURRENT_TIMESTAMP + ) + ON CONFLICT ("sourceSystem", "sourceType", "sourceId") + DO UPDATE SET + "businessUnitId" = EXCLUDED."businessUnitId", + "sourceUrl" = EXCLUDED."sourceUrl", + "observedAt" = EXCLUDED."observedAt", + payload = EXCLUDED.payload + RETURNING id + `; + + if (!saved) { + throw new Error("Signal ingest did not return a source record."); + } + + this.logger.log({ + message: "Signal ingested", + project: input.project, + source: input.source, + sourceType: input.sourceType, + sourceId: input.sourceId, + sourceRecordId: saved.id, + deduplicated: Boolean(existing), + }); + + return { + status: "accepted", + sourceRecordId: saved.id, + project: input.project, + deduplicated: Boolean(existing), + promoted: false, + }; + } +} From fd809273ef0dcfa021530d48acdf29ab164d1500 Mon Sep 17 00:00:00 2001 From: lumenstech <124342884+lumenstech@users.noreply.github.com> Date: Thu, 3 Sep 2026 21:15:06 -0400 Subject: [PATCH 04/33] feat(api): expose central signal ingest endpoint --- apps/api/src/ingest/ingest.router.ts | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 apps/api/src/ingest/ingest.router.ts diff --git a/apps/api/src/ingest/ingest.router.ts b/apps/api/src/ingest/ingest.router.ts new file mode 100644 index 000000000..c5b56d6db --- /dev/null +++ b/apps/api/src/ingest/ingest.router.ts @@ -0,0 +1,25 @@ +import { Inject } from "@nestjs/common"; +import { Input, Mutation, Router, UseMiddlewares } from "nestjs-trpc"; +import type { z } from "zod"; +import { AuthMiddleware } from "../trpc/middlewares/auth.middleware"; +import { restMeta } from "../trpc/openapi"; +import { + ingestSignalInput, + ingestSignalOutput, +} from "./ingest.contracts"; +import { IngestService } from "./ingest.service"; + +@Router({ alias: "ingest" }) +@UseMiddlewares(AuthMiddleware) +export class IngestRouter { + constructor(@Inject(IngestService) private readonly ingest: IngestService) {} + + @Mutation({ + input: ingestSignalInput, + output: ingestSignalOutput, + meta: restMeta("POST", "/ingest/signal", ["Ingest"]), + }) + async signal(@Input() input: z.infer) { + return this.ingest.signal(input); + } +} From fc1fa7cf8b476b9c51514a4ffbfe1d05bb5c3c42 Mon Sep 17 00:00:00 2001 From: lumenstech <124342884+lumenstech@users.noreply.github.com> Date: Thu, 3 Sep 2026 21:15:11 -0400 Subject: [PATCH 05/33] feat(api): register ingest module --- apps/api/src/ingest/ingest.module.ts | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 apps/api/src/ingest/ingest.module.ts diff --git a/apps/api/src/ingest/ingest.module.ts b/apps/api/src/ingest/ingest.module.ts new file mode 100644 index 000000000..c9061f1c4 --- /dev/null +++ b/apps/api/src/ingest/ingest.module.ts @@ -0,0 +1,9 @@ +import { Module } from "@nestjs/common"; +import { IngestRouter } from "./ingest.router"; +import { IngestService } from "./ingest.service"; + +@Module({ + providers: [IngestService, IngestRouter], + exports: [IngestService], +}) +export class IngestModule {} From defcff1db29462c0aeef909455855bb0576b83d2 Mon Sep 17 00:00:00 2001 From: lumenstech <124342884+lumenstech@users.noreply.github.com> Date: Thu, 3 Sep 2026 21:15:23 -0400 Subject: [PATCH 06/33] feat(api): wire central signal ingest module --- apps/api/src/app.module.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apps/api/src/app.module.ts b/apps/api/src/app.module.ts index 82a345df2..872a972cf 100644 --- a/apps/api/src/app.module.ts +++ b/apps/api/src/app.module.ts @@ -22,6 +22,7 @@ import { EnrichmentModule } from "./enrichment/enrichment.module"; import { FieldsModule } from "./fields/fields.module"; import { GoogleModule } from "./google/google.module"; import { HealthModule } from "./health/health.module"; +import { IngestModule } from "./ingest/ingest.module"; import { LoggingModule } from "./logging/logging.module"; import { logAuthRoute } from "./logging/request-logger.middleware"; import { MailboxModule } from "./mailbox/mailbox.module"; @@ -75,6 +76,7 @@ import { WorkspaceModule } from "./workspace/workspace.module"; SsoModule, SlackModule, BackfillModule, + IngestModule, TelemetryModule, TrackingModule, ArchiveModule, From 929fb2713bf51bf19d18faa5bc41d6339d343a41 Mon Sep 17 00:00:00 2001 From: lumenstech <124342884+lumenstech@users.noreply.github.com> Date: Thu, 3 Sep 2026 21:20:48 -0400 Subject: [PATCH 07/33] feat(api): add signal inbox contracts --- apps/api/src/ingest/ingest.contracts.ts | 34 +++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/apps/api/src/ingest/ingest.contracts.ts b/apps/api/src/ingest/ingest.contracts.ts index d1dc31397..6c839531c 100644 --- a/apps/api/src/ingest/ingest.contracts.ts +++ b/apps/api/src/ingest/ingest.contracts.ts @@ -23,5 +23,39 @@ export const ingestSignalOutput = z.object({ promoted: z.literal(false), }); +export const signalInboxInput = z.object({ + project: z.string().trim().min(1).max(96).optional(), + minScore: z.number().min(0).max(100).optional(), + status: z.enum(["all", "unresolved", "mapped"]).default("unresolved"), + limit: z.number().int().min(1).max(200).default(50), +}); + +export const signalInboxItem = z.object({ + sourceRecordId: z.string(), + project: z.string(), + source: z.string(), + sourceType: z.string(), + sourceId: z.string(), + sourceUrl: z.string().nullable(), + observedAt: z.string(), + entity: z.string().nullable(), + signalScore: z.number().nullable(), + company: z.string().nullable(), + subject: z.string().nullable(), + stageKey: z.string().nullable(), + priority: z.string().nullable(), + demandTrigger: z.string().nullable(), + nextAction: z.string().nullable(), + mapped: z.boolean(), + payload: signalPayload, +}); + +export const signalInboxOutput = z.object({ + rows: z.array(signalInboxItem), + count: z.number().int().nonnegative(), +}); + export type IngestSignalInput = z.infer; export type IngestSignalOutput = z.infer; +export type SignalInboxInput = z.infer; +export type SignalInboxOutput = z.infer; From b13ebaed0310c5c72951f762cdd40e4266402ab8 Mon Sep 17 00:00:00 2001 From: lumenstech <124342884+lumenstech@users.noreply.github.com> Date: Thu, 3 Sep 2026 21:21:05 -0400 Subject: [PATCH 08/33] feat(api): add signal inbox query service --- apps/api/src/ingest/ingest.service.ts | 73 ++++++++++++++++++++++++++- 1 file changed, 72 insertions(+), 1 deletion(-) diff --git a/apps/api/src/ingest/ingest.service.ts b/apps/api/src/ingest/ingest.service.ts index 2aa233f01..b79ce6171 100644 --- a/apps/api/src/ingest/ingest.service.ts +++ b/apps/api/src/ingest/ingest.service.ts @@ -2,7 +2,12 @@ import type { Db } from "@crm/db"; import { BadRequestException, Injectable, Logger } from "@nestjs/common"; import { randomUUID } from "node:crypto"; import { InjectDatabase } from "../database/database.constants"; -import type { IngestSignalInput, IngestSignalOutput } from "./ingest.contracts"; +import type { + IngestSignalInput, + IngestSignalOutput, + SignalInboxInput, + SignalInboxOutput, +} from "./ingest.contracts"; type BusinessUnitRow = { id: string; @@ -13,6 +18,26 @@ type SourceRecordRow = { id: string; }; +type InboxRow = { + sourceRecordId: string; + project: string; + source: string; + sourceType: string; + sourceId: string; + sourceUrl: string | null; + observedAt: Date; + entity: string | null; + signalScore: number | null; + company: string | null; + subject: string | null; + stageKey: string | null; + priority: string | null; + demandTrigger: string | null; + nextAction: string | null; + mapped: boolean; + payload: Record; +}; + @Injectable() export class IngestService { private readonly logger = new Logger(IngestService.name); @@ -116,4 +141,50 @@ export class IngestService { promoted: false, }; } + + async inbox(input: SignalInboxInput): Promise { + const rows = await this.db.$queryRaw` + SELECT + sr.id AS "sourceRecordId", + bu.key AS project, + sr."sourceSystem" AS source, + sr."sourceType" AS "sourceType", + sr."sourceId" AS "sourceId", + sr."sourceUrl" AS "sourceUrl", + sr."observedAt" AS "observedAt", + COALESCE(sr.payload->>'entity', sr.payload->>'company') AS entity, + NULLIF(sr.payload->>'signal_score','')::float AS "signalScore", + sr.payload->>'company' AS company, + sr.payload->>'subject' AS subject, + sr.payload->>'stage_key' AS "stageKey", + sr.payload->>'priority' AS priority, + sr.payload->>'demand_trigger' AS "demandTrigger", + sr.payload->>'next_action' AS "nextAction", + EXISTS ( + SELECT 1 FROM record_mapping rm WHERE rm."sourceRecordId" = sr.id + ) AS mapped, + sr.payload AS payload + FROM source_record sr + JOIN business_unit bu ON bu.id = sr."businessUnitId" + WHERE (${input.project ?? null}::text IS NULL OR bu.key = ${input.project ?? null}) + AND (${input.minScore ?? null}::float IS NULL OR COALESCE(NULLIF(sr.payload->>'signal_score','')::float, NULLIF(sr.payload->'metadata'->>'fit_score','')::float, 0) >= ${input.minScore ?? null}) + AND ( + ${input.status} = 'all' + OR (${input.status} = 'mapped' AND EXISTS (SELECT 1 FROM record_mapping rm WHERE rm."sourceRecordId" = sr.id)) + OR (${input.status} = 'unresolved' AND NOT EXISTS (SELECT 1 FROM record_mapping rm WHERE rm."sourceRecordId" = sr.id)) + ) + ORDER BY + COALESCE(NULLIF(sr.payload->>'signal_score','')::float, NULLIF(sr.payload->'metadata'->>'fit_score','')::float, 0) DESC, + sr."observedAt" DESC + LIMIT ${input.limit} + `; + + return { + rows: rows.map((row) => ({ + ...row, + observedAt: row.observedAt.toISOString(), + })), + count: rows.length, + }; + } } From 07e35ad3026559c174b55f317f346a9f0591c08d Mon Sep 17 00:00:00 2001 From: lumenstech <124342884+lumenstech@users.noreply.github.com> Date: Thu, 3 Sep 2026 21:21:12 -0400 Subject: [PATCH 09/33] feat(api): expose signal inbox endpoint --- apps/api/src/ingest/ingest.router.ts | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/apps/api/src/ingest/ingest.router.ts b/apps/api/src/ingest/ingest.router.ts index c5b56d6db..48170f5b3 100644 --- a/apps/api/src/ingest/ingest.router.ts +++ b/apps/api/src/ingest/ingest.router.ts @@ -1,11 +1,13 @@ import { Inject } from "@nestjs/common"; -import { Input, Mutation, Router, UseMiddlewares } from "nestjs-trpc"; +import { Input, Mutation, Query, Router, UseMiddlewares } from "nestjs-trpc"; import type { z } from "zod"; import { AuthMiddleware } from "../trpc/middlewares/auth.middleware"; import { restMeta } from "../trpc/openapi"; import { ingestSignalInput, ingestSignalOutput, + signalInboxInput, + signalInboxOutput, } from "./ingest.contracts"; import { IngestService } from "./ingest.service"; @@ -22,4 +24,13 @@ export class IngestRouter { async signal(@Input() input: z.infer) { return this.ingest.signal(input); } + + @Query({ + input: signalInboxInput, + output: signalInboxOutput, + meta: restMeta("GET", "/ingest/signals", ["Ingest"]), + }) + async inbox(@Input() input: z.infer) { + return this.ingest.inbox(input); + } } From 5f6181cdfa2f19370319aff69c1ff904527f1b46 Mon Sep 17 00:00:00 2001 From: lumenstech <124342884+lumenstech@users.noreply.github.com> Date: Thu, 3 Sep 2026 21:24:27 -0400 Subject: [PATCH 10/33] feat(api): add signal company resolution contracts --- apps/api/src/ingest/ingest.contracts.ts | 49 +++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/apps/api/src/ingest/ingest.contracts.ts b/apps/api/src/ingest/ingest.contracts.ts index 6c839531c..c1df7c2eb 100644 --- a/apps/api/src/ingest/ingest.contracts.ts +++ b/apps/api/src/ingest/ingest.contracts.ts @@ -55,7 +55,56 @@ export const signalInboxOutput = z.object({ count: z.number().int().nonnegative(), }); +export const signalSourceRecordInput = z.object({ + sourceRecordId: z.string().trim().min(1), +}); + +export const companyCandidateOutput = z.object({ + companyId: z.string(), + name: z.string(), + domain: z.string().nullable(), + businessUnitId: z.string().nullable(), + score: z.number().min(0).max(110), + reasons: z.array(z.string()), +}); + +export const signalCompanyCandidatesOutput = z.object({ + sourceRecordId: z.string(), + project: z.string(), + entity: z.string().nullable(), + domain: z.string().nullable(), + mappedCompanyId: z.string().nullable(), + candidates: z.array(companyCandidateOutput), +}); + +export const resolveSignalCompanyInput = signalSourceRecordInput.extend({ + companyId: z.string().trim().min(1).nullable().optional(), + companyName: z.string().trim().min(1).max(320).nullable().optional(), + domain: z.string().trim().max(320).nullable().optional(), + createIfMissing: z.boolean().default(false), + queueResearch: z.boolean().default(true), +}); + +export const resolveSignalCompanyOutput = z.object({ + sourceRecordId: z.string(), + canonicalCompanyId: z.string(), + companyId: z.string(), + companyName: z.string(), + matchMethod: z.string(), + created: z.boolean(), + researchQueued: z.boolean(), +}); + export type IngestSignalInput = z.infer; export type IngestSignalOutput = z.infer; export type SignalInboxInput = z.infer; export type SignalInboxOutput = z.infer; +export type SignalCompanyCandidatesOutput = z.infer< + typeof signalCompanyCandidatesOutput +>; +export type ResolveSignalCompanyInput = z.infer< + typeof resolveSignalCompanyInput +>; +export type ResolveSignalCompanyOutput = z.infer< + typeof resolveSignalCompanyOutput +>; From d04ce539366ee0723433af23a53fdc47c44b75ed Mon Sep 17 00:00:00 2001 From: lumenstech <124342884+lumenstech@users.noreply.github.com> Date: Thu, 3 Sep 2026 21:25:19 -0400 Subject: [PATCH 11/33] feat(api): resolve signal companies and fix mapping lookup --- apps/api/src/ingest/ingest.service.ts | 372 ++++++++++++++++++++++---- 1 file changed, 313 insertions(+), 59 deletions(-) diff --git a/apps/api/src/ingest/ingest.service.ts b/apps/api/src/ingest/ingest.service.ts index b79ce6171..75572785b 100644 --- a/apps/api/src/ingest/ingest.service.ts +++ b/apps/api/src/ingest/ingest.service.ts @@ -1,23 +1,41 @@ import type { Db } from "@crm/db"; -import { BadRequestException, Injectable, Logger } from "@nestjs/common"; +import { + BadRequestException, + Injectable, + Logger, + NotFoundException, +} from "@nestjs/common"; import { randomUUID } from "node:crypto"; +import { CompaniesService } from "../companies/companies.service"; +import { normalizeDomain } from "../companies/domain"; import { InjectDatabase } from "../database/database.constants"; import type { IngestSignalInput, IngestSignalOutput, + ResolveSignalCompanyInput, + ResolveSignalCompanyOutput, + SignalCompanyCandidatesOutput, SignalInboxInput, SignalInboxOutput, } from "./ingest.contracts"; -type BusinessUnitRow = { +type BusinessUnitRow = { id: string; key?: string; enabled: boolean }; +type SourceRecordRow = { id: string }; +type SignalRow = { id: string; - enabled: boolean; + businessUnitId: string; + project: string; + sourceSystem: string; + sourceType: string; + sourceId: string; + payload: Record; }; - -type SourceRecordRow = { +type CandidateRow = { id: string; + name: string; + domain: string | null; + businessUnitId: string | null; }; - type InboxRow = { sourceRecordId: string; project: string; @@ -42,31 +60,26 @@ type InboxRow = { export class IngestService { private readonly logger = new Logger(IngestService.name); - constructor(@InjectDatabase() private readonly db: Db) {} + constructor( + @InjectDatabase() private readonly db: Db, + private readonly companies: CompaniesService, + ) {} async signal(input: IngestSignalInput): Promise { const [businessUnit] = await this.db.$queryRaw` - SELECT id, enabled - FROM business_unit - WHERE key = ${input.project} - LIMIT 1 + SELECT id, enabled FROM business_unit WHERE key = ${input.project} LIMIT 1 `; - if (!businessUnit) { throw new BadRequestException( `Unknown business unit: ${input.project}. Register it before ingesting signals.`, ); } - if (!businessUnit.enabled) { - throw new BadRequestException( - `Business unit is disabled: ${input.project}.`, - ); + throw new BadRequestException(`Business unit is disabled: ${input.project}.`); } const [existing] = await this.db.$queryRaw` - SELECT id - FROM source_record + SELECT id FROM source_record WHERE "sourceSystem" = ${input.source} AND "sourceType" = ${input.sourceType} AND "sourceId" = ${input.sourceId} @@ -89,25 +102,12 @@ export class IngestService { const [saved] = await this.db.$queryRaw` INSERT INTO source_record ( - id, - "businessUnitId", - "sourceSystem", - "sourceType", - "sourceId", - "sourceUrl", - "observedAt", - payload, - "createdAt" + id, "businessUnitId", "sourceSystem", "sourceType", "sourceId", + "sourceUrl", "observedAt", payload, "createdAt" ) VALUES ( - ${sourceRecordId}, - ${businessUnit.id}, - ${input.source}, - ${input.sourceType}, - ${input.sourceId}, - ${input.sourceUrl ?? null}, - ${observedAt}, - ${payload}::jsonb, + ${sourceRecordId}, ${businessUnit.id}, ${input.source}, ${input.sourceType}, + ${input.sourceId}, ${input.sourceUrl ?? null}, ${observedAt}, ${payload}::jsonb, CURRENT_TIMESTAMP ) ON CONFLICT ("sourceSystem", "sourceType", "sourceId") @@ -118,10 +118,7 @@ export class IngestService { payload = EXCLUDED.payload RETURNING id `; - - if (!saved) { - throw new Error("Signal ingest did not return a source record."); - } + if (!saved) throw new Error("Signal ingest did not return a source record."); this.logger.log({ message: "Signal ingested", @@ -132,7 +129,6 @@ export class IngestService { sourceRecordId: saved.id, deduplicated: Boolean(existing), }); - return { status: "accepted", sourceRecordId: saved.id, @@ -145,15 +141,13 @@ export class IngestService { async inbox(input: SignalInboxInput): Promise { const rows = await this.db.$queryRaw` SELECT - sr.id AS "sourceRecordId", - bu.key AS project, - sr."sourceSystem" AS source, - sr."sourceType" AS "sourceType", - sr."sourceId" AS "sourceId", - sr."sourceUrl" AS "sourceUrl", + sr.id AS "sourceRecordId", bu.key AS project, + sr."sourceSystem" AS source, sr."sourceType" AS "sourceType", + sr."sourceId" AS "sourceId", sr."sourceUrl" AS "sourceUrl", sr."observedAt" AS "observedAt", COALESCE(sr.payload->>'entity', sr.payload->>'company') AS entity, - NULLIF(sr.payload->>'signal_score','')::float AS "signalScore", + COALESCE(NULLIF(sr.payload->>'signal_score','')::float, + NULLIF(sr.payload->'metadata'->>'fit_score','')::float) AS "signalScore", sr.payload->>'company' AS company, sr.payload->>'subject' AS subject, sr.payload->>'stage_key' AS "stageKey", @@ -161,30 +155,290 @@ export class IngestService { sr.payload->>'demand_trigger' AS "demandTrigger", sr.payload->>'next_action' AS "nextAction", EXISTS ( - SELECT 1 FROM record_mapping rm WHERE rm."sourceRecordId" = sr.id + SELECT 1 FROM record_mapping rm + WHERE rm."sourceSystem" = sr."sourceSystem" + AND rm."sourceType" = sr."sourceType" + AND rm."sourceId" = sr."sourceId" + AND rm."canonicalType" = 'company' + AND rm.status = 'active' ) AS mapped, sr.payload AS payload FROM source_record sr JOIN business_unit bu ON bu.id = sr."businessUnitId" WHERE (${input.project ?? null}::text IS NULL OR bu.key = ${input.project ?? null}) - AND (${input.minScore ?? null}::float IS NULL OR COALESCE(NULLIF(sr.payload->>'signal_score','')::float, NULLIF(sr.payload->'metadata'->>'fit_score','')::float, 0) >= ${input.minScore ?? null}) + AND (${input.minScore ?? null}::float IS NULL OR COALESCE( + NULLIF(sr.payload->>'signal_score','')::float, + NULLIF(sr.payload->'metadata'->>'fit_score','')::float, 0 + ) >= ${input.minScore ?? null}) AND ( ${input.status} = 'all' - OR (${input.status} = 'mapped' AND EXISTS (SELECT 1 FROM record_mapping rm WHERE rm."sourceRecordId" = sr.id)) - OR (${input.status} = 'unresolved' AND NOT EXISTS (SELECT 1 FROM record_mapping rm WHERE rm."sourceRecordId" = sr.id)) + OR (${input.status} = 'mapped' AND EXISTS ( + SELECT 1 FROM record_mapping rm + WHERE rm."sourceSystem" = sr."sourceSystem" + AND rm."sourceType" = sr."sourceType" + AND rm."sourceId" = sr."sourceId" + AND rm."canonicalType" = 'company' AND rm.status = 'active' + )) + OR (${input.status} = 'unresolved' AND NOT EXISTS ( + SELECT 1 FROM record_mapping rm + WHERE rm."sourceSystem" = sr."sourceSystem" + AND rm."sourceType" = sr."sourceType" + AND rm."sourceId" = sr."sourceId" + AND rm."canonicalType" = 'company' AND rm.status = 'active' + )) ) - ORDER BY - COALESCE(NULLIF(sr.payload->>'signal_score','')::float, NULLIF(sr.payload->'metadata'->>'fit_score','')::float, 0) DESC, - sr."observedAt" DESC + ORDER BY COALESCE( + NULLIF(sr.payload->>'signal_score','')::float, + NULLIF(sr.payload->'metadata'->>'fit_score','')::float, 0 + ) DESC, sr."observedAt" DESC LIMIT ${input.limit} `; - return { - rows: rows.map((row) => ({ - ...row, - observedAt: row.observedAt.toISOString(), - })), + rows: rows.map((row) => ({ ...row, observedAt: row.observedAt.toISOString() })), count: rows.length, }; } + + async companyCandidates(sourceRecordId: string): Promise { + const signal = await this.loadSignal(sourceRecordId); + const entity = this.signalEntity(signal.payload); + const domain = this.signalDomain(signal.payload); + + const [mapped] = await this.db.$queryRaw>` + SELECT "applicationId" FROM record_mapping + WHERE "sourceSystem" = ${signal.sourceSystem} + AND "sourceType" = ${signal.sourceType} + AND "sourceId" = ${signal.sourceId} + AND "canonicalType" = 'company' AND status = 'active' + LIMIT 1 + `; + + const rows = entity || domain + ? await this.db.company.findMany({ + where: { + archivedAt: null, + OR: [ + ...(entity + ? [ + { name: { equals: entity, mode: "insensitive" as const } }, + { name: { contains: entity, mode: "insensitive" as const } }, + ] + : []), + ...(domain ? [{ domain }] : []), + ], + }, + select: { id: true, name: true, domain: true, businessUnitId: true }, + take: 20, + }) + : []; + + const candidates = (rows as CandidateRow[]) + .map((row) => this.scoreCandidate(row, entity, domain, signal.businessUnitId)) + .sort((a, b) => b.score - a.score || a.name.localeCompare(b.name)); + + return { + sourceRecordId, + project: signal.project, + entity, + domain, + mappedCompanyId: mapped?.applicationId ?? null, + candidates, + }; + } + + async resolveCompany(input: ResolveSignalCompanyInput): Promise { + const signal = await this.loadSignal(input.sourceRecordId); + const entity = input.companyName ?? this.signalEntity(signal.payload); + const requestedDomain = normalizeDomain(input.domain ?? this.signalDomain(signal.payload)); + let matchMethod = "manual"; + let created = false; + + let company = input.companyId + ? await this.db.company.findFirst({ + where: { id: input.companyId, archivedAt: null }, + select: { id: true, name: true, domain: true, businessUnitId: true }, + }) + : null; + + if (!company && requestedDomain) { + company = await this.db.company.findFirst({ + where: { domain: requestedDomain, archivedAt: null }, + select: { id: true, name: true, domain: true, businessUnitId: true }, + }); + if (company) matchMethod = "domain"; + } + if (!company && entity) { + company = await this.db.company.findFirst({ + where: { name: { equals: entity, mode: "insensitive" }, archivedAt: null }, + select: { id: true, name: true, domain: true, businessUnitId: true }, + }); + if (company) matchMethod = "name-exact"; + } + + if (!company && !input.createIfMissing) { + throw new BadRequestException( + "No exact company match. Review /ingest/signals/{sourceRecordId}/company-candidates or set createIfMissing=true after verification.", + ); + } + if (!company) { + if (!entity) throw new BadRequestException("A company name is required to create a company."); + const made = await this.companies.create({ + name: entity, + domain: requestedDomain ?? undefined, + }); + await this.db.company.update({ + where: { id: made.id }, + data: { businessUnitId: signal.businessUnitId }, + }); + company = { + id: made.id, + name: made.name, + domain: made.domain, + businessUnitId: signal.businessUnitId, + }; + matchMethod = "created"; + created = true; + } + + const normalizedName = company.name.trim().toLowerCase(); + const canonicalId = randomUUID(); + const [canonical] = await this.db.$queryRaw>` + INSERT INTO canonical_company ( + id, "businessUnitId", name, "normalizedName", domain, website, + fields, "createdAt", "updatedAt" + ) + VALUES ( + ${canonicalId}, ${signal.businessUnitId}, ${company.name}, ${normalizedName}, + ${company.domain}, ${company.domain ? `https://${company.domain}` : null}, + ${JSON.stringify({ source_signal_id: signal.id })}::jsonb, + CURRENT_TIMESTAMP, CURRENT_TIMESTAMP + ) + ON CONFLICT ("businessUnitId", "normalizedName") DO UPDATE SET + domain = COALESCE(canonical_company.domain, EXCLUDED.domain), + website = COALESCE(canonical_company.website, EXCLUDED.website), + "updatedAt" = CURRENT_TIMESTAMP + RETURNING id + `; + if (!canonical) throw new Error("Canonical company resolution failed."); + + await this.db.$queryRaw` + INSERT INTO record_mapping ( + id, "sourceSystem", "sourceType", "sourceId", "canonicalType", + "canonicalId", application, "applicationId", "matchMethod", status, + "createdAt", "updatedAt" + ) + VALUES ( + ${randomUUID()}, ${signal.sourceSystem}, ${signal.sourceType}, ${signal.sourceId}, + 'company', ${canonical.id}, 'comp-ai-core', ${company.id}, ${matchMethod}, + 'active', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP + ) + ON CONFLICT ("sourceSystem", "sourceType", "sourceId", "canonicalType") + DO UPDATE SET + "canonicalId" = EXCLUDED."canonicalId", + application = EXCLUDED.application, + "applicationId" = EXCLUDED."applicationId", + "matchMethod" = EXCLUDED."matchMethod", + status = 'active', + "updatedAt" = CURRENT_TIMESTAMP + `; + + let researchQueued = false; + if (input.queueResearch) { + const queued = await this.companies.enrich(company.id); + researchQueued = queued.queued; + } + + this.logger.log({ + message: "Signal company resolved", + sourceRecordId: input.sourceRecordId, + companyId: company.id, + canonicalCompanyId: canonical.id, + matchMethod, + created, + researchQueued, + }); + + return { + sourceRecordId: input.sourceRecordId, + canonicalCompanyId: canonical.id, + companyId: company.id, + companyName: company.name, + matchMethod, + created, + researchQueued, + }; + } + + private async loadSignal(sourceRecordId: string): Promise { + const [row] = await this.db.$queryRaw` + SELECT sr.id, sr."businessUnitId" AS "businessUnitId", bu.key AS project, + sr."sourceSystem" AS "sourceSystem", sr."sourceType" AS "sourceType", + sr."sourceId" AS "sourceId", sr.payload + FROM source_record sr + JOIN business_unit bu ON bu.id = sr."businessUnitId" + WHERE sr.id = ${sourceRecordId} + LIMIT 1 + `; + if (!row) throw new NotFoundException(`No source signal with id ${sourceRecordId}.`); + return row; + } + + private signalEntity(payload: Record): string | null { + for (const key of ["entity", "company"] as const) { + const value = payload[key]; + if (typeof value === "string" && value.trim()) return value.trim(); + } + return null; + } + + private signalDomain(payload: Record): string | null { + for (const key of ["domain", "website"] as const) { + const value = payload[key]; + if (typeof value === "string") { + const domain = normalizeDomain(value); + if (domain) return domain; + } + } + return null; + } + + private scoreCandidate( + row: CandidateRow, + entity: string | null, + domain: string | null, + businessUnitId: string, + ) { + let score = 0; + const reasons: string[] = []; + if (domain && row.domain === domain) { + score += 100; + reasons.push("exact-domain"); + } + if (entity) { + const wanted = entity.toLowerCase(); + const actual = row.name.toLowerCase(); + if (actual === wanted) { + score += 95; + reasons.push("exact-name"); + } else if (actual.startsWith(wanted) || wanted.startsWith(actual)) { + score += 80; + reasons.push("name-prefix"); + } else if (actual.includes(wanted) || wanted.includes(actual)) { + score += 70; + reasons.push("name-contains"); + } + } + if (row.businessUnitId === businessUnitId) { + score += 10; + reasons.push("same-business-unit"); + } + return { + companyId: row.id, + name: row.name, + domain: row.domain, + businessUnitId: row.businessUnitId, + score: Math.min(110, score), + reasons, + }; + } } From 617f71184df07cd69dc44242f3c45c5c406303b5 Mon Sep 17 00:00:00 2001 From: lumenstech <124342884+lumenstech@users.noreply.github.com> Date: Thu, 3 Sep 2026 21:25:24 -0400 Subject: [PATCH 12/33] feat(api): connect ingest resolver to companies service --- apps/api/src/ingest/ingest.module.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apps/api/src/ingest/ingest.module.ts b/apps/api/src/ingest/ingest.module.ts index c9061f1c4..3b0f51aee 100644 --- a/apps/api/src/ingest/ingest.module.ts +++ b/apps/api/src/ingest/ingest.module.ts @@ -1,8 +1,10 @@ import { Module } from "@nestjs/common"; +import { CompaniesModule } from "../companies/companies.module"; import { IngestRouter } from "./ingest.router"; import { IngestService } from "./ingest.service"; @Module({ + imports: [CompaniesModule], providers: [IngestService, IngestRouter], exports: [IngestService], }) From c712b8207d94a1d3afc024839049d2e7414de62b Mon Sep 17 00:00:00 2001 From: lumenstech <124342884+lumenstech@users.noreply.github.com> Date: Thu, 3 Sep 2026 21:25:30 -0400 Subject: [PATCH 13/33] feat(api): expose signal company resolution endpoints --- apps/api/src/ingest/ingest.router.ts | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/apps/api/src/ingest/ingest.router.ts b/apps/api/src/ingest/ingest.router.ts index 48170f5b3..0b3759cb4 100644 --- a/apps/api/src/ingest/ingest.router.ts +++ b/apps/api/src/ingest/ingest.router.ts @@ -6,8 +6,12 @@ import { restMeta } from "../trpc/openapi"; import { ingestSignalInput, ingestSignalOutput, + resolveSignalCompanyInput, + resolveSignalCompanyOutput, + signalCompanyCandidatesOutput, signalInboxInput, signalInboxOutput, + signalSourceRecordInput, } from "./ingest.contracts"; import { IngestService } from "./ingest.service"; @@ -33,4 +37,26 @@ export class IngestRouter { async inbox(@Input() input: z.infer) { return this.ingest.inbox(input); } + + @Query({ + input: signalSourceRecordInput, + output: signalCompanyCandidatesOutput, + meta: restMeta("GET", "/ingest/signals/{sourceRecordId}/company-candidates", [ + "Ingest", + ]), + }) + async companyCandidates(@Input("sourceRecordId") sourceRecordId: string) { + return this.ingest.companyCandidates(sourceRecordId); + } + + @Mutation({ + input: resolveSignalCompanyInput, + output: resolveSignalCompanyOutput, + meta: restMeta("POST", "/ingest/signals/{sourceRecordId}/resolve-company", [ + "Ingest", + ]), + }) + async resolveCompany(@Input() input: z.infer) { + return this.ingest.resolveCompany(input); + } } From 28c69693179946a1bc0cc7206900cc102ffbc6ca Mon Sep 17 00:00:00 2001 From: lumenstech <124342884+lumenstech@users.noreply.github.com> Date: Thu, 3 Sep 2026 21:26:08 -0400 Subject: [PATCH 14/33] fix(api): use production reconciliation columns without Prisma assumptions --- apps/api/src/ingest/ingest.service.ts | 173 +++++++++----------------- 1 file changed, 58 insertions(+), 115 deletions(-) diff --git a/apps/api/src/ingest/ingest.service.ts b/apps/api/src/ingest/ingest.service.ts index 75572785b..6ab82e13d 100644 --- a/apps/api/src/ingest/ingest.service.ts +++ b/apps/api/src/ingest/ingest.service.ts @@ -19,7 +19,7 @@ import type { SignalInboxOutput, } from "./ingest.contracts"; -type BusinessUnitRow = { id: string; key?: string; enabled: boolean }; +type BusinessUnitRow = { id: string; enabled: boolean }; type SourceRecordRow = { id: string }; type SignalRow = { id: string; @@ -69,14 +69,8 @@ export class IngestService { const [businessUnit] = await this.db.$queryRaw` SELECT id, enabled FROM business_unit WHERE key = ${input.project} LIMIT 1 `; - if (!businessUnit) { - throw new BadRequestException( - `Unknown business unit: ${input.project}. Register it before ingesting signals.`, - ); - } - if (!businessUnit.enabled) { - throw new BadRequestException(`Business unit is disabled: ${input.project}.`); - } + if (!businessUnit) throw new BadRequestException(`Unknown business unit: ${input.project}.`); + if (!businessUnit.enabled) throw new BadRequestException(`Business unit is disabled: ${input.project}.`); const [existing] = await this.db.$queryRaw` SELECT id FROM source_record @@ -110,8 +104,7 @@ export class IngestService { ${input.sourceId}, ${input.sourceUrl ?? null}, ${observedAt}, ${payload}::jsonb, CURRENT_TIMESTAMP ) - ON CONFLICT ("sourceSystem", "sourceType", "sourceId") - DO UPDATE SET + ON CONFLICT ("sourceSystem", "sourceType", "sourceId") DO UPDATE SET "businessUnitId" = EXCLUDED."businessUnitId", "sourceUrl" = EXCLUDED."sourceUrl", "observedAt" = EXCLUDED."observedAt", @@ -119,16 +112,6 @@ export class IngestService { RETURNING id `; if (!saved) throw new Error("Signal ingest did not return a source record."); - - this.logger.log({ - message: "Signal ingested", - project: input.project, - source: input.source, - sourceType: input.sourceType, - sourceId: input.sourceId, - sourceRecordId: saved.id, - deduplicated: Boolean(existing), - }); return { status: "accepted", sourceRecordId: saved.id, @@ -159,8 +142,7 @@ export class IngestService { WHERE rm."sourceSystem" = sr."sourceSystem" AND rm."sourceType" = sr."sourceType" AND rm."sourceId" = sr."sourceId" - AND rm."canonicalType" = 'company' - AND rm.status = 'active' + AND rm."canonicalType" = 'company' AND rm.status = 'active' ) AS mapped, sr.payload AS payload FROM source_record sr @@ -173,24 +155,17 @@ export class IngestService { AND ( ${input.status} = 'all' OR (${input.status} = 'mapped' AND EXISTS ( - SELECT 1 FROM record_mapping rm - WHERE rm."sourceSystem" = sr."sourceSystem" - AND rm."sourceType" = sr."sourceType" - AND rm."sourceId" = sr."sourceId" - AND rm."canonicalType" = 'company' AND rm.status = 'active' - )) + SELECT 1 FROM record_mapping rm WHERE rm."sourceSystem" = sr."sourceSystem" + AND rm."sourceType" = sr."sourceType" AND rm."sourceId" = sr."sourceId" + AND rm."canonicalType" = 'company' AND rm.status = 'active')) OR (${input.status} = 'unresolved' AND NOT EXISTS ( - SELECT 1 FROM record_mapping rm - WHERE rm."sourceSystem" = sr."sourceSystem" - AND rm."sourceType" = sr."sourceType" - AND rm."sourceId" = sr."sourceId" - AND rm."canonicalType" = 'company' AND rm.status = 'active' - )) + SELECT 1 FROM record_mapping rm WHERE rm."sourceSystem" = sr."sourceSystem" + AND rm."sourceType" = sr."sourceType" AND rm."sourceId" = sr."sourceId" + AND rm."canonicalType" = 'company' AND rm.status = 'active')) ) - ORDER BY COALESCE( - NULLIF(sr.payload->>'signal_score','')::float, - NULLIF(sr.payload->'metadata'->>'fit_score','')::float, 0 - ) DESC, sr."observedAt" DESC + ORDER BY COALESCE(NULLIF(sr.payload->>'signal_score','')::float, + NULLIF(sr.payload->'metadata'->>'fit_score','')::float, 0) DESC, + sr."observedAt" DESC LIMIT ${input.limit} `; return { @@ -203,39 +178,27 @@ export class IngestService { const signal = await this.loadSignal(sourceRecordId); const entity = this.signalEntity(signal.payload); const domain = this.signalDomain(signal.payload); - const [mapped] = await this.db.$queryRaw>` SELECT "applicationId" FROM record_mapping WHERE "sourceSystem" = ${signal.sourceSystem} AND "sourceType" = ${signal.sourceType} AND "sourceId" = ${signal.sourceId} - AND "canonicalType" = 'company' AND status = 'active' - LIMIT 1 + AND "canonicalType" = 'company' AND status = 'active' LIMIT 1 `; - const rows = entity || domain - ? await this.db.company.findMany({ - where: { - archivedAt: null, - OR: [ - ...(entity - ? [ - { name: { equals: entity, mode: "insensitive" as const } }, - { name: { contains: entity, mode: "insensitive" as const } }, - ] - : []), - ...(domain ? [{ domain }] : []), - ], - }, - select: { id: true, name: true, domain: true, businessUnitId: true }, - take: 20, - }) + ? await this.db.$queryRaw` + SELECT id, name, domain, "businessUnitId" AS "businessUnitId" + FROM company + WHERE "archivedAt" IS NULL AND ( + (${entity}::text IS NOT NULL AND lower(name) LIKE '%' || lower(${entity}) || '%') + OR (${domain}::text IS NOT NULL AND domain = ${domain}) + ) + LIMIT 20 + ` : []; - - const candidates = (rows as CandidateRow[]) + const candidates = rows .map((row) => this.scoreCandidate(row, entity, domain, signal.businessUnitId)) .sort((a, b) => b.score - a.score || a.name.localeCompare(b.name)); - return { sourceRecordId, project: signal.project, @@ -252,32 +215,32 @@ export class IngestService { const requestedDomain = normalizeDomain(input.domain ?? this.signalDomain(signal.payload)); let matchMethod = "manual"; let created = false; + let company: CandidateRow | undefined; - let company = input.companyId - ? await this.db.company.findFirst({ - where: { id: input.companyId, archivedAt: null }, - select: { id: true, name: true, domain: true, businessUnitId: true }, - }) - : null; - + if (input.companyId) { + [company] = await this.db.$queryRaw` + SELECT id, name, domain, "businessUnitId" AS "businessUnitId" + FROM company WHERE id = ${input.companyId} AND "archivedAt" IS NULL LIMIT 1 + `; + if (!company) throw new NotFoundException(`No company with id ${input.companyId}.`); + } if (!company && requestedDomain) { - company = await this.db.company.findFirst({ - where: { domain: requestedDomain, archivedAt: null }, - select: { id: true, name: true, domain: true, businessUnitId: true }, - }); + [company] = await this.db.$queryRaw` + SELECT id, name, domain, "businessUnitId" AS "businessUnitId" + FROM company WHERE domain = ${requestedDomain} AND "archivedAt" IS NULL LIMIT 1 + `; if (company) matchMethod = "domain"; } if (!company && entity) { - company = await this.db.company.findFirst({ - where: { name: { equals: entity, mode: "insensitive" }, archivedAt: null }, - select: { id: true, name: true, domain: true, businessUnitId: true }, - }); + [company] = await this.db.$queryRaw` + SELECT id, name, domain, "businessUnitId" AS "businessUnitId" + FROM company WHERE lower(name) = lower(${entity}) AND "archivedAt" IS NULL LIMIT 1 + `; if (company) matchMethod = "name-exact"; } - if (!company && !input.createIfMissing) { throw new BadRequestException( - "No exact company match. Review /ingest/signals/{sourceRecordId}/company-candidates or set createIfMissing=true after verification.", + "No exact company match. Review the company-candidates endpoint or set createIfMissing=true after verification.", ); } if (!company) { @@ -286,10 +249,10 @@ export class IngestService { name: entity, domain: requestedDomain ?? undefined, }); - await this.db.company.update({ - where: { id: made.id }, - data: { businessUnitId: signal.businessUnitId }, - }); + await this.db.$queryRaw` + UPDATE company SET "businessUnitId" = ${signal.businessUnitId} + WHERE id = ${made.id} AND "businessUnitId" IS NULL + `; company = { id: made.id, name: made.name, @@ -301,14 +264,13 @@ export class IngestService { } const normalizedName = company.name.trim().toLowerCase(); - const canonicalId = randomUUID(); const [canonical] = await this.db.$queryRaw>` INSERT INTO canonical_company ( - id, "businessUnitId", name, "normalizedName", domain, website, - fields, "createdAt", "updatedAt" + id, "businessUnitId", name, "normalizedName", domain, website, fields, + "createdAt", "updatedAt" ) VALUES ( - ${canonicalId}, ${signal.businessUnitId}, ${company.name}, ${normalizedName}, + ${randomUUID()}, ${signal.businessUnitId}, ${company.name}, ${normalizedName}, ${company.domain}, ${company.domain ? `https://${company.domain}` : null}, ${JSON.stringify({ source_signal_id: signal.id })}::jsonb, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP @@ -323,41 +285,24 @@ export class IngestService { await this.db.$queryRaw` INSERT INTO record_mapping ( - id, "sourceSystem", "sourceType", "sourceId", "canonicalType", - "canonicalId", application, "applicationId", "matchMethod", status, - "createdAt", "updatedAt" + id, "sourceSystem", "sourceType", "sourceId", "canonicalType", "canonicalId", + application, "applicationId", "matchMethod", status, "createdAt", "updatedAt" ) VALUES ( ${randomUUID()}, ${signal.sourceSystem}, ${signal.sourceType}, ${signal.sourceId}, - 'company', ${canonical.id}, 'comp-ai-core', ${company.id}, ${matchMethod}, - 'active', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP + 'company', ${canonical.id}, 'comp-ai-core', ${company.id}, ${matchMethod}, 'active', + CURRENT_TIMESTAMP, CURRENT_TIMESTAMP ) - ON CONFLICT ("sourceSystem", "sourceType", "sourceId", "canonicalType") - DO UPDATE SET + ON CONFLICT ("sourceSystem", "sourceType", "sourceId", "canonicalType") DO UPDATE SET "canonicalId" = EXCLUDED."canonicalId", application = EXCLUDED.application, "applicationId" = EXCLUDED."applicationId", "matchMethod" = EXCLUDED."matchMethod", - status = 'active', - "updatedAt" = CURRENT_TIMESTAMP + status = 'active', "updatedAt" = CURRENT_TIMESTAMP `; let researchQueued = false; - if (input.queueResearch) { - const queued = await this.companies.enrich(company.id); - researchQueued = queued.queued; - } - - this.logger.log({ - message: "Signal company resolved", - sourceRecordId: input.sourceRecordId, - companyId: company.id, - canonicalCompanyId: canonical.id, - matchMethod, - created, - researchQueued, - }); - + if (input.queueResearch) researchQueued = (await this.companies.enrich(company.id)).queued; return { sourceRecordId: input.sourceRecordId, canonicalCompanyId: canonical.id, @@ -374,10 +319,8 @@ export class IngestService { SELECT sr.id, sr."businessUnitId" AS "businessUnitId", bu.key AS project, sr."sourceSystem" AS "sourceSystem", sr."sourceType" AS "sourceType", sr."sourceId" AS "sourceId", sr.payload - FROM source_record sr - JOIN business_unit bu ON bu.id = sr."businessUnitId" - WHERE sr.id = ${sourceRecordId} - LIMIT 1 + FROM source_record sr JOIN business_unit bu ON bu.id = sr."businessUnitId" + WHERE sr.id = ${sourceRecordId} LIMIT 1 `; if (!row) throw new NotFoundException(`No source signal with id ${sourceRecordId}.`); return row; From 4155b5b5af7aff804b89bd28d80f51a55af4c30b Mon Sep 17 00:00:00 2001 From: lumenstech <124342884+lumenstech@users.noreply.github.com> Date: Fri, 4 Sep 2026 00:43:49 -0400 Subject: [PATCH 15/33] feat(api): add signal qualification and promotion contracts --- apps/api/src/ingest/ingest.contracts.ts | 67 +++++++++++++++++++++---- 1 file changed, 58 insertions(+), 9 deletions(-) diff --git a/apps/api/src/ingest/ingest.contracts.ts b/apps/api/src/ingest/ingest.contracts.ts index c1df7c2eb..fc722ef97 100644 --- a/apps/api/src/ingest/ingest.contracts.ts +++ b/apps/api/src/ingest/ingest.contracts.ts @@ -95,16 +95,65 @@ export const resolveSignalCompanyOutput = z.object({ researchQueued: z.boolean(), }); +export const signalScoreComponents = z.object({ + icpMatch: z.number().min(0).max(25), + commercialTrigger: z.number().min(0).max(20), + projectRelevance: z.number().min(0).max(20), + companyValue: z.number().min(0).max(10), + location: z.number().min(0).max(10), + decisionMaker: z.number().min(0).max(10), + recency: z.number().min(0).max(5), +}); + +export const signalClassification = z.enum([ + "signal", + "research", + "qualified", + "priority", +]); + +export const qualifySignalInput = signalSourceRecordInput.extend({ + components: signalScoreComponents.optional(), + evidence: z.record(z.string(), z.string().max(2000)).default({}), + notes: z.string().trim().max(4000).nullable().optional(), +}); + +export const qualifySignalOutput = z.object({ + sourceRecordId: z.string(), + score: z.number().min(0).max(100), + classification: signalClassification, + method: z.enum(["components", "legacy-score", "unscored"]), + companyResolved: z.boolean(), + canonicalOpportunityEligible: z.boolean(), + visibleDealEligible: z.boolean(), +}); + +export const promoteSignalInput = signalSourceRecordInput.extend({ + createDeal: z.boolean().default(false), + ownerId: z.string().trim().min(1).nullable().optional(), + amountUsd: z.number().nonnegative().nullable().optional(), +}); + +export const promoteSignalOutput = z.object({ + sourceRecordId: z.string(), + score: z.number().min(0).max(100), + classification: signalClassification, + canonicalOpportunityId: z.string(), + dealId: z.string().nullable(), + createdCanonicalOpportunity: z.boolean(), + createdDeal: z.boolean(), +}); + export type IngestSignalInput = z.infer; export type IngestSignalOutput = z.infer; export type SignalInboxInput = z.infer; export type SignalInboxOutput = z.infer; -export type SignalCompanyCandidatesOutput = z.infer< - typeof signalCompanyCandidatesOutput ->; -export type ResolveSignalCompanyInput = z.infer< - typeof resolveSignalCompanyInput ->; -export type ResolveSignalCompanyOutput = z.infer< - typeof resolveSignalCompanyOutput ->; +export type SignalCompanyCandidatesOutput = z.infer; +export type ResolveSignalCompanyInput = z.infer; +export type ResolveSignalCompanyOutput = z.infer; +export type SignalScoreComponents = z.infer; +export type SignalClassification = z.infer; +export type QualifySignalInput = z.infer; +export type QualifySignalOutput = z.infer; +export type PromoteSignalInput = z.infer; +export type PromoteSignalOutput = z.infer; From 3023b61dde575c6dee6e4bd538a52626397f31dd Mon Sep 17 00:00:00 2001 From: lumenstech <124342884+lumenstech@users.noreply.github.com> Date: Fri, 4 Sep 2026 00:44:19 -0400 Subject: [PATCH 16/33] feat(api): add signal qualification and promotion service --- .../ingest/signal-qualification.service.ts | 269 ++++++++++++++++++ 1 file changed, 269 insertions(+) create mode 100644 apps/api/src/ingest/signal-qualification.service.ts diff --git a/apps/api/src/ingest/signal-qualification.service.ts b/apps/api/src/ingest/signal-qualification.service.ts new file mode 100644 index 000000000..79b16cec8 --- /dev/null +++ b/apps/api/src/ingest/signal-qualification.service.ts @@ -0,0 +1,269 @@ +import type { Db } from "@crm/db"; +import { BadRequestException, Injectable, NotFoundException } from "@nestjs/common"; +import { randomUUID } from "node:crypto"; +import { InjectDatabase } from "../database/database.constants"; +import { DealsService } from "../deals/deals.service"; +import type { + PromoteSignalInput, + PromoteSignalOutput, + QualifySignalInput, + QualifySignalOutput, + SignalClassification, + SignalScoreComponents, +} from "./ingest.contracts"; + +type SignalRow = { + id: string; + businessUnitId: string; + sourceSystem: string; + sourceType: string; + sourceId: string; + payload: Record; +}; + +type CompanyMapping = { + canonicalId: string; + applicationId: string | null; +}; + +type OpportunityMapping = { + canonicalId: string; + applicationId: string | null; +}; + +@Injectable() +export class SignalQualificationService { + constructor( + @InjectDatabase() private readonly db: Db, + private readonly deals: DealsService, + ) {} + + async qualify(input: QualifySignalInput): Promise { + const signal = await this.loadSignal(input.sourceRecordId); + const scored = this.score(signal.payload, input.components); + const company = await this.companyMapping(signal); + + const qualification = { + score: scored.score, + classification: scored.classification, + method: scored.method, + components: input.components ?? null, + evidence: input.evidence, + notes: input.notes ?? null, + qualified_at: new Date().toISOString(), + }; + + await this.db.$queryRaw` + UPDATE source_record + SET payload = payload || ${JSON.stringify({ + signal_score: scored.score, + qualification, + })}::jsonb + WHERE id = ${signal.id} + `; + + return { + sourceRecordId: signal.id, + score: scored.score, + classification: scored.classification, + method: scored.method, + companyResolved: Boolean(company?.applicationId), + canonicalOpportunityEligible: scored.score >= 70 && Boolean(company), + visibleDealEligible: scored.score >= 85 && Boolean(company?.applicationId), + }; + } + + async promote(input: PromoteSignalInput): Promise { + const signal = await this.loadSignal(input.sourceRecordId); + const scored = this.score(signal.payload); + if (scored.score < 70) { + throw new BadRequestException( + `Signal score ${scored.score} is below the canonical opportunity threshold of 70.`, + ); + } + + const company = await this.companyMapping(signal); + if (!company) { + throw new BadRequestException( + "Resolve the signal to a company before promoting it.", + ); + } + + const payload = signal.payload; + const name = this.text(payload.subject) ?? this.text(payload.message) ?? "Qualified opportunity"; + const amount = input.amountUsd ?? this.number(payload.estimated_value_usd); + const stage = scored.score >= 85 ? "priority" : "qualified"; + + let createdCanonicalOpportunity = false; + let canonicalOpportunityId: string; + const existing = await this.opportunityMapping(signal); + if (existing) { + canonicalOpportunityId = existing.canonicalId; + await this.db.$queryRaw` + UPDATE canonical_opportunity + SET "companyId" = ${company.canonicalId}, + name = ${name}, stage = ${stage}, amount = ${amount}, + fields = COALESCE(fields, '{}'::jsonb) || ${JSON.stringify({ + source_signal_id: signal.id, + signal_score: scored.score, + classification: scored.classification, + })}::jsonb, + "updatedAt" = CURRENT_TIMESTAMP + WHERE id = ${canonicalOpportunityId} + `; + } else { + canonicalOpportunityId = randomUUID(); + await this.db.$queryRaw` + INSERT INTO canonical_opportunity ( + id, "businessUnitId", "companyId", "pipelineId", name, stage, + amount, fields, "createdAt", "updatedAt" + ) + VALUES ( + ${canonicalOpportunityId}, ${signal.businessUnitId}, ${company.canonicalId}, + NULL, ${name}, ${stage}, ${amount}, ${JSON.stringify({ + source_signal_id: signal.id, + signal_score: scored.score, + classification: scored.classification, + })}::jsonb, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP + ) + `; + createdCanonicalOpportunity = true; + } + + let dealId = existing?.applicationId ?? null; + let createdDeal = false; + if (input.createDeal) { + if (scored.score < 85) { + throw new BadRequestException( + `Signal score ${scored.score} is below the visible deal threshold of 85.`, + ); + } + if (!company.applicationId) { + throw new BadRequestException("The resolved company is not linked to a visible Comp company."); + } + if (!dealId) { + if (!input.ownerId) { + throw new BadRequestException("ownerId is required when creating a visible deal."); + } + const deal = await this.deals.create({ + name, + companyId: company.applicationId, + ownerId: input.ownerId, + stage: "DEMO_BOOKED", + amountCents: amount == null ? null : Math.round(amount * 100), + currency: "USD", + }); + dealId = deal.id; + createdDeal = true; + await this.db.$queryRaw` + UPDATE deal SET "businessUnitId" = ${signal.businessUnitId} + WHERE id = ${deal.id} + `; + } + } + + await this.db.$queryRaw` + INSERT INTO record_mapping ( + id, "sourceSystem", "sourceType", "sourceId", "canonicalType", "canonicalId", + application, "applicationId", "matchMethod", status, "createdAt", "updatedAt" + ) + VALUES ( + ${randomUUID()}, ${signal.sourceSystem}, ${signal.sourceType}, ${signal.sourceId}, + 'opportunity', ${canonicalOpportunityId}, 'comp-ai-crm', ${dealId}, + 'signal-score', 'active', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP + ) + ON CONFLICT ("sourceSystem", "sourceType", "sourceId", "canonicalType") DO UPDATE SET + "canonicalId" = EXCLUDED."canonicalId", + application = EXCLUDED.application, + "applicationId" = COALESCE(EXCLUDED."applicationId", record_mapping."applicationId"), + "matchMethod" = EXCLUDED."matchMethod", status = 'active', + "updatedAt" = CURRENT_TIMESTAMP + `; + + return { + sourceRecordId: signal.id, + score: scored.score, + classification: scored.classification, + canonicalOpportunityId, + dealId, + createdCanonicalOpportunity, + createdDeal, + }; + } + + private score(payload: Record, components?: SignalScoreComponents) { + if (components) { + const score = Math.round( + components.icpMatch + components.commercialTrigger + components.projectRelevance + + components.companyValue + components.location + components.decisionMaker + components.recency, + ); + return { score, classification: this.classification(score), method: "components" as const }; + } + const explicit = this.number(payload.signal_score); + const metadata = this.object(payload.metadata); + const legacy = explicit ?? this.number(metadata?.fit_score); + if (legacy != null) { + const score = Math.max(0, Math.min(100, Math.round(legacy))); + return { score, classification: this.classification(score), method: "legacy-score" as const }; + } + return { score: 0, classification: "signal" as const, method: "unscored" as const }; + } + + private classification(score: number): SignalClassification { + if (score >= 85) return "priority"; + if (score >= 70) return "qualified"; + if (score >= 50) return "research"; + return "signal"; + } + + private async loadSignal(id: string): Promise { + const [row] = await this.db.$queryRaw` + SELECT id, "businessUnitId" AS "businessUnitId", "sourceSystem" AS "sourceSystem", + "sourceType" AS "sourceType", "sourceId" AS "sourceId", payload + FROM source_record WHERE id = ${id} LIMIT 1 + `; + if (!row) throw new NotFoundException(`No source signal with id ${id}.`); + return row; + } + + private async companyMapping(signal: SignalRow): Promise { + const [row] = await this.db.$queryRaw` + SELECT "canonicalId" AS "canonicalId", "applicationId" AS "applicationId" + FROM record_mapping + WHERE "sourceSystem" = ${signal.sourceSystem} AND "sourceType" = ${signal.sourceType} + AND "sourceId" = ${signal.sourceId} AND "canonicalType" = 'company' + AND status = 'active' LIMIT 1 + `; + return row ?? null; + } + + private async opportunityMapping(signal: SignalRow): Promise { + const [row] = await this.db.$queryRaw` + SELECT "canonicalId" AS "canonicalId", "applicationId" AS "applicationId" + FROM record_mapping + WHERE "sourceSystem" = ${signal.sourceSystem} AND "sourceType" = ${signal.sourceType} + AND "sourceId" = ${signal.sourceId} AND "canonicalType" = 'opportunity' + AND status = 'active' LIMIT 1 + `; + return row ?? null; + } + + private object(value: unknown): Record | null { + return value && typeof value === "object" && !Array.isArray(value) + ? (value as Record) + : null; + } + + private text(value: unknown): string | null { + return typeof value === "string" && value.trim() ? value.trim() : null; + } + + private number(value: unknown): number | null { + if (typeof value === "number" && Number.isFinite(value)) return value; + if (typeof value === "string" && value.trim()) { + const parsed = Number(value); + return Number.isFinite(parsed) ? parsed : null; + } + return null; + } +} From fb0dfc60faf8aa6ca5e60f006c8b935dcf086928 Mon Sep 17 00:00:00 2001 From: lumenstech <124342884+lumenstech@users.noreply.github.com> Date: Fri, 4 Sep 2026 00:44:27 -0400 Subject: [PATCH 17/33] feat(api): wire signal qualification service --- apps/api/src/ingest/ingest.module.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/apps/api/src/ingest/ingest.module.ts b/apps/api/src/ingest/ingest.module.ts index 3b0f51aee..74af8829c 100644 --- a/apps/api/src/ingest/ingest.module.ts +++ b/apps/api/src/ingest/ingest.module.ts @@ -1,11 +1,13 @@ import { Module } from "@nestjs/common"; import { CompaniesModule } from "../companies/companies.module"; +import { DealsModule } from "../deals/deals.module"; import { IngestRouter } from "./ingest.router"; import { IngestService } from "./ingest.service"; +import { SignalQualificationService } from "./signal-qualification.service"; @Module({ - imports: [CompaniesModule], - providers: [IngestService, IngestRouter], - exports: [IngestService], + imports: [CompaniesModule, DealsModule], + providers: [IngestService, SignalQualificationService, IngestRouter], + exports: [IngestService, SignalQualificationService], }) export class IngestModule {} From db28d908be16ae7c97eb4ed36d51748c420f9399 Mon Sep 17 00:00:00 2001 From: lumenstech <124342884+lumenstech@users.noreply.github.com> Date: Fri, 4 Sep 2026 00:44:34 -0400 Subject: [PATCH 18/33] feat(api): expose qualification and promotion endpoints --- apps/api/src/ingest/ingest.router.ts | 29 +++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/apps/api/src/ingest/ingest.router.ts b/apps/api/src/ingest/ingest.router.ts index 0b3759cb4..ba5302fd2 100644 --- a/apps/api/src/ingest/ingest.router.ts +++ b/apps/api/src/ingest/ingest.router.ts @@ -6,6 +6,10 @@ import { restMeta } from "../trpc/openapi"; import { ingestSignalInput, ingestSignalOutput, + promoteSignalInput, + promoteSignalOutput, + qualifySignalInput, + qualifySignalOutput, resolveSignalCompanyInput, resolveSignalCompanyOutput, signalCompanyCandidatesOutput, @@ -14,11 +18,16 @@ import { signalSourceRecordInput, } from "./ingest.contracts"; import { IngestService } from "./ingest.service"; +import { SignalQualificationService } from "./signal-qualification.service"; @Router({ alias: "ingest" }) @UseMiddlewares(AuthMiddleware) export class IngestRouter { - constructor(@Inject(IngestService) private readonly ingest: IngestService) {} + constructor( + @Inject(IngestService) private readonly ingest: IngestService, + @Inject(SignalQualificationService) + private readonly qualification: SignalQualificationService, + ) {} @Mutation({ input: ingestSignalInput, @@ -59,4 +68,22 @@ export class IngestRouter { async resolveCompany(@Input() input: z.infer) { return this.ingest.resolveCompany(input); } + + @Mutation({ + input: qualifySignalInput, + output: qualifySignalOutput, + meta: restMeta("POST", "/ingest/signals/{sourceRecordId}/qualify", ["Ingest"]), + }) + async qualify(@Input() input: z.infer) { + return this.qualification.qualify(input); + } + + @Mutation({ + input: promoteSignalInput, + output: promoteSignalOutput, + meta: restMeta("POST", "/ingest/signals/{sourceRecordId}/promote", ["Ingest"]), + }) + async promote(@Input() input: z.infer) { + return this.qualification.promote(input); + } } From cbcb6ab7719b1b3702f2dac5b8af3f3ab9a525ca Mon Sep 17 00:00:00 2001 From: lumenstech <124342884+lumenstech@users.noreply.github.com> Date: Fri, 4 Sep 2026 01:27:50 -0400 Subject: [PATCH 19/33] feat(api): expose intelligence inbox resolution state --- apps/api/src/ingest/ingest.contracts.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/apps/api/src/ingest/ingest.contracts.ts b/apps/api/src/ingest/ingest.contracts.ts index fc722ef97..0c8fe9849 100644 --- a/apps/api/src/ingest/ingest.contracts.ts +++ b/apps/api/src/ingest/ingest.contracts.ts @@ -25,6 +25,7 @@ export const ingestSignalOutput = z.object({ export const signalInboxInput = z.object({ project: z.string().trim().min(1).max(96).optional(), + source: z.string().trim().min(1).max(96).optional(), minScore: z.number().min(0).max(100).optional(), status: z.enum(["all", "unresolved", "mapped"]).default("unresolved"), limit: z.number().int().min(1).max(200).default(50), @@ -47,6 +48,9 @@ export const signalInboxItem = z.object({ demandTrigger: z.string().nullable(), nextAction: z.string().nullable(), mapped: z.boolean(), + mappedCompanyId: z.string().nullable(), + canonicalOpportunityId: z.string().nullable(), + dealId: z.string().nullable(), payload: signalPayload, }); From 7a9b6c9a680c4707fa623fd22fad805aa152725f Mon Sep 17 00:00:00 2001 From: lumenstech <124342884+lumenstech@users.noreply.github.com> Date: Fri, 4 Sep 2026 01:28:29 -0400 Subject: [PATCH 20/33] fix(api): keep inbox response lean --- apps/api/src/ingest/ingest.contracts.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/apps/api/src/ingest/ingest.contracts.ts b/apps/api/src/ingest/ingest.contracts.ts index 0c8fe9849..f6a151944 100644 --- a/apps/api/src/ingest/ingest.contracts.ts +++ b/apps/api/src/ingest/ingest.contracts.ts @@ -48,9 +48,6 @@ export const signalInboxItem = z.object({ demandTrigger: z.string().nullable(), nextAction: z.string().nullable(), mapped: z.boolean(), - mappedCompanyId: z.string().nullable(), - canonicalOpportunityId: z.string().nullable(), - dealId: z.string().nullable(), payload: signalPayload, }); From 130e24dbac7f527fc9680a1fe370c4edc3a2f9ef Mon Sep 17 00:00:00 2001 From: lumenstech <124342884+lumenstech@users.noreply.github.com> Date: Fri, 4 Sep 2026 01:28:43 -0400 Subject: [PATCH 21/33] feat(app): add intelligence inbox page --- .../app/(app)/[slug]/intelligence/page.tsx | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 apps/app/app/(app)/[slug]/intelligence/page.tsx diff --git a/apps/app/app/(app)/[slug]/intelligence/page.tsx b/apps/app/app/(app)/[slug]/intelligence/page.tsx new file mode 100644 index 000000000..4cec32d6a --- /dev/null +++ b/apps/app/app/(app)/[slug]/intelligence/page.tsx @@ -0,0 +1,61 @@ +import type { Metadata } from "next"; +import { Suspense } from "react"; +import { + PageShell, + PageShellContent, + PageShellDescription, + PageShellHeader, + PageShellHeading, + PageShellLoading, + PageShellTitle, +} from "@/components/page-shell"; +import { requireSession } from "@/lib/session"; +import { HydrateClient } from "@/lib/trpc/hydrate"; +import { getServerQueryClient, getServerTrpc } from "@/lib/trpc/server"; +import { IntelligenceInbox } from "./intelligence-inbox"; + +export const metadata: Metadata = { + title: "Intelligence", +}; + +const DEFAULT_INBOX = { + status: "unresolved" as const, + limit: 100, +}; + +export default function IntelligencePage() { + return ( + + + + Intelligence + + Review incoming commercial signals before they become CRM records or opportunities. + + + + + + }> + + + + + ); +} + +async function Intelligence() { + await requireSession(); + const trpc = getServerTrpc(); + const queryClient = getServerQueryClient(); + await Promise.all([ + queryClient.prefetchQuery(trpc.ingest.inbox.queryOptions(DEFAULT_INBOX)), + queryClient.prefetchQuery(trpc.users.list.queryOptions()), + ]); + + return ( + + + + ); +} From 66d9cf5af63d4fb84da163602e187e434ba28a81 Mon Sep 17 00:00:00 2001 From: lumenstech <124342884+lumenstech@users.noreply.github.com> Date: Fri, 4 Sep 2026 01:29:22 -0400 Subject: [PATCH 22/33] feat(app): build intelligence inbox workflow --- .../intelligence/intelligence-inbox.tsx | 338 ++++++++++++++++++ 1 file changed, 338 insertions(+) create mode 100644 apps/app/app/(app)/[slug]/intelligence/intelligence-inbox.tsx diff --git a/apps/app/app/(app)/[slug]/intelligence/intelligence-inbox.tsx b/apps/app/app/(app)/[slug]/intelligence/intelligence-inbox.tsx new file mode 100644 index 000000000..da36351aa --- /dev/null +++ b/apps/app/app/(app)/[slug]/intelligence/intelligence-inbox.tsx @@ -0,0 +1,338 @@ +"use client"; + +import Launch from "@carbon/icons-react/es/Launch"; +import Renew from "@carbon/icons-react/es/Renew"; +import Search from "@carbon/icons-react/es/Search"; +import { Badge } from "@crm/ui/components/badge"; +import { Button } from "@crm/ui/components/button"; +import { Icon } from "@crm/ui/components/icon"; +import { Input } from "@crm/ui/components/input"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@crm/ui/components/select"; +import { + Sheet, + SheetContent, + SheetDescription, + SheetHeader, + SheetTitle, +} from "@crm/ui/components/sheet"; +import { Spinner } from "@crm/ui/components/spinner"; +import { useMutation, useQuery } from "@tanstack/react-query"; +import { useMemo, useState } from "react"; +import { toast } from "sonner"; +import { LocalRelativeTime } from "@/components/local-date-time"; +import { useTRPC } from "@/lib/trpc/client"; +import type { RouterOutputs } from "@/lib/trpc/types"; + +type SignalRow = RouterOutputs["ingest"]["inbox"]["rows"][number]; +type InboxStatus = "all" | "unresolved" | "mapped"; + +type ScoreComponents = { + icpMatch: number; + commercialTrigger: number; + projectRelevance: number; + companyValue: number; + location: number; + decisionMaker: number; + recency: number; +}; + +const EMPTY_SCORE: ScoreComponents = { + icpMatch: 0, + commercialTrigger: 0, + projectRelevance: 0, + companyValue: 0, + location: 0, + decisionMaker: 0, + recency: 0, +}; + +export function IntelligenceInbox({ + initialInput, +}: { + initialInput: { status: "unresolved"; limit: number }; +}) { + const trpc = useTRPC(); + const [project, setProject] = useState("all"); + const [status, setStatus] = useState(initialInput.status); + const [minScore, setMinScore] = useState("0"); + const [selected, setSelected] = useState(null); + + const input = useMemo( + () => ({ + status, + limit: initialInput.limit, + project: project === "all" ? undefined : project, + minScore: Number(minScore) > 0 ? Number(minScore) : undefined, + }), + [status, initialInput.limit, project, minScore], + ); + + const signals = useQuery({ + ...trpc.ingest.inbox.queryOptions(input), + placeholderData: (previous) => previous, + }); + + const rows = signals.data?.rows ?? []; + const projects = useMemo( + () => [...new Set(rows.map((row) => row.project))].sort(), + [rows], + ); + + return ( + <> +
+
+ + {signals.data?.count ?? 0} signals in this view +
+
+ + + + +
+
+ +
+ + + + + + + + + + + + + + {rows.map((row) => ( + + + + + + + + + + ))} + {rows.length === 0 ? ( + + ) : null} + +
SignalProjectSourceScoreCompanyObservedAction
+
{row.subject ?? row.entity ?? row.sourceId}
+ {row.demandTrigger ?
{row.demandTrigger}
: null} + 		{row.project}{row.source} +
{row.company ?? row.entity ?? "Unresolved"}
+
{row.mapped ? "Resolved" : "Needs match"}
+ 		+ +
No signals match this view.
+
+ + setSelected(null)} + onChanged={async () => { + await signals.refetch(); + }} + /> + + ); +} + +function SignalReviewSheet({ + row, + onClose, + onChanged, +}: { + row: SignalRow | null; + onClose: () => void; + onChanged: () => Promise; +}) { + const trpc = useTRPC(); + const [score, setScore] = useState(EMPTY_SCORE); + const [ownerId, setOwnerId] = useState("unassigned"); + + const candidates = useQuery({ + ...trpc.ingest.companyCandidates.queryOptions({ sourceRecordId: row?.sourceRecordId ?? "" }), + enabled: Boolean(row), + }); + const users = useQuery(trpc.users.list.queryOptions()); + + const refresh = async () => { + await Promise.all([onChanged(), candidates.refetch()]); + }; + + const resolve = useMutation(trpc.ingest.resolveCompany.mutationOptions({ + onSuccess: async (result) => { + toast.success(`${result.companyName} resolved${result.researchQueued ? " and queued for research" : ""}.`); + await refresh(); + }, + onError: (error) => toast.error(error.message), + })); + const qualify = useMutation(trpc.ingest.qualify.mutationOptions({ + onSuccess: async (result) => { + toast.success(`Signal scored ${result.score}/100 — ${result.classification}.`); + await refresh(); + }, + onError: (error) => toast.error(error.message), + })); + const promote = useMutation(trpc.ingest.promote.mutationOptions({ + onSuccess: async (result) => { + toast.success(result.createdDeal ? "Opportunity and visible deal created." : "Canonical opportunity created."); + await refresh(); + }, + onError: (error) => toast.error(error.message), + })); + const research = useMutation(trpc.companies.enrich.mutationOptions({ + onSuccess: (result) => toast.success(result.queued ? "Company research queued." : "Research is already queued."), + onError: (error) => toast.error(error.message), + })); + + if (!row) return null; + const mappedCompanyId = candidates.data?.mappedCompanyId ?? null; + const pending = resolve.isPending || qualify.isPending || promote.isPending || research.isPending; + + return ( + { if (!open) onClose(); }}> + + + {row.subject ?? row.entity ?? "Signal review"} + {row.project} · {row.source} · {row.sourceType} + + +
+
+
+
Current score
+ {row.mapped ? "Company resolved" : "Company unresolved"} +
+ {row.demandTrigger ? : null} + {row.nextAction ? : null} + {row.sourceUrl ? ( + + ) : null} +
+ +
+

1. Resolve company

Match this signal to one existing account before research or opportunity promotion.

+ {candidates.isLoading ? : null} + {mappedCompanyId ? ( +
Mapped to Comp company
+ ) : ( +
+ {(candidates.data?.candidates ?? []).slice(0, 5).map((candidate) => ( +
+
{candidate.name}
{candidate.domain ?? "No domain"} · match {candidate.score}
+ +
+ ))} + {(candidates.data?.candidates ?? []).length === 0 ? ( +
No existing company candidate found. Create only after verifying the entity name.
+ ) : null} + +
+ )} +
+ +
+

2. Qualify signal

Use the transparent 100-point commercial score.

+
+ setScore((s) => ({ ...s, icpMatch: value }))} /> + setScore((s) => ({ ...s, commercialTrigger: value }))} /> + setScore((s) => ({ ...s, projectRelevance: value }))} /> + setScore((s) => ({ ...s, companyValue: value }))} /> + setScore((s) => ({ ...s, location: value }))} /> + setScore((s) => ({ ...s, decisionMaker: value }))} /> + setScore((s) => ({ ...s, recency: value }))} /> +
Total
{Object.values(score).reduce((sum, value) => sum + value, 0)} / 100
+
+
+ + {row.signalScore != null ? : null} +
+
+ +
+

3. Promote

70+ can become a canonical opportunity. A visible deal requires 85+, a resolved company, and an owner.

+
+ +
+
+ + +
+
+
+ + + ); +} + +function ScoreBadge({ score }: { score: number | null }) { + if (score == null) return Unscored; + if (score >= 85) return {score} · Priority; + if (score >= 70) return {score} · Qualified; + if (score >= 50) return {score} · Research; + return {score} · Signal; +} + +function Info({ label, value }: { label: string; value: string }) { + return
{label}
{value}
; +} + +function ScoreInput({ label, value, max, onChange }: { label: string; value: number; max: number; onChange: (value: number) => void }) { + return ( + + ); +} From 2511adff6ddc19803d1e6fc90912eb52fa3e8b78 Mon Sep 17 00:00:00 2001 From: lumenstech <124342884+lumenstech@users.noreply.github.com> Date: Fri, 4 Sep 2026 01:29:49 -0400 Subject: [PATCH 23/33] feat(app): add intelligence to primary navigation --- apps/app/components/app-icon-rail.tsx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apps/app/components/app-icon-rail.tsx b/apps/app/components/app-icon-rail.tsx index ad3440e00..f751a8a13 100644 --- a/apps/app/components/app-icon-rail.tsx +++ b/apps/app/components/app-icon-rail.tsx @@ -4,6 +4,7 @@ import Building from "@carbon/icons-react/es/Building"; import Close from "@carbon/icons-react/es/Close"; import Dashboard from "@carbon/icons-react/es/Dashboard"; import Partnership from "@carbon/icons-react/es/Partnership"; +import Search from "@carbon/icons-react/es/Search"; import Settings from "@carbon/icons-react/es/Settings"; import UserMultiple from "@carbon/icons-react/es/UserMultiple"; import { Button } from "@crm/ui/components/button"; @@ -49,6 +50,7 @@ const ITEMS: RailItem[] = [ match: "prefix", related: ["/agents"], }, + { title: "Intelligence", href: "/intelligence", icon: Search, match: "prefix" }, { title: "Companies", href: "/companies", icon: Building, match: "prefix" }, { title: "Contacts", From 0443822b5b298aece03919307d60518fc9d90aa1 Mon Sep 17 00:00:00 2001 From: lumenstech <124342884+lumenstech@users.noreply.github.com> Date: Fri, 4 Sep 2026 01:30:58 -0400 Subject: [PATCH 24/33] feat(app): prefetch intelligence inbox from navigation --- apps/app/components/crm/section-prefetch.ts | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/apps/app/components/crm/section-prefetch.ts b/apps/app/components/crm/section-prefetch.ts index 8636fa5ff..e6eaf1cf2 100644 --- a/apps/app/components/crm/section-prefetch.ts +++ b/apps/app/components/crm/section-prefetch.ts @@ -7,7 +7,13 @@ import { contactsSearchParams } from "@/app/(app)/[slug]/contacts/contacts-searc import { dealsSearchParams } from "@/app/(app)/[slug]/deals/deals-search-params"; import { useTRPC } from "@/lib/trpc/client"; -export type Section = "/" | "/companies" | "/contacts" | "/deals" | "/settings"; +export type Section = + | "/" + | "/intelligence" + | "/companies" + | "/contacts" + | "/deals" + | "/settings"; export function usePrefetchSection(): (section: string) => void { const trpc = useTRPC(); @@ -21,6 +27,11 @@ export function usePrefetchSection(): (section: string) => void { trpc.dashboard.summary.queryOptions({ scope: "me" }), ); return; + case "/intelligence": + void queryClient.prefetchQuery( + trpc.ingest.inbox.queryOptions({ status: "unresolved", limit: 100 }), + ); + return; case "/companies": void queryClient.prefetchQuery( trpc.companies.list.queryOptions( From 91f29994c696939da38bd84cc77689596ecc553c Mon Sep 17 00:00:00 2001 From: Danny Date: Fri, 4 Sep 2026 02:21:09 -0400 Subject: [PATCH 25/33] fix: sync intelligence workflow with foundation schema --- apps/api/src/generated/server.ts | 27 + apps/api/src/ingest/ingest.contracts.ts | 12 +- apps/api/src/ingest/ingest.router.ts | 20 +- apps/api/src/ingest/ingest.service.ts | 68 ++- .../ingest/signal-qualification.service.ts | 64 ++- .../intelligence/intelligence-inbox.tsx | 474 +++++++++++++++--- .../app/(app)/[slug]/intelligence/page.tsx | 3 +- apps/app/components/app-icon-rail.tsx | 7 +- apps/app/components/crm/section-prefetch.ts | 5 +- .../migration.sql | 318 ++++++++++++ packages/db/prisma/schema.prisma | 255 ++++++++++ 11 files changed, 1125 insertions(+), 128 deletions(-) create mode 100644 packages/db/prisma/migrations/20260902184536_sequencenow_crm_foundation/migration.sql diff --git a/apps/api/src/generated/server.ts b/apps/api/src/generated/server.ts index b77da4001..9cf683813 100644 --- a/apps/api/src/generated/server.ts +++ b/apps/api/src/generated/server.ts @@ -26,6 +26,7 @@ import { enrichmentQueueInput } from "@crm/validation/enrichment-queue"; import { fieldListInput, fieldListOutput, fieldByKeyInput, serializedFieldOutput, fieldEntityInput, fieldFiltersOutput, fieldIdInput, fieldCoverageOutput, fieldCreateInput, fieldUpdateArgs, fieldReorderInput, fieldReorderOutput, fieldDeleteOutput, fieldBackfillOutput } from "../fields/fields.contracts"; import { googleConnectionStatusOutput, setAutoCreateInput, suppressDomainInput, suppressDomainOutput, threadInput, emailThreadOutput, calendarEventInput, calendarEventOutput } from "../google/google.contracts"; import { purgeSyncedDataOutput, revokeAccessOutput, microsoftConnectionStatusOutput, setOutlookAutoCreateInput } from "../microsoft/microsoft.contracts"; +import { ingestSignalInput, ingestSignalOutput, signalInboxInput, signalInboxOutput, signalSourceRecordInput, signalCompanyCandidatesOutput, resolveSignalCompanyInput, resolveSignalCompanyOutput, qualifySignalInput, qualifySignalOutput, promoteSignalInput, promoteSignalOutput } from "../ingest/ingest.contracts"; import { savedViewListInput, savedViewListOutput, savedViewCreateInput, savedViewOutput, savedViewUpdateArgs, savedViewIdInput, savedViewDeleteOutput } from "../saved-views/saved-views.contracts"; import { agentModelOutput, modelCatalogOutput, setAgentModelInput, researchKeyOutput, setResearchKeyInput, archiveRetentionOutput, setArchiveRetentionDaysInput } from "../settings/settings.contracts"; import { slackStatusOutput, slackMatchesOutput, slackChannelsInput, slackChannelsOutput, slackJoinChannelInput, slackJoinChannelOutput, slackRefreshPeopleOutput, slackCreateChannelInput, slackCreateChannelOutput, slackDisconnectOutput } from "../slack/slack.contracts"; @@ -558,6 +559,32 @@ const appRouter = t.router({ .output(calendarEventOutput) .query(async () => "PLACEHOLDER_DO_NOT_REMOVE" as any) }), + ingest: t.router({ + signal: publicProcedure + .input(ingestSignalInput) + .output(ingestSignalOutput) + .mutation(async () => "PLACEHOLDER_DO_NOT_REMOVE" as any), + inbox: publicProcedure + .input(signalInboxInput) + .output(signalInboxOutput) + .query(async () => "PLACEHOLDER_DO_NOT_REMOVE" as any), + companyCandidates: publicProcedure + .input(signalSourceRecordInput) + .output(signalCompanyCandidatesOutput) + .query(async () => "PLACEHOLDER_DO_NOT_REMOVE" as any), + resolveCompany: publicProcedure + .input(resolveSignalCompanyInput) + .output(resolveSignalCompanyOutput) + .mutation(async () => "PLACEHOLDER_DO_NOT_REMOVE" as any), + qualify: publicProcedure + .input(qualifySignalInput) + .output(qualifySignalOutput) + .mutation(async () => "PLACEHOLDER_DO_NOT_REMOVE" as any), + promote: publicProcedure + .input(promoteSignalInput) + .output(promoteSignalOutput) + .mutation(async () => "PLACEHOLDER_DO_NOT_REMOVE" as any) + }), microsoft: t.router({ status: publicProcedure .output(microsoftConnectionStatusOutput) diff --git a/apps/api/src/ingest/ingest.contracts.ts b/apps/api/src/ingest/ingest.contracts.ts index f6a151944..304866c11 100644 --- a/apps/api/src/ingest/ingest.contracts.ts +++ b/apps/api/src/ingest/ingest.contracts.ts @@ -149,9 +149,15 @@ export type IngestSignalInput = z.infer; export type IngestSignalOutput = z.infer; export type SignalInboxInput = z.infer; export type SignalInboxOutput = z.infer; -export type SignalCompanyCandidatesOutput = z.infer; -export type ResolveSignalCompanyInput = z.infer; -export type ResolveSignalCompanyOutput = z.infer; +export type SignalCompanyCandidatesOutput = z.infer< + typeof signalCompanyCandidatesOutput +>; +export type ResolveSignalCompanyInput = z.infer< + typeof resolveSignalCompanyInput +>; +export type ResolveSignalCompanyOutput = z.infer< + typeof resolveSignalCompanyOutput +>; export type SignalScoreComponents = z.infer; export type SignalClassification = z.infer; export type QualifySignalInput = z.infer; diff --git a/apps/api/src/ingest/ingest.router.ts b/apps/api/src/ingest/ingest.router.ts index ba5302fd2..6352246b6 100644 --- a/apps/api/src/ingest/ingest.router.ts +++ b/apps/api/src/ingest/ingest.router.ts @@ -50,9 +50,11 @@ export class IngestRouter { @Query({ input: signalSourceRecordInput, output: signalCompanyCandidatesOutput, - meta: restMeta("GET", "/ingest/signals/{sourceRecordId}/company-candidates", [ - "Ingest", - ]), + meta: restMeta( + "GET", + "/ingest/signals/{sourceRecordId}/company-candidates", + ["Ingest"], + ), }) async companyCandidates(@Input("sourceRecordId") sourceRecordId: string) { return this.ingest.companyCandidates(sourceRecordId); @@ -65,14 +67,18 @@ export class IngestRouter { "Ingest", ]), }) - async resolveCompany(@Input() input: z.infer) { + async resolveCompany( + @Input() input: z.infer, + ) { return this.ingest.resolveCompany(input); } @Mutation({ input: qualifySignalInput, output: qualifySignalOutput, - meta: restMeta("POST", "/ingest/signals/{sourceRecordId}/qualify", ["Ingest"]), + meta: restMeta("POST", "/ingest/signals/{sourceRecordId}/qualify", [ + "Ingest", + ]), }) async qualify(@Input() input: z.infer) { return this.qualification.qualify(input); @@ -81,7 +87,9 @@ export class IngestRouter { @Mutation({ input: promoteSignalInput, output: promoteSignalOutput, - meta: restMeta("POST", "/ingest/signals/{sourceRecordId}/promote", ["Ingest"]), + meta: restMeta("POST", "/ingest/signals/{sourceRecordId}/promote", [ + "Ingest", + ]), }) async promote(@Input() input: z.infer) { return this.qualification.promote(input); diff --git a/apps/api/src/ingest/ingest.service.ts b/apps/api/src/ingest/ingest.service.ts index 6ab82e13d..bc0f50b13 100644 --- a/apps/api/src/ingest/ingest.service.ts +++ b/apps/api/src/ingest/ingest.service.ts @@ -1,11 +1,10 @@ +import { randomUUID } from "node:crypto"; import type { Db } from "@crm/db"; import { BadRequestException, Injectable, - Logger, NotFoundException, } from "@nestjs/common"; -import { randomUUID } from "node:crypto"; import { CompaniesService } from "../companies/companies.service"; import { normalizeDomain } from "../companies/domain"; import { InjectDatabase } from "../database/database.constants"; @@ -58,8 +57,6 @@ type InboxRow = { @Injectable() export class IngestService { - private readonly logger = new Logger(IngestService.name); - constructor( @InjectDatabase() private readonly db: Db, private readonly companies: CompaniesService, @@ -69,8 +66,12 @@ export class IngestService { const [businessUnit] = await this.db.$queryRaw` SELECT id, enabled FROM business_unit WHERE key = ${input.project} LIMIT 1 `; - if (!businessUnit) throw new BadRequestException(`Unknown business unit: ${input.project}.`); - if (!businessUnit.enabled) throw new BadRequestException(`Business unit is disabled: ${input.project}.`); + if (!businessUnit) + throw new BadRequestException(`Unknown business unit: ${input.project}.`); + if (!businessUnit.enabled) + throw new BadRequestException( + `Business unit is disabled: ${input.project}.`, + ); const [existing] = await this.db.$queryRaw` SELECT id FROM source_record @@ -80,7 +81,9 @@ export class IngestService { LIMIT 1 `; - const observedAt = input.observedAt ? new Date(input.observedAt) : new Date(); + const observedAt = input.observedAt + ? new Date(input.observedAt) + : new Date(); const payload = JSON.stringify({ ...input.payload, project: input.project, @@ -111,7 +114,8 @@ export class IngestService { payload = EXCLUDED.payload RETURNING id `; - if (!saved) throw new Error("Signal ingest did not return a source record."); + if (!saved) + throw new Error("Signal ingest did not return a source record."); return { status: "accepted", sourceRecordId: saved.id, @@ -169,24 +173,32 @@ export class IngestService { LIMIT ${input.limit} `; return { - rows: rows.map((row) => ({ ...row, observedAt: row.observedAt.toISOString() })), + rows: rows.map((row) => ({ + ...row, + observedAt: row.observedAt.toISOString(), + })), count: rows.length, }; } - async companyCandidates(sourceRecordId: string): Promise { + async companyCandidates( + sourceRecordId: string, + ): Promise { const signal = await this.loadSignal(sourceRecordId); const entity = this.signalEntity(signal.payload); const domain = this.signalDomain(signal.payload); - const [mapped] = await this.db.$queryRaw>` + const [mapped] = await this.db.$queryRaw< + Array<{ applicationId: string | null }> + >` SELECT "applicationId" FROM record_mapping WHERE "sourceSystem" = ${signal.sourceSystem} AND "sourceType" = ${signal.sourceType} AND "sourceId" = ${signal.sourceId} AND "canonicalType" = 'company' AND status = 'active' LIMIT 1 `; - const rows = entity || domain - ? await this.db.$queryRaw` + const rows = + entity || domain + ? await this.db.$queryRaw` SELECT id, name, domain, "businessUnitId" AS "businessUnitId" FROM company WHERE "archivedAt" IS NULL AND ( @@ -195,9 +207,11 @@ export class IngestService { ) LIMIT 20 ` - : []; + : []; const candidates = rows - .map((row) => this.scoreCandidate(row, entity, domain, signal.businessUnitId)) + .map((row) => + this.scoreCandidate(row, entity, domain, signal.businessUnitId), + ) .sort((a, b) => b.score - a.score || a.name.localeCompare(b.name)); return { sourceRecordId, @@ -209,10 +223,14 @@ export class IngestService { }; } - async resolveCompany(input: ResolveSignalCompanyInput): Promise { + async resolveCompany( + input: ResolveSignalCompanyInput, + ): Promise { const signal = await this.loadSignal(input.sourceRecordId); const entity = input.companyName ?? this.signalEntity(signal.payload); - const requestedDomain = normalizeDomain(input.domain ?? this.signalDomain(signal.payload)); + const requestedDomain = normalizeDomain( + input.domain ?? this.signalDomain(signal.payload), + ); let matchMethod = "manual"; let created = false; let company: CandidateRow | undefined; @@ -222,7 +240,8 @@ export class IngestService { SELECT id, name, domain, "businessUnitId" AS "businessUnitId" FROM company WHERE id = ${input.companyId} AND "archivedAt" IS NULL LIMIT 1 `; - if (!company) throw new NotFoundException(`No company with id ${input.companyId}.`); + if (!company) + throw new NotFoundException(`No company with id ${input.companyId}.`); } if (!company && requestedDomain) { [company] = await this.db.$queryRaw` @@ -244,7 +263,10 @@ export class IngestService { ); } if (!company) { - if (!entity) throw new BadRequestException("A company name is required to create a company."); + if (!entity) + throw new BadRequestException( + "A company name is required to create a company.", + ); const made = await this.companies.create({ name: entity, domain: requestedDomain ?? undefined, @@ -302,7 +324,8 @@ export class IngestService { `; let researchQueued = false; - if (input.queueResearch) researchQueued = (await this.companies.enrich(company.id)).queued; + if (input.queueResearch) + researchQueued = (await this.companies.enrich(company.id)).queued; return { sourceRecordId: input.sourceRecordId, canonicalCompanyId: canonical.id, @@ -322,7 +345,10 @@ export class IngestService { FROM source_record sr JOIN business_unit bu ON bu.id = sr."businessUnitId" WHERE sr.id = ${sourceRecordId} LIMIT 1 `; - if (!row) throw new NotFoundException(`No source signal with id ${sourceRecordId}.`); + if (!row) + throw new NotFoundException( + `No source signal with id ${sourceRecordId}.`, + ); return row; } diff --git a/apps/api/src/ingest/signal-qualification.service.ts b/apps/api/src/ingest/signal-qualification.service.ts index 79b16cec8..172350448 100644 --- a/apps/api/src/ingest/signal-qualification.service.ts +++ b/apps/api/src/ingest/signal-qualification.service.ts @@ -1,6 +1,10 @@ -import type { Db } from "@crm/db"; -import { BadRequestException, Injectable, NotFoundException } from "@nestjs/common"; import { randomUUID } from "node:crypto"; +import type { Db } from "@crm/db"; +import { + BadRequestException, + Injectable, + NotFoundException, +} from "@nestjs/common"; import { InjectDatabase } from "../database/database.constants"; import { DealsService } from "../deals/deals.service"; import type { @@ -69,7 +73,8 @@ export class SignalQualificationService { method: scored.method, companyResolved: Boolean(company?.applicationId), canonicalOpportunityEligible: scored.score >= 70 && Boolean(company), - visibleDealEligible: scored.score >= 85 && Boolean(company?.applicationId), + visibleDealEligible: + scored.score >= 85 && Boolean(company?.applicationId), }; } @@ -90,7 +95,10 @@ export class SignalQualificationService { } const payload = signal.payload; - const name = this.text(payload.subject) ?? this.text(payload.message) ?? "Qualified opportunity"; + const name = + this.text(payload.subject) ?? + this.text(payload.message) ?? + "Qualified opportunity"; const amount = input.amountUsd ?? this.number(payload.estimated_value_usd); const stage = scored.score >= 85 ? "priority" : "qualified"; @@ -139,11 +147,15 @@ export class SignalQualificationService { ); } if (!company.applicationId) { - throw new BadRequestException("The resolved company is not linked to a visible Comp company."); + throw new BadRequestException( + "The resolved company is not linked to a visible Comp company.", + ); } if (!dealId) { if (!input.ownerId) { - throw new BadRequestException("ownerId is required when creating a visible deal."); + throw new BadRequestException( + "ownerId is required when creating a visible deal.", + ); } const deal = await this.deals.create({ name, @@ -191,22 +203,42 @@ export class SignalQualificationService { }; } - private score(payload: Record, components?: SignalScoreComponents) { + private score( + payload: Record, + components?: SignalScoreComponents, + ) { if (components) { const score = Math.round( - components.icpMatch + components.commercialTrigger + components.projectRelevance + - components.companyValue + components.location + components.decisionMaker + components.recency, + components.icpMatch + + components.commercialTrigger + + components.projectRelevance + + components.companyValue + + components.location + + components.decisionMaker + + components.recency, ); - return { score, classification: this.classification(score), method: "components" as const }; + return { + score, + classification: this.classification(score), + method: "components" as const, + }; } const explicit = this.number(payload.signal_score); const metadata = this.object(payload.metadata); const legacy = explicit ?? this.number(metadata?.fit_score); if (legacy != null) { const score = Math.max(0, Math.min(100, Math.round(legacy))); - return { score, classification: this.classification(score), method: "legacy-score" as const }; + return { + score, + classification: this.classification(score), + method: "legacy-score" as const, + }; } - return { score: 0, classification: "signal" as const, method: "unscored" as const }; + return { + score: 0, + classification: "signal" as const, + method: "unscored" as const, + }; } private classification(score: number): SignalClassification { @@ -226,7 +258,9 @@ export class SignalQualificationService { return row; } - private async companyMapping(signal: SignalRow): Promise { + private async companyMapping( + signal: SignalRow, + ): Promise { const [row] = await this.db.$queryRaw` SELECT "canonicalId" AS "canonicalId", "applicationId" AS "applicationId" FROM record_mapping @@ -237,7 +271,9 @@ export class SignalQualificationService { return row ?? null; } - private async opportunityMapping(signal: SignalRow): Promise { + private async opportunityMapping( + signal: SignalRow, + ): Promise { const [row] = await this.db.$queryRaw` SELECT "canonicalId" AS "canonicalId", "applicationId" AS "applicationId" FROM record_mapping diff --git a/apps/app/app/(app)/[slug]/intelligence/intelligence-inbox.tsx b/apps/app/app/(app)/[slug]/intelligence/intelligence-inbox.tsx index da36351aa..413e5d7fe 100644 --- a/apps/app/app/(app)/[slug]/intelligence/intelligence-inbox.tsx +++ b/apps/app/app/(app)/[slug]/intelligence/intelligence-inbox.tsx @@ -92,7 +92,10 @@ export function IntelligenceInbox({ {signals.data?.count ?? 0} signals in this view
- setStatus(value as InboxStatus)} + > @@ -109,7 +112,9 @@ export function IntelligenceInbox({ All projects {projects.map((value) => ( - {value} + + {value} + ))} @@ -124,7 +129,12 @@ export function IntelligenceInbox({ 85+ priority -
@@ -147,24 +157,57 @@ export function IntelligenceInbox({ {rows.map((row) => ( -
{row.subject ?? row.entity ?? row.sourceId}
- {row.demandTrigger ?
{row.demandTrigger}
: null} +
+ {row.subject ?? row.entity ?? row.sourceId} +
+ {row.demandTrigger ? ( +
+ {row.demandTrigger} +
+ ) : null} + + + {row.project} + + + {row.source} + + + - {row.project} - {row.source} - -
{row.company ?? row.entity ?? "Unresolved"}
-
{row.mapped ? "Resolved" : "Needs match"}
+
+ {row.company ?? row.entity ?? "Unresolved"} +
+
+ + {row.mapped ? "Resolved" : "Needs match"} + +
+ + + - - + ))} {rows.length === 0 ? ( - No signals match this view. + + + No signals match this view. + + ) : null} @@ -195,7 +238,9 @@ function SignalReviewSheet({ const [ownerId, setOwnerId] = useState("unassigned"); const candidates = useQuery({ - ...trpc.ingest.companyCandidates.queryOptions({ sourceRecordId: row?.sourceRecordId ?? "" }), + ...trpc.ingest.companyCandidates.queryOptions({ + sourceRecordId: row?.sourceRecordId ?? "", + }), enabled: Boolean(row), }); const users = useQuery(trpc.users.list.queryOptions()); @@ -204,110 +249,348 @@ function SignalReviewSheet({ await Promise.all([onChanged(), candidates.refetch()]); }; - const resolve = useMutation(trpc.ingest.resolveCompany.mutationOptions({ - onSuccess: async (result) => { - toast.success(`${result.companyName} resolved${result.researchQueued ? " and queued for research" : ""}.`); - await refresh(); - }, - onError: (error) => toast.error(error.message), - })); - const qualify = useMutation(trpc.ingest.qualify.mutationOptions({ - onSuccess: async (result) => { - toast.success(`Signal scored ${result.score}/100 — ${result.classification}.`); - await refresh(); - }, - onError: (error) => toast.error(error.message), - })); - const promote = useMutation(trpc.ingest.promote.mutationOptions({ - onSuccess: async (result) => { - toast.success(result.createdDeal ? "Opportunity and visible deal created." : "Canonical opportunity created."); - await refresh(); - }, - onError: (error) => toast.error(error.message), - })); - const research = useMutation(trpc.companies.enrich.mutationOptions({ - onSuccess: (result) => toast.success(result.queued ? "Company research queued." : "Research is already queued."), - onError: (error) => toast.error(error.message), - })); + const resolve = useMutation( + trpc.ingest.resolveCompany.mutationOptions({ + onSuccess: async (result) => { + toast.success( + `${result.companyName} resolved${result.researchQueued ? " and queued for research" : ""}.`, + ); + await refresh(); + }, + onError: (error) => toast.error(error.message), + }), + ); + const qualify = useMutation( + trpc.ingest.qualify.mutationOptions({ + onSuccess: async (result) => { + toast.success( + `Signal scored ${result.score}/100 — ${result.classification}.`, + ); + await refresh(); + }, + onError: (error) => toast.error(error.message), + }), + ); + const promote = useMutation( + trpc.ingest.promote.mutationOptions({ + onSuccess: async (result) => { + toast.success( + result.createdDeal + ? "Opportunity and visible deal created." + : "Canonical opportunity created.", + ); + await refresh(); + }, + onError: (error) => toast.error(error.message), + }), + ); + const research = useMutation( + trpc.companies.enrich.mutationOptions({ + onSuccess: (result) => + toast.success( + result.queued + ? "Company research queued." + : "Research is already queued.", + ), + onError: (error) => toast.error(error.message), + }), + ); if (!row) return null; const mappedCompanyId = candidates.data?.mappedCompanyId ?? null; - const pending = resolve.isPending || qualify.isPending || promote.isPending || research.isPending; + const pending = + resolve.isPending || + qualify.isPending || + promote.isPending || + research.isPending; return ( - { if (!open) onClose(); }}> + { + if (!open) onClose(); + }} + > - {row.subject ?? row.entity ?? "Signal review"} - {row.project} · {row.source} · {row.sourceType} + + {row.subject ?? row.entity ?? "Signal review"} + + + {row.project} · {row.source} · {row.sourceType} +
-
Current score
- {row.mapped ? "Company resolved" : "Company unresolved"} +
+
+ Current score +
+
+ +
+
+ + {row.mapped ? "Company resolved" : "Company unresolved"} +
- {row.demandTrigger ? : null} - {row.nextAction ? : null} + {row.demandTrigger ? ( + + ) : null} + {row.nextAction ? ( + + ) : null} {row.sourceUrl ? ( - + ) : null}
-

1. Resolve company

Match this signal to one existing account before research or opportunity promotion.

+
+

1. Resolve company

+

+ Match this signal to one existing account before research or + opportunity promotion. +

+
{candidates.isLoading ? : null} {mappedCompanyId ? ( -
Mapped to Comp company
+
+ Mapped to Comp company + +
) : (
- {(candidates.data?.candidates ?? []).slice(0, 5).map((candidate) => ( -
-
{candidate.name}
{candidate.domain ?? "No domain"} · match {candidate.score}
- -
- ))} + {(candidates.data?.candidates ?? []) + .slice(0, 5) + .map((candidate) => ( +
+
+
+ {candidate.name} +
+
+ {candidate.domain ?? "No domain"} · match{" "} + {candidate.score} +
+
+ +
+ ))} {(candidates.data?.candidates ?? []).length === 0 ? ( -
No existing company candidate found. Create only after verifying the entity name.
+
+ No existing company candidate found. Create only after + verifying the entity name. +
) : null} - +
)}
-

2. Qualify signal

Use the transparent 100-point commercial score.

+
+

2. Qualify signal

+

+ Use the transparent 100-point commercial score. +

+
- setScore((s) => ({ ...s, icpMatch: value }))} /> - setScore((s) => ({ ...s, commercialTrigger: value }))} /> - setScore((s) => ({ ...s, projectRelevance: value }))} /> - setScore((s) => ({ ...s, companyValue: value }))} /> - setScore((s) => ({ ...s, location: value }))} /> - setScore((s) => ({ ...s, decisionMaker: value }))} /> - setScore((s) => ({ ...s, recency: value }))} /> -
Total
{Object.values(score).reduce((sum, value) => sum + value, 0)} / 100
+ + setScore((s) => ({ ...s, icpMatch: value })) + } + /> + + setScore((s) => ({ ...s, commercialTrigger: value })) + } + /> + + setScore((s) => ({ ...s, projectRelevance: value })) + } + /> + + setScore((s) => ({ ...s, companyValue: value })) + } + /> + + setScore((s) => ({ ...s, location: value })) + } + /> + + setScore((s) => ({ ...s, decisionMaker: value })) + } + /> + + setScore((s) => ({ ...s, recency: value })) + } + /> +
+
+ Total +
+ {Object.values(score).reduce( + (sum, value) => sum + value, + 0, + )}{" "} + / 100 +
+
+
- - {row.signalScore != null ? : null} + + {row.signalScore != null ? ( + + ) : null}
-

3. Promote

70+ can become a canonical opportunity. A visible deal requires 85+, a resolved company, and an owner.

+
+

3. Promote

+

+ 70+ can become a canonical opportunity. A visible deal requires + 85+, a resolved company, and an owner. +

+
- +
- +
@@ -319,20 +602,49 @@ function SignalReviewSheet({ function ScoreBadge({ score }: { score: number | null }) { if (score == null) return Unscored; if (score >= 85) return {score} · Priority; - if (score >= 70) return {score} · Qualified; + if (score >= 70) + return {score} · Qualified; if (score >= 50) return {score} · Research; return {score} · Signal; } function Info({ label, value }: { label: string; value: string }) { - return
{label}
{value}
; + return ( +
+
{label}
+
{value}
+
+ ); } -function ScoreInput({ label, value, max, onChange }: { label: string; value: number; max: number; onChange: (value: number) => void }) { +function ScoreInput({ + label, + value, + max, + onChange, +}: { + label: string; + value: number; + max: number; + onChange: (value: number) => void; +}) { + const inputId = `score-${label.toLowerCase().replaceAll(" ", "-")}`; return ( - +
+ + {label} / {max} + + + onChange(Math.max(0, Math.min(max, Number(event.target.value) || 0))) + } + aria-label={label} + /> +
); } diff --git a/apps/app/app/(app)/[slug]/intelligence/page.tsx b/apps/app/app/(app)/[slug]/intelligence/page.tsx index 4cec32d6a..3085f4308 100644 --- a/apps/app/app/(app)/[slug]/intelligence/page.tsx +++ b/apps/app/app/(app)/[slug]/intelligence/page.tsx @@ -30,7 +30,8 @@ export default function IntelligencePage() { Intelligence - Review incoming commercial signals before they become CRM records or opportunities. + Review incoming commercial signals before they become CRM records or + opportunities. diff --git a/apps/app/components/app-icon-rail.tsx b/apps/app/components/app-icon-rail.tsx index f751a8a13..f5434c7cf 100644 --- a/apps/app/components/app-icon-rail.tsx +++ b/apps/app/components/app-icon-rail.tsx @@ -50,7 +50,12 @@ const ITEMS: RailItem[] = [ match: "prefix", related: ["/agents"], }, - { title: "Intelligence", href: "/intelligence", icon: Search, match: "prefix" }, + { + title: "Intelligence", + href: "/intelligence", + icon: Search, + match: "prefix", + }, { title: "Companies", href: "/companies", icon: Building, match: "prefix" }, { title: "Contacts", diff --git a/apps/app/components/crm/section-prefetch.ts b/apps/app/components/crm/section-prefetch.ts index e6eaf1cf2..9dd736fd9 100644 --- a/apps/app/components/crm/section-prefetch.ts +++ b/apps/app/components/crm/section-prefetch.ts @@ -29,7 +29,10 @@ export function usePrefetchSection(): (section: string) => void { return; case "/intelligence": void queryClient.prefetchQuery( - trpc.ingest.inbox.queryOptions({ status: "unresolved", limit: 100 }), + trpc.ingest.inbox.queryOptions({ + status: "unresolved", + limit: 100, + }), ); return; case "/companies": diff --git a/packages/db/prisma/migrations/20260902184536_sequencenow_crm_foundation/migration.sql b/packages/db/prisma/migrations/20260902184536_sequencenow_crm_foundation/migration.sql new file mode 100644 index 000000000..3427a8483 --- /dev/null +++ b/packages/db/prisma/migrations/20260902184536_sequencenow_crm_foundation/migration.sql @@ -0,0 +1,318 @@ +-- AlterTable +ALTER TABLE "agentTask" ADD COLUMN "businessUnitId" TEXT; + +-- AlterTable +ALTER TABLE "company" ADD COLUMN "businessUnitId" TEXT; + +-- AlterTable +ALTER TABLE "deal" ADD COLUMN "businessUnitId" TEXT; + +-- CreateTable +CREATE TABLE "business_unit" ( + "id" TEXT NOT NULL, + "key" TEXT NOT NULL, + "name" TEXT NOT NULL, + "description" TEXT, + "enabled" BOOLEAN NOT NULL DEFAULT true, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "business_unit_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "canonical_company" ( + "id" TEXT NOT NULL, + "businessUnitId" TEXT NOT NULL, + "name" TEXT NOT NULL, + "normalizedName" TEXT NOT NULL, + "domain" TEXT, + "phone" TEXT, + "website" TEXT, + "fields" JSONB, + "verifiedAt" TIMESTAMP(3), + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "canonical_company_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "person" ( + "id" TEXT NOT NULL, + "businessUnitId" TEXT NOT NULL, + "firstName" TEXT NOT NULL, + "lastName" TEXT, + "normalizedName" TEXT NOT NULL, + "email" TEXT, + "phone" TEXT, + "title" TEXT, + "fields" JSONB, + "verifiedAt" TIMESTAMP(3), + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "person_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "company_person" ( + "companyId" TEXT NOT NULL, + "personId" TEXT NOT NULL, + "role" TEXT, + "isPrimary" BOOLEAN NOT NULL DEFAULT false, + + CONSTRAINT "company_person_pkey" PRIMARY KEY ("companyId","personId") +); + +-- CreateTable +CREATE TABLE "pipeline" ( + "id" TEXT NOT NULL, + "businessUnitId" TEXT NOT NULL, + "name" TEXT NOT NULL, + "stages" JSONB NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "pipeline_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "canonical_opportunity" ( + "id" TEXT NOT NULL, + "businessUnitId" TEXT NOT NULL, + "companyId" TEXT, + "pipelineId" TEXT, + "name" TEXT NOT NULL, + "stage" TEXT NOT NULL, + "amount" DECIMAL(14,2), + "fields" JSONB, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "canonical_opportunity_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "source_record" ( + "id" TEXT NOT NULL, + "businessUnitId" TEXT NOT NULL, + "sourceSystem" TEXT NOT NULL, + "sourceType" TEXT NOT NULL, + "sourceId" TEXT NOT NULL, + "sourceUrl" TEXT, + "observedAt" TIMESTAMP(3), + "payload" JSONB, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "companyId" TEXT, + "personId" TEXT, + "opportunityId" TEXT, + + CONSTRAINT "source_record_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "record_mapping" ( + "id" TEXT NOT NULL, + "sourceSystem" TEXT NOT NULL, + "sourceType" TEXT NOT NULL, + "sourceId" TEXT NOT NULL, + "canonicalType" TEXT NOT NULL, + "canonicalId" TEXT NOT NULL, + "application" TEXT, + "applicationId" TEXT, + "matchMethod" TEXT NOT NULL, + "status" TEXT NOT NULL DEFAULT 'active', + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "record_mapping_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "campaign" ( + "id" TEXT NOT NULL, + "businessUnitId" TEXT NOT NULL, + "name" TEXT NOT NULL, + "status" TEXT NOT NULL DEFAULT 'draft', + "segment" JSONB, + "template" TEXT, + "sendingEnabled" BOOLEAN NOT NULL DEFAULT false, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "campaign_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "campaign_member" ( + "id" TEXT NOT NULL, + "campaignId" TEXT NOT NULL, + "companyId" TEXT, + "personId" TEXT, + "state" TEXT NOT NULL DEFAULT 'queued', + "draft" TEXT, + "sentAt" TIMESTAMP(3), + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "campaign_member_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "outreach_approval" ( + "id" TEXT NOT NULL, + "campaignId" TEXT NOT NULL, + "memberId" TEXT, + "state" TEXT NOT NULL DEFAULT 'pending', + "approvedBy" TEXT, + "approvedAt" TIMESTAMP(3), + "rejectionReason" TEXT, + "immutableHash" TEXT NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "outreach_approval_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "suppression_entry" ( + "id" TEXT NOT NULL, + "value" TEXT NOT NULL, + "normalized" TEXT NOT NULL, + "kind" TEXT NOT NULL, + "reason" TEXT, + "source" TEXT, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "suppression_entry_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "agent_run_context" ( + "id" TEXT NOT NULL, + "businessUnitId" TEXT NOT NULL, + "agentRunId" TEXT NOT NULL, + "budget" INTEGER NOT NULL DEFAULT 0, + "fitScores" JSONB, + "reasons" JSONB, + "evidenceCount" INTEGER NOT NULL DEFAULT 0, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "finishedAt" TIMESTAMP(3), + + CONSTRAINT "agent_run_context_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE UNIQUE INDEX "business_unit_key_key" ON "business_unit"("key"); + +-- CreateIndex +CREATE INDEX "business_unit_enabled_key_idx" ON "business_unit"("enabled", "key"); + +-- CreateIndex +CREATE INDEX "canonical_company_domain_idx" ON "canonical_company"("domain"); + +-- CreateIndex +CREATE UNIQUE INDEX "canonical_company_businessUnitId_normalizedName_key" ON "canonical_company"("businessUnitId", "normalizedName"); + +-- CreateIndex +CREATE INDEX "person_email_idx" ON "person"("email"); + +-- CreateIndex +CREATE INDEX "person_phone_idx" ON "person"("phone"); + +-- CreateIndex +CREATE UNIQUE INDEX "pipeline_businessUnitId_name_key" ON "pipeline"("businessUnitId", "name"); + +-- CreateIndex +CREATE INDEX "canonical_opportunity_businessUnitId_stage_idx" ON "canonical_opportunity"("businessUnitId", "stage"); + +-- CreateIndex +CREATE INDEX "source_record_businessUnitId_sourceType_idx" ON "source_record"("businessUnitId", "sourceType"); + +-- CreateIndex +CREATE UNIQUE INDEX "source_record_sourceSystem_sourceType_sourceId_key" ON "source_record"("sourceSystem", "sourceType", "sourceId"); + +-- CreateIndex +CREATE INDEX "record_mapping_canonicalType_canonicalId_idx" ON "record_mapping"("canonicalType", "canonicalId"); + +-- CreateIndex +CREATE UNIQUE INDEX "record_mapping_sourceSystem_sourceType_sourceId_canonicalTy_key" ON "record_mapping"("sourceSystem", "sourceType", "sourceId", "canonicalType"); + +-- CreateIndex +CREATE INDEX "campaign_businessUnitId_status_idx" ON "campaign"("businessUnitId", "status"); + +-- CreateIndex +CREATE INDEX "campaign_member_state_idx" ON "campaign_member"("state"); + +-- CreateIndex +CREATE UNIQUE INDEX "campaign_member_campaignId_companyId_personId_key" ON "campaign_member"("campaignId", "companyId", "personId"); + +-- CreateIndex +CREATE INDEX "outreach_approval_campaignId_state_idx" ON "outreach_approval"("campaignId", "state"); + +-- CreateIndex +CREATE UNIQUE INDEX "suppression_entry_normalized_kind_key" ON "suppression_entry"("normalized", "kind"); + +-- CreateIndex +CREATE INDEX "agent_run_context_businessUnitId_createdAt_idx" ON "agent_run_context"("businessUnitId", "createdAt"); + +-- CreateIndex +CREATE UNIQUE INDEX "agent_run_context_agentRunId_key" ON "agent_run_context"("agentRunId"); + +-- AddForeignKey +ALTER TABLE "company" ADD CONSTRAINT "company_businessUnitId_fkey" FOREIGN KEY ("businessUnitId") REFERENCES "business_unit"("id") ON DELETE SET NULL ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "agentTask" ADD CONSTRAINT "agentTask_businessUnitId_fkey" FOREIGN KEY ("businessUnitId") REFERENCES "business_unit"("id") ON DELETE SET NULL ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "deal" ADD CONSTRAINT "deal_businessUnitId_fkey" FOREIGN KEY ("businessUnitId") REFERENCES "business_unit"("id") ON DELETE SET NULL ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "canonical_company" ADD CONSTRAINT "canonical_company_businessUnitId_fkey" FOREIGN KEY ("businessUnitId") REFERENCES "business_unit"("id") ON DELETE RESTRICT ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "person" ADD CONSTRAINT "person_businessUnitId_fkey" FOREIGN KEY ("businessUnitId") REFERENCES "business_unit"("id") ON DELETE RESTRICT ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "company_person" ADD CONSTRAINT "company_person_companyId_fkey" FOREIGN KEY ("companyId") REFERENCES "canonical_company"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "company_person" ADD CONSTRAINT "company_person_personId_fkey" FOREIGN KEY ("personId") REFERENCES "person"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "pipeline" ADD CONSTRAINT "pipeline_businessUnitId_fkey" FOREIGN KEY ("businessUnitId") REFERENCES "business_unit"("id") ON DELETE RESTRICT ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "canonical_opportunity" ADD CONSTRAINT "canonical_opportunity_businessUnitId_fkey" FOREIGN KEY ("businessUnitId") REFERENCES "business_unit"("id") ON DELETE RESTRICT ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "canonical_opportunity" ADD CONSTRAINT "canonical_opportunity_companyId_fkey" FOREIGN KEY ("companyId") REFERENCES "canonical_company"("id") ON DELETE SET NULL ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "canonical_opportunity" ADD CONSTRAINT "canonical_opportunity_pipelineId_fkey" FOREIGN KEY ("pipelineId") REFERENCES "pipeline"("id") ON DELETE SET NULL ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "source_record" ADD CONSTRAINT "source_record_businessUnitId_fkey" FOREIGN KEY ("businessUnitId") REFERENCES "business_unit"("id") ON DELETE RESTRICT ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "source_record" ADD CONSTRAINT "source_record_companyId_fkey" FOREIGN KEY ("companyId") REFERENCES "canonical_company"("id") ON DELETE SET NULL ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "source_record" ADD CONSTRAINT "source_record_personId_fkey" FOREIGN KEY ("personId") REFERENCES "person"("id") ON DELETE SET NULL ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "source_record" ADD CONSTRAINT "source_record_opportunityId_fkey" FOREIGN KEY ("opportunityId") REFERENCES "canonical_opportunity"("id") ON DELETE SET NULL ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "campaign" ADD CONSTRAINT "campaign_businessUnitId_fkey" FOREIGN KEY ("businessUnitId") REFERENCES "business_unit"("id") ON DELETE RESTRICT ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "campaign_member" ADD CONSTRAINT "campaign_member_campaignId_fkey" FOREIGN KEY ("campaignId") REFERENCES "campaign"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "outreach_approval" ADD CONSTRAINT "outreach_approval_campaignId_fkey" FOREIGN KEY ("campaignId") REFERENCES "campaign"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "agent_run_context" ADD CONSTRAINT "agent_run_context_businessUnitId_fkey" FOREIGN KEY ("businessUnitId") REFERENCES "business_unit"("id") ON DELETE RESTRICT ON UPDATE CASCADE; diff --git a/packages/db/prisma/schema.prisma b/packages/db/prisma/schema.prisma index 82bc1f368..d315ed9ea 100644 --- a/packages/db/prisma/schema.prisma +++ b/packages/db/prisma/schema.prisma @@ -318,6 +318,9 @@ model Company { source RecordSource @default(MANUAL) + businessUnitId String? + businessUnit BusinessUnit? @relation("CompCompanyBusinessUnit", fields: [businessUnitId], references: [id], onDelete: SetNull) + lastActivityAt DateTime? archivedAt DateTime? @@ -487,6 +490,9 @@ model AgentTask { subject String? + businessUnitId String? + businessUnit BusinessUnit? @relation("CompAgentTaskBusinessUnit", fields: [businessUnitId], references: [id], onDelete: SetNull) + createdAt DateTime @default(now()) @@index([dueAt, leasedUntil]) @@ -923,6 +929,9 @@ model Deal { ownerId String owner User @relation("DealOwner", fields: [ownerId], references: [id]) + businessUnitId String? + businessUnit BusinessUnit? @relation("CompDealBusinessUnit", fields: [businessUnitId], references: [id], onDelete: SetNull) + stage DealStage @default(DEMO_BOOKED) stageChangedAt DateTime @default(now()) amount Decimal? @db.Decimal(14, 2) @@ -1583,3 +1592,249 @@ model Apikey { @@index([key]) @@map("apikey") } + +// Sequence Now foundation. These models are isolated from Comp AI's core +// records so canonical data and source provenance remain portable. +model BusinessUnit { + id String @id @default(uuid()) + key String @unique + name String + description String? + enabled Boolean @default(true) + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + compCompanies Company[] @relation("CompCompanyBusinessUnit") + compDeals Deal[] @relation("CompDealBusinessUnit") + compAgentTasks AgentTask[] @relation("CompAgentTaskBusinessUnit") + companies CanonicalCompany[] + people CanonicalPerson[] + opportunities CanonicalOpportunity[] + pipelines Pipeline[] + campaigns Campaign[] + agentRuns AgentRunContext[] + sourceRecords SourceRecord[] + + @@index([enabled, key]) + @@map("business_unit") +} + +model CanonicalCompany { + id String @id @default(uuid()) + businessUnitId String + businessUnit BusinessUnit @relation(fields: [businessUnitId], references: [id], onDelete: Restrict) + name String + normalizedName String + domain String? + phone String? + website String? + fields Json? + verifiedAt DateTime? + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + people CompanyPerson[] + opportunities CanonicalOpportunity[] + sources SourceRecord[] + + @@unique([businessUnitId, normalizedName]) + @@index([domain]) + @@map("canonical_company") +} + +model CanonicalPerson { + id String @id @default(uuid()) + businessUnitId String + businessUnit BusinessUnit @relation(fields: [businessUnitId], references: [id], onDelete: Restrict) + firstName String + lastName String? + normalizedName String + email String? + phone String? + title String? + fields Json? + verifiedAt DateTime? + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + companies CompanyPerson[] + sources SourceRecord[] + + @@index([email]) + @@index([phone]) + @@map("person") +} + +model CompanyPerson { + companyId String + personId String + company CanonicalCompany @relation(fields: [companyId], references: [id], onDelete: Cascade) + person CanonicalPerson @relation(fields: [personId], references: [id], onDelete: Cascade) + role String? + isPrimary Boolean @default(false) + + @@id([companyId, personId]) + @@map("company_person") +} + +model Pipeline { + id String @id @default(uuid()) + businessUnitId String + businessUnit BusinessUnit @relation(fields: [businessUnitId], references: [id], onDelete: Restrict) + name String + stages Json + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + opportunities CanonicalOpportunity[] + + @@unique([businessUnitId, name]) + @@map("pipeline") +} + +model CanonicalOpportunity { + id String @id @default(uuid()) + businessUnitId String + businessUnit BusinessUnit @relation(fields: [businessUnitId], references: [id], onDelete: Restrict) + companyId String? + company CanonicalCompany? @relation(fields: [companyId], references: [id], onDelete: SetNull) + pipelineId String? + pipeline Pipeline? @relation(fields: [pipelineId], references: [id], onDelete: SetNull) + name String + stage String + amount Decimal? @db.Decimal(14, 2) + fields Json? + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + sources SourceRecord[] + + @@index([businessUnitId, stage]) + @@map("canonical_opportunity") +} + +model SourceRecord { + id String @id @default(uuid()) + businessUnitId String + businessUnit BusinessUnit @relation(fields: [businessUnitId], references: [id], onDelete: Restrict) + sourceSystem String + sourceType String + sourceId String + sourceUrl String? + observedAt DateTime? + payload Json? + createdAt DateTime @default(now()) + + companyId String? + company CanonicalCompany? @relation(fields: [companyId], references: [id], onDelete: SetNull) + personId String? + person CanonicalPerson? @relation(fields: [personId], references: [id], onDelete: SetNull) + opportunityId String? + opportunity CanonicalOpportunity? @relation(fields: [opportunityId], references: [id], onDelete: SetNull) + + @@unique([sourceSystem, sourceType, sourceId]) + @@index([businessUnitId, sourceType]) + @@map("source_record") +} + +model RecordMapping { + id String @id @default(uuid()) + sourceSystem String + sourceType String + sourceId String + canonicalType String + canonicalId String + application String? + applicationId String? + matchMethod String + status String @default("active") + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + @@unique([sourceSystem, sourceType, sourceId, canonicalType]) + @@index([canonicalType, canonicalId]) + @@map("record_mapping") +} + +model Campaign { + id String @id @default(uuid()) + businessUnitId String + businessUnit BusinessUnit @relation(fields: [businessUnitId], references: [id], onDelete: Restrict) + name String + status String @default("draft") + segment Json? + template String? + sendingEnabled Boolean @default(false) + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + members CampaignMember[] + approvals OutreachApproval[] + + @@index([businessUnitId, status]) + @@map("campaign") +} + +model CampaignMember { + id String @id @default(uuid()) + campaignId String + campaign Campaign @relation(fields: [campaignId], references: [id], onDelete: Cascade) + companyId String? + personId String? + state String @default("queued") + draft String? + sentAt DateTime? + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + @@unique([campaignId, companyId, personId]) + @@index([state]) + @@map("campaign_member") +} + +model OutreachApproval { + id String @id @default(uuid()) + campaignId String + campaign Campaign @relation(fields: [campaignId], references: [id], onDelete: Cascade) + memberId String? + state String @default("pending") + approvedBy String? + approvedAt DateTime? + rejectionReason String? + immutableHash String + createdAt DateTime @default(now()) + + @@index([campaignId, state]) + @@map("outreach_approval") +} + +model SuppressionEntry { + id String @id @default(uuid()) + value String + normalized String + kind String + reason String? + source String? + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + @@unique([normalized, kind]) + @@map("suppression_entry") +} + +model AgentRunContext { + id String @id @default(uuid()) + businessUnitId String + businessUnit BusinessUnit @relation(fields: [businessUnitId], references: [id], onDelete: Restrict) + agentRunId String + budget Int @default(0) + fitScores Json? + reasons Json? + evidenceCount Int @default(0) + createdAt DateTime @default(now()) + finishedAt DateTime? + + @@unique([agentRunId]) + @@index([businessUnitId, createdAt]) + @@map("agent_run_context") +} From 85394dc6b2dff4912e8629862b12aed761f84365 Mon Sep 17 00:00:00 2001 From: Danny Date: Fri, 4 Sep 2026 02:26:19 -0400 Subject: [PATCH 26/33] fix: parse signal payloads at the API boundary --- apps/api/src/ingest/ingest.contracts.ts | 22 +++++++++- apps/api/src/ingest/ingest.service.ts | 26 ++++++------ .../ingest/signal-qualification.service.ts | 41 ++++++++++--------- 3 files changed, 57 insertions(+), 32 deletions(-) diff --git a/apps/api/src/ingest/ingest.contracts.ts b/apps/api/src/ingest/ingest.contracts.ts index 304866c11..29b55a1d8 100644 --- a/apps/api/src/ingest/ingest.contracts.ts +++ b/apps/api/src/ingest/ingest.contracts.ts @@ -1,6 +1,26 @@ import { z } from "zod"; -const signalPayload = z.record(z.string(), z.unknown()); +export type SignalPayloadValue = + | string + | number + | boolean + | null + | SignalPayloadValue[] + | { [key: string]: SignalPayloadValue }; + +const signalPayloadValue: z.ZodType = z.lazy(() => + z.union([ + z.string(), + z.number().finite(), + z.boolean(), + z.null(), + z.array(signalPayloadValue), + z.record(z.string(), signalPayloadValue), + ]), +); + +export const signalPayload = z.record(z.string(), signalPayloadValue); +export type SignalPayload = z.infer; export const ingestSignalInput = z.object({ project: z.string().trim().min(1).max(96), diff --git a/apps/api/src/ingest/ingest.service.ts b/apps/api/src/ingest/ingest.service.ts index bc0f50b13..52f025042 100644 --- a/apps/api/src/ingest/ingest.service.ts +++ b/apps/api/src/ingest/ingest.service.ts @@ -5,6 +5,7 @@ import { Injectable, NotFoundException, } from "@nestjs/common"; +import { z } from "zod"; import { CompaniesService } from "../companies/companies.service"; import { normalizeDomain } from "../companies/domain"; import { InjectDatabase } from "../database/database.constants"; @@ -16,7 +17,9 @@ import type { SignalCompanyCandidatesOutput, SignalInboxInput, SignalInboxOutput, + SignalPayload, } from "./ingest.contracts"; +import { signalPayload } from "./ingest.contracts"; type BusinessUnitRow = { id: string; enabled: boolean }; type SourceRecordRow = { id: string }; @@ -27,7 +30,7 @@ type SignalRow = { sourceSystem: string; sourceType: string; sourceId: string; - payload: Record; + payload: SignalPayload; }; type CandidateRow = { id: string; @@ -52,7 +55,7 @@ type InboxRow = { demandTrigger: string | null; nextAction: string | null; mapped: boolean; - payload: Record; + payload: SignalPayload; }; @Injectable() @@ -349,24 +352,23 @@ export class IngestService { throw new NotFoundException( `No source signal with id ${sourceRecordId}.`, ); - return row; + return { ...row, payload: signalPayload.parse(row.payload) }; } - private signalEntity(payload: Record): string | null { + private signalEntity(payload: SignalPayload): string | null { for (const key of ["entity", "company"] as const) { - const value = payload[key]; - if (typeof value === "string" && value.trim()) return value.trim(); + const value = z.string().trim().min(1).safeParse(payload[key]); + if (value.success) return value.data; } return null; } - private signalDomain(payload: Record): string | null { + private signalDomain(payload: SignalPayload): string | null { for (const key of ["domain", "website"] as const) { - const value = payload[key]; - if (typeof value === "string") { - const domain = normalizeDomain(value); - if (domain) return domain; - } + const value = z.string().safeParse(payload[key]); + if (!value.success) continue; + const domain = normalizeDomain(value.data); + if (domain) return domain; } return null; } diff --git a/apps/api/src/ingest/signal-qualification.service.ts b/apps/api/src/ingest/signal-qualification.service.ts index 172350448..c00241c4f 100644 --- a/apps/api/src/ingest/signal-qualification.service.ts +++ b/apps/api/src/ingest/signal-qualification.service.ts @@ -5,6 +5,7 @@ import { Injectable, NotFoundException, } from "@nestjs/common"; +import { z } from "zod"; import { InjectDatabase } from "../database/database.constants"; import { DealsService } from "../deals/deals.service"; import type { @@ -15,6 +16,11 @@ import type { SignalClassification, SignalScoreComponents, } from "./ingest.contracts"; +import { + type SignalPayload, + type SignalPayloadValue, + signalPayload, +} from "./ingest.contracts"; type SignalRow = { id: string; @@ -22,7 +28,7 @@ type SignalRow = { sourceSystem: string; sourceType: string; sourceId: string; - payload: Record; + payload: SignalPayload; }; type CompanyMapping = { @@ -203,10 +209,7 @@ export class SignalQualificationService { }; } - private score( - payload: Record, - components?: SignalScoreComponents, - ) { + private score(payload: SignalPayload, components?: SignalScoreComponents) { if (components) { const score = Math.round( components.icpMatch + @@ -255,7 +258,7 @@ export class SignalQualificationService { FROM source_record WHERE id = ${id} LIMIT 1 `; if (!row) throw new NotFoundException(`No source signal with id ${id}.`); - return row; + return { ...row, payload: signalPayload.parse(row.payload) }; } private async companyMapping( @@ -284,22 +287,22 @@ export class SignalQualificationService { return row ?? null; } - private object(value: unknown): Record | null { - return value && typeof value === "object" && !Array.isArray(value) - ? (value as Record) - : null; + private object(value: SignalPayloadValue | undefined): SignalPayload | null { + const parsed = signalPayload.safeParse(value); + return parsed.success ? parsed.data : null; } - private text(value: unknown): string | null { - return typeof value === "string" && value.trim() ? value.trim() : null; + private text(value: SignalPayloadValue | undefined): string | null { + const parsed = z.string().trim().min(1).safeParse(value); + return parsed.success ? parsed.data : null; } - private number(value: unknown): number | null { - if (typeof value === "number" && Number.isFinite(value)) return value; - if (typeof value === "string" && value.trim()) { - const parsed = Number(value); - return Number.isFinite(parsed) ? parsed : null; - } - return null; + private number(value: SignalPayloadValue | undefined): number | null { + const numeric = z.number().finite().safeParse(value); + if (numeric.success) return numeric.data; + const text = z.string().trim().min(1).safeParse(value); + if (!text.success) return null; + const parsed = Number(text.data); + return Number.isFinite(parsed) ? parsed : null; } } From bf73c363f63f14c595247f27590a5bbf49842592 Mon Sep 17 00:00:00 2001 From: Danny Date: Wed, 16 Sep 2026 14:10:02 -0400 Subject: [PATCH 27/33] Add local lead worker dry-run extraction --- apps/agent/agent/cli/local-lead-worker.ts | 72 +++++ apps/agent/agent/lib/local-lead-extraction.ts | 261 ++++++++++++++++++ apps/agent/agent/lib/local-lead-schema.ts | 31 +++ apps/agent/agent/lib/local-lead-staging.ts | 55 ++++ apps/agent/agent/lib/local-ollama.ts | 52 ++++ apps/agent/package.json | 1 + apps/agent/test/local-lead-extraction.spec.ts | 135 +++++++++ apps/agent/test/local-lead-staging.spec.ts | 56 ++++ 8 files changed, 663 insertions(+) create mode 100644 apps/agent/agent/cli/local-lead-worker.ts create mode 100644 apps/agent/agent/lib/local-lead-extraction.ts create mode 100644 apps/agent/agent/lib/local-lead-schema.ts create mode 100644 apps/agent/agent/lib/local-lead-staging.ts create mode 100644 apps/agent/agent/lib/local-ollama.ts create mode 100644 apps/agent/test/local-lead-extraction.spec.ts create mode 100644 apps/agent/test/local-lead-staging.spec.ts diff --git a/apps/agent/agent/cli/local-lead-worker.ts b/apps/agent/agent/cli/local-lead-worker.ts new file mode 100644 index 000000000..b332851d3 --- /dev/null +++ b/apps/agent/agent/cli/local-lead-worker.ts @@ -0,0 +1,72 @@ +import { + extractLocalLead, + fetchLocalLeadSource, +} from "../lib/local-lead-extraction"; +import { + defaultLocalLeadStagingPath, + stageLocalLead, +} from "../lib/local-lead-staging"; + +function valueAfter(args: string[], flag: string): string | undefined { + const index = args.indexOf(flag); + return index >= 0 ? args[index + 1] : undefined; +} + +function valueEquals(args: string[], flag: string): string | undefined { + const prefix = `${flag}=`; + return args.find((arg) => arg.startsWith(prefix))?.slice(prefix.length); +} + +function argument(args: string[], flag: string): string | undefined { + return valueEquals(args, flag) ?? valueAfter(args, flag); +} + +function assertDryRun(args: string[]): void { + const forbidden = args.filter((arg) => + ["--sync", "--sync-crm", "--production", "--neon"].includes( + arg.split("=", 1)[0] ?? "", + ), + ); + if (forbidden.length > 0) { + throw new Error( + `CRM synchronization is not implemented. Refused: ${forbidden.join(", ")}`, + ); + } +} + +async function main(): Promise { + const args = process.argv.slice(2); + assertDryRun(args); + + const text = argument(args, "--text"); + const url = argument(args, "--url"); + if ((text ? 1 : 0) + (url ? 1 : 0) !== 1) { + throw new Error("Provide exactly one of --text or --url."); + } + + const source = url + ? await fetchLocalLeadSource(url) + : { + sourceUrl: + argument(args, "--source-url") ?? "https://local.invalid/sample", + text: text as string, + }; + const lead = await extractLocalLead(source.text, source.sourceUrl); + const staged = await stageLocalLead( + lead, + argument(args, "--staging-path") ?? defaultLocalLeadStagingPath(), + ); + + console.log( + JSON.stringify( + { mode: "dry-run", extracted: lead, staging: staged }, + null, + 2, + ), + ); +} + +main().catch((error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; +}); diff --git a/apps/agent/agent/lib/local-lead-extraction.ts b/apps/agent/agent/lib/local-lead-extraction.ts new file mode 100644 index 000000000..629ff4776 --- /dev/null +++ b/apps/agent/agent/lib/local-lead-extraction.ts @@ -0,0 +1,261 @@ +import { createHash } from "node:crypto"; +import { lookup as resolveHostname } from "node:dns/promises"; +import { type LocalLead, localLeadSchema } from "./local-lead-schema"; +import { createLocalOllamaClient, type OllamaClient } from "./local-ollama"; + +export const LOCAL_LEAD_MODEL = "qwen2.5:14b"; +const DEFAULT_FETCH_TIMEOUT_MS = 15_000; +const DEFAULT_MAX_SOURCE_BYTES = 512_000; + +export type DnsLookup = ( + hostname: string, + options: { all: true; verbatim: true }, +) => Promise>; + +export type LocalLeadRecord = LocalLead & { + model_name: string; + latency_ms: number; + content_hash: string; + parser_status: "accepted"; + attempt_count: number; + created_at: string; +}; + +export function parseLocalLeadResponse( + raw: string, + sourceUrl: string, +): LocalLead { + const trimmed = raw.trim(); + if ( + trimmed.includes("```") || + !trimmed.startsWith("{") || + !trimmed.endsWith("}") + ) { + throw new Error("Model response is not a plain JSON object."); + } + + let value: unknown; + try { + value = JSON.parse(trimmed); + } catch { + throw new Error("Model response is invalid JSON."); + } + + const lead = localLeadSchema.parse(value); + const nullFields = [ + "company_name", + "website", + "contact_name", + "role", + "email", + "phone", + ] as const; + const expectedMissing = nullFields.filter((field) => lead[field] === null); + const actualMissing = [...lead.missing_fields].sort(); + const sortedExpected = [...expectedMissing].sort(); + if (JSON.stringify(actualMissing) !== JSON.stringify(sortedExpected)) { + throw new Error("missing_fields does not match null fields."); + } + + if (lead.source_url !== sourceUrl) { + throw new Error("source_url does not match the requested source."); + } + + for (const field of [ + "company_name", + "contact_name", + "email", + "phone", + ] as const) { + const valueForEvidence = lead[field]; + if ( + valueForEvidence && + !lead.evidence_excerpt + .toLocaleLowerCase() + .includes(valueForEvidence.toLocaleLowerCase()) + ) { + throw new Error(`${field} is not present in evidence_excerpt.`); + } + } + + return lead; +} + +export async function extractLocalLead( + text: string, + sourceUrl: string, + client: OllamaClient = createLocalOllamaClient(), +): Promise { + if (!text.trim()) throw new Error("Source text is empty."); + + const prompt = [ + "Return exactly one plain JSON object. Do not use markdown fences.", + "Use these keys exactly: company_name, website, contact_name, role, email, phone, source_url, evidence_excerpt, confidence_reason, missing_fields.", + "Use null for every unknown scalar field.", + "Use missing_fields for exactly the scalar fields that are null.", + "Never invent an email, phone number, or contact name.", + "Every non-null company_name, contact_name, email, and phone must appear verbatim in evidence_excerpt.", + `The source URL is ${sourceUrl}. Return that exact value as source_url.`, + "Page text:", + text, + ].join("\n"); + + const result = await client.generate({ model: LOCAL_LEAD_MODEL, prompt }); + const lead = parseLocalLeadResponse(result.text, sourceUrl); + const contentHash = createHash("sha256").update(text, "utf8").digest("hex"); + + return { + ...lead, + model_name: LOCAL_LEAD_MODEL, + latency_ms: result.latencyMs, + content_hash: contentHash, + parser_status: "accepted", + attempt_count: 1, + created_at: new Date().toISOString(), + }; +} + +export async function fetchLocalLeadSource( + url: string, + fetchImpl: typeof fetch = fetch, + options: { + timeoutMs?: number; + maxBytes?: number; + lookup?: DnsLookup; + } = {}, +): Promise<{ sourceUrl: string; text: string }> { + const parsed = new URL(url); + if (!["http:", "https:"].includes(parsed.protocol)) { + throw new Error("Source URL must use HTTP or HTTPS."); + } + if (isBlockedHostname(parsed.hostname)) { + throw new Error("Local source URLs are blocked."); + } + await assertPublicHostname( + parsed.hostname, + options.lookup ?? resolveHostname, + ); + + const response = await fetchImpl(url, { + signal: AbortSignal.timeout(options.timeoutMs ?? DEFAULT_FETCH_TIMEOUT_MS), + headers: { accept: "text/html,text/plain;q=0.9" }, + }); + if (!response.ok) throw new Error(`Source returned HTTP ${response.status}.`); + const maxBytes = options.maxBytes ?? DEFAULT_MAX_SOURCE_BYTES; + const contentLength = Number(response.headers.get("content-length")); + if (Number.isFinite(contentLength) && contentLength > maxBytes) { + throw new Error("Source exceeds the maximum content size."); + } + if (!response.body) throw new Error("Source returned no body."); + + const reader = response.body.getReader(); + const chunks: Uint8Array[] = []; + let size = 0; + try { + while (true) { + const next = await reader.read(); + if (next.done) break; + size += next.value.byteLength; + if (size > maxBytes) { + await reader.cancel(); + throw new Error("Source exceeds the maximum content size."); + } + chunks.push(next.value); + } + } finally { + reader.releaseLock(); + } + + const bytes = new Uint8Array(size); + let offset = 0; + for (const chunk of chunks) { + bytes.set(chunk, offset); + offset += chunk.byteLength; + } + return { sourceUrl: url, text: new TextDecoder().decode(bytes) }; +} + +async function assertPublicHostname( + hostname: string, + lookup: DnsLookup, +): Promise { + try { + const addresses = await lookup(hostname, { all: true, verbatim: true }); + if ( + addresses.length === 0 || + addresses.some((entry) => isBlockedHostname(entry.address)) + ) { + throw new Error("Source hostname resolves to a local address."); + } + } catch (error) { + if ( + error instanceof Error && + error.message === "Source hostname resolves to a local address." + ) { + throw error; + } + throw new Error("Source hostname DNS lookup failed."); + } +} + +function isBlockedHostname(hostname: string): boolean { + const host = hostname.toLocaleLowerCase().replace(/\.$/, ""); + const internalSuffixes = [ + ".localhost", + ".local", + ".internal", + ".lan", + ".home.arpa", + ]; + if ( + ["localhost", "internal", "intranet"].includes(host) || + internalSuffixes.some((suffix) => host.endsWith(suffix)) + ) { + return true; + } + + const parts = host.split(".").map(Number); + if (parts.length === 4 && parts.every(Number.isInteger)) { + const first = parts[0] ?? -1; + const second = parts[1] ?? -1; + return ( + first === 0 || + first === 10 || + first === 127 || + (first === 169 && second === 254) || + (first === 172 && second >= 16 && second <= 31) || + (first === 192 && second === 168) || + (first === 192 && second === 0) || + (first === 198 && second >= 18 && second <= 19) || + (first === 198 && second === 51) || + (first === 203 && second === 0) || + (first >= 224 && first <= 255) || + (first === 100 && second >= 64 && second <= 127) + ); + } + + if (host.includes(":")) { + return ( + host === "::" || + host.startsWith("ff") || + host.startsWith("2001:db8:") || + host.startsWith("::ffff:0.") || + host.startsWith("::ffff:100.") || + host.startsWith("::ffff:127.") || + host.startsWith("::ffff:169.254.") || + host.startsWith("::ffff:172.") || + host.startsWith("::ffff:192.0.") || + host === "::1" || + host.startsWith("fc") || + host.startsWith("fd") || + host.startsWith("fe8") || + host.startsWith("fe9") || + host.startsWith("fea") || + host.startsWith("feb") || + host.startsWith("::ffff:10.") || + host.startsWith("::ffff:192.168.") + ); + } + + return false; +} diff --git a/apps/agent/agent/lib/local-lead-schema.ts b/apps/agent/agent/lib/local-lead-schema.ts new file mode 100644 index 000000000..e6935b5f9 --- /dev/null +++ b/apps/agent/agent/lib/local-lead-schema.ts @@ -0,0 +1,31 @@ +import { z } from "zod"; + +export const localLeadSchema = z + .object({ + company_name: z.string().trim().min(1).nullable(), + website: z.string().trim().min(1).nullable(), + contact_name: z.string().trim().min(1).nullable(), + role: z.string().trim().min(1).nullable(), + email: z.string().trim().email().nullable(), + phone: z.string().trim().min(1).nullable(), + source_url: z.string().trim().url().nullable(), + evidence_excerpt: z.string().trim().min(1), + confidence_reason: z.string().trim().min(1), + missing_fields: z.array(z.string().trim().min(1)), + }) + .strict(); + +export type LocalLead = z.infer; + +export const LOCAL_LEAD_FIELDS = [ + "company_name", + "website", + "contact_name", + "role", + "email", + "phone", + "source_url", + "evidence_excerpt", + "confidence_reason", + "missing_fields", +] as const; diff --git a/apps/agent/agent/lib/local-lead-staging.ts b/apps/agent/agent/lib/local-lead-staging.ts new file mode 100644 index 000000000..f3db92b5b --- /dev/null +++ b/apps/agent/agent/lib/local-lead-staging.ts @@ -0,0 +1,55 @@ +import { appendFile, mkdir, readFile } from "node:fs/promises"; +import { basename, dirname, isAbsolute, relative, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import type { LocalLeadRecord } from "./local-lead-extraction"; + +const DEFAULT_STAGING_PATH = "var/local-leads/leads.jsonl"; +const LOCAL_AGENT_ROOT = resolve( + dirname(fileURLToPath(import.meta.url)), + "../..", +); + +export function defaultLocalLeadStagingPath(): string { + return resolve(LOCAL_AGENT_ROOT, DEFAULT_STAGING_PATH); +} + +export function assertLocalLeadStagingPath(path: string): string { + const resolved = resolve(LOCAL_AGENT_ROOT, path); + const allowed = resolve(LOCAL_AGENT_ROOT, "var/local-leads"); + const relativePath = relative(allowed, resolved); + if ( + !relativePath || + relativePath.startsWith("..") || + isAbsolute(relativePath) + ) { + throw new Error("Staging path must stay under var/local-leads."); + } + if (basename(resolved).startsWith(".env") || resolved.includes("/.git/")) { + throw new Error("Staging path cannot target configuration or Git files."); + } + return resolved; +} + +export async function stageLocalLead( + lead: LocalLeadRecord, + path = defaultLocalLeadStagingPath(), +): Promise<{ path: string; staged: boolean }> { + const safePath = assertLocalLeadStagingPath(path); + let existing = ""; + try { + existing = await readFile(safePath, "utf8"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + + for (const line of existing.split("\n")) { + if (!line.trim()) continue; + const row = JSON.parse(line) as { content_hash?: unknown }; + if (row.content_hash === lead.content_hash) + return { path: safePath, staged: false }; + } + + await mkdir(dirname(safePath), { recursive: true }); + await appendFile(safePath, `${JSON.stringify(lead)}\n`, "utf8"); + return { path: safePath, staged: true }; +} diff --git a/apps/agent/agent/lib/local-ollama.ts b/apps/agent/agent/lib/local-ollama.ts new file mode 100644 index 000000000..b29ef9f23 --- /dev/null +++ b/apps/agent/agent/lib/local-ollama.ts @@ -0,0 +1,52 @@ +const OLLAMA_URL = "http://127.0.0.1:11434/api/generate"; +const DEFAULT_TIMEOUT_MS = 120_000; + +export type OllamaGeneration = { + text: string; + latencyMs: number; +}; + +export type OllamaClient = { + generate(input: { + model: string; + prompt: string; + timeoutMs?: number; + }): Promise; +}; + +export function createLocalOllamaClient( + fetchImpl: typeof fetch = fetch, + endpoint = OLLAMA_URL, +): OllamaClient { + return { + async generate({ model, prompt, timeoutMs = DEFAULT_TIMEOUT_MS }) { + const startedAt = performance.now(); + const response = await fetchImpl(endpoint, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model, + prompt, + format: "json", + stream: false, + options: { temperature: 0 }, + }), + signal: AbortSignal.timeout(timeoutMs), + }); + + if (!response.ok) { + throw new Error(`Ollama returned HTTP ${response.status}.`); + } + + const body = (await response.json()) as { response?: unknown }; + if (typeof body.response !== "string") { + throw new Error("Ollama returned no text response."); + } + + return { + text: body.response, + latencyMs: Math.round(performance.now() - startedAt), + }; + }, + }; +} diff --git a/apps/agent/package.json b/apps/agent/package.json index 2360a75ad..16ea35377 100644 --- a/apps/agent/package.json +++ b/apps/agent/package.json @@ -9,6 +9,7 @@ "backfill:images": "bun scripts/backfill-brand-images.ts", "dev": "eve dev", "dev:headless": "eve dev --no-ui", + "local-lead:dry-run": "bun agent/cli/local-lead-worker.ts", "dispatch": "curl -fsS -X POST \"${AGENT_URL:-http://127.0.0.1:2000}/eve/v1/dev/schedules/dispatch\"", "build": "eve build", "start": "bun scripts/start.ts", diff --git a/apps/agent/test/local-lead-extraction.spec.ts b/apps/agent/test/local-lead-extraction.spec.ts new file mode 100644 index 000000000..fc2edc2a1 --- /dev/null +++ b/apps/agent/test/local-lead-extraction.spec.ts @@ -0,0 +1,135 @@ +import { describe, expect, it } from "bun:test"; +import { + extractLocalLead, + fetchLocalLeadSource, + parseLocalLeadResponse, +} from "../agent/lib/local-lead-extraction"; + +const sourceUrl = "https://example.com/acme"; +const valid = JSON.stringify({ + company_name: "Acme Field Systems", + website: "https://acmefield.example", + contact_name: "Jordan Lee", + role: "Head of Partnerships", + email: null, + phone: null, + source_url: sourceUrl, + evidence_excerpt: "Acme Field Systems. Jordan Lee — Head of Partnerships.", + confidence_reason: "The page states the company, contact, and role.", + missing_fields: ["email", "phone"], +}); + +describe("local lead extraction", () => { + it("accepts valid strict JSON", () => { + expect(parseLocalLeadResponse(valid, sourceUrl).contact_name).toBe( + "Jordan Lee", + ); + }); + + it("requires null for missing email and phone", () => { + const lead = parseLocalLeadResponse(valid, sourceUrl); + expect(lead.email).toBeNull(); + expect(lead.phone).toBeNull(); + }); + + it("rejects an invented-looking email without evidence", () => { + const value = JSON.parse(valid) as Record; + value.email = "jordan@example.com"; + value.missing_fields = ["phone"]; + expect(() => + parseLocalLeadResponse(JSON.stringify(value), sourceUrl), + ).toThrow("email is not present in evidence_excerpt"); + }); + + it("rejects markdown-wrapped JSON", () => { + expect(() => + parseLocalLeadResponse(`\`\`\`json\n${valid}\n\`\`\``, sourceUrl), + ).toThrow("plain JSON object"); + }); + + it("rejects invalid JSON", () => { + expect(() => parseLocalLeadResponse("{not-json}", sourceUrl)).toThrow( + "invalid JSON", + ); + }); + + it("requires an evidence excerpt", () => { + const value = JSON.parse(valid) as Record; + value.evidence_excerpt = ""; + expect(() => + parseLocalLeadResponse(JSON.stringify(value), sourceUrl), + ).toThrow(); + }); + + it("blocks private and internal source URLs before fetching", async () => { + const fetchImpl = async () => { + throw new Error("fetch must not run"); + }; + for (const url of [ + "http://127.0.0.1/private", + "http://192.168.1.10/private", + "http://service.internal/private", + "http://printer.local/private", + ]) { + await expect(fetchLocalLeadSource(url, fetchImpl)).rejects.toThrow( + "Local source URLs are blocked", + ); + } + }); + + it("enforces the source byte limit", async () => { + const response = new Response("small", { + status: 200, + headers: { "content-length": "20" }, + }); + expect( + fetchLocalLeadSource("https://example.com", async () => response, { + maxBytes: 4, + lookup: async () => [{ address: "93.184.216.34", family: 4 }], + }), + ).rejects.toThrow("maximum content size"); + }); + + it("rejects hostnames that resolve to local addresses", async () => { + const fetchImpl: typeof fetch = async () => { + throw new Error("fetch must not run"); + }; + for (const address of ["127.0.0.1", "10.0.0.4", "fd00::4"]) { + await expect( + fetchLocalLeadSource("https://public.example", fetchImpl, { + lookup: async () => [ + { address, family: address.includes(":") ? 6 : 4 }, + ], + }), + ).rejects.toThrow("local address"); + } + }); + + it("allows a hostname that resolves to a public address", async () => { + const source = await fetchLocalLeadSource( + "https://public.example", + async () => new Response("public page", { status: 200 }), + { lookup: async () => [{ address: "93.184.216.34", family: 4 }] }, + ); + expect(source.text).toBe("public page"); + }); + + it("rejects DNS lookup failure", async () => { + await expect( + fetchLocalLeadSource("https://public.example", fetch, { + lookup: async () => { + throw new Error("not found"); + }, + }), + ).rejects.toThrow("DNS lookup failed"); + }); + + it("does not require DATABASE_URL or CRM modules", async () => { + const result = await extractLocalLead( + "Acme Field Systems. Jordan Lee — Head of Partnerships.", + sourceUrl, + { generate: async () => ({ text: valid, latencyMs: 5 }) }, + ); + expect(result.parser_status).toBe("accepted"); + }); +}); diff --git a/apps/agent/test/local-lead-staging.spec.ts b/apps/agent/test/local-lead-staging.spec.ts new file mode 100644 index 000000000..745fdfbfc --- /dev/null +++ b/apps/agent/test/local-lead-staging.spec.ts @@ -0,0 +1,56 @@ +import { describe, expect, it } from "bun:test"; +import { randomUUID } from "node:crypto"; +import { rm } from "node:fs/promises"; +import { extractLocalLead } from "../agent/lib/local-lead-extraction"; +import { + defaultLocalLeadStagingPath, + stageLocalLead, +} from "../agent/lib/local-lead-staging"; + +const response = JSON.stringify({ + company_name: "Acme Field Systems", + website: "https://acmefield.example", + contact_name: "Jordan Lee", + role: "Head of Partnerships", + email: null, + phone: null, + source_url: "https://example.com/acme", + evidence_excerpt: "Acme Field Systems. Jordan Lee — Head of Partnerships.", + confidence_reason: "The page states the company, contact, and role.", + missing_fields: ["email", "phone"], +}); + +describe("local lead staging", () => { + it("anchors the default path inside apps/agent", () => { + expect(defaultLocalLeadStagingPath()).toEndWith( + "/apps/agent/var/local-leads/leads.jsonl", + ); + }); + + it("deduplicates by content hash", async () => { + const path = `var/local-leads/test-${randomUUID()}.jsonl`; + let stagedPath: string | undefined; + try { + const lead = await extractLocalLead( + "Acme Field Systems. Jordan Lee — Head of Partnerships.", + "https://example.com/acme", + { generate: async () => ({ text: response, latencyMs: 5 }) }, + ); + const first = await stageLocalLead(lead, path); + stagedPath = first.path; + expect(first.staged).toBe(true); + expect((await stageLocalLead(lead, path)).staged).toBe(false); + } finally { + if (stagedPath) await rm(stagedPath, { force: true }); + } + }); + + it("rejects paths outside local staging", async () => { + const lead = await extractLocalLead( + "Acme Field Systems. Jordan Lee — Head of Partnerships.", + "https://example.com/acme", + { generate: async () => ({ text: response, latencyMs: 5 }) }, + ); + expect(stageLocalLead(lead, ".env")).rejects.toThrow("var/local-leads"); + }); +}); From 6aa6e53ef0dc8a50b6e23a2c898a31658b49c6ec Mon Sep 17 00:00:00 2001 From: Danny Date: Wed, 16 Sep 2026 15:51:55 -0400 Subject: [PATCH 28/33] Add local lead campaign runner --- apps/agent/agent/cli/local-campaign-worker.ts | 90 ++++++++ apps/agent/agent/lib/local-campaign-cache.ts | 55 +++++ apps/agent/agent/lib/local-campaign-export.ts | 23 ++ .../agent/lib/local-campaign-extraction.ts | 93 ++++++++ .../agent/lib/local-campaign-lead-schema.ts | 29 +++ apps/agent/agent/lib/local-campaign-runner.ts | 153 +++++++++++++ apps/agent/agent/lib/local-campaign-schema.ts | 34 +++ .../agent/agent/lib/local-campaign-scoring.ts | 75 +++++++ .../agent/agent/lib/local-campaign-staging.ts | 113 ++++++++++ apps/agent/campaigns/README.md | 9 + apps/agent/campaigns/partwall-2027.json | 29 +++ apps/agent/package.json | 1 + apps/agent/test/local-campaign.spec.ts | 205 ++++++++++++++++++ 13 files changed, 909 insertions(+) create mode 100644 apps/agent/agent/cli/local-campaign-worker.ts create mode 100644 apps/agent/agent/lib/local-campaign-cache.ts create mode 100644 apps/agent/agent/lib/local-campaign-export.ts create mode 100644 apps/agent/agent/lib/local-campaign-extraction.ts create mode 100644 apps/agent/agent/lib/local-campaign-lead-schema.ts create mode 100644 apps/agent/agent/lib/local-campaign-runner.ts create mode 100644 apps/agent/agent/lib/local-campaign-schema.ts create mode 100644 apps/agent/agent/lib/local-campaign-scoring.ts create mode 100644 apps/agent/agent/lib/local-campaign-staging.ts create mode 100644 apps/agent/campaigns/README.md create mode 100644 apps/agent/campaigns/partwall-2027.json create mode 100644 apps/agent/test/local-campaign.spec.ts diff --git a/apps/agent/agent/cli/local-campaign-worker.ts b/apps/agent/agent/cli/local-campaign-worker.ts new file mode 100644 index 000000000..b51737910 --- /dev/null +++ b/apps/agent/agent/cli/local-campaign-worker.ts @@ -0,0 +1,90 @@ +import { writeFile } from "node:fs/promises"; +import { + localCampaignCsv, + localCampaignJsonl, +} from "../lib/local-campaign-export"; +import { + assertLocalCampaignOnly, + loadCampaignFile, + loadCampaignSeeds, + runLocalCampaign, +} from "../lib/local-campaign-runner"; +import { + assertLocalCampaignPath, + defaultLocalCampaignStagingPath, + readStagedLeads, +} from "../lib/local-campaign-staging"; + +function argument(args: string[], flag: string): string | undefined { + const index = args.indexOf(flag); + return ( + args + .find((value) => value.startsWith(`${flag}=`)) + ?.slice(flag.length + 1) ?? args[index + 1] + ); +} + +async function main(): Promise { + const args = process.argv.slice(2); + assertLocalCampaignOnly(args); + const command = args[0]; + if (command === "validate") { + const campaign = await loadCampaignFile(argument(args, "--campaign") ?? ""); + console.log( + JSON.stringify( + { valid: true, campaign_id: campaign.campaign_id }, + null, + 2, + ), + ); + return; + } + if (command === "run") { + const campaign = await loadCampaignFile(argument(args, "--campaign") ?? ""); + const seedFile = argument(args, "--seed-file"); + const seeds = seedFile + ? await loadCampaignSeeds(seedFile) + : campaign.seed_urls; + const result = await runLocalCampaign(campaign, seeds, { + stagingPath: argument(args, "--staging-path"), + }); + console.log(JSON.stringify({ mode: "dry-run", ...result }, null, 2)); + return; + } + if (command === "list") { + const path = + argument(args, "--staging-path") ?? defaultLocalCampaignStagingPath(); + console.log(JSON.stringify(await readStagedLeads(path), null, 2)); + return; + } + if (command === "export") { + const format = argument(args, "--format"); + const path = + argument(args, "--staging-path") ?? defaultLocalCampaignStagingPath(); + const leads = await readStagedLeads(path); + const output = argument(args, "--out"); + if (!output) throw new Error("Export needs --out."); + const safeOutput = assertLocalCampaignPath(output); + if (format === "jsonl") { + await writeFile(safeOutput, localCampaignJsonl(leads), "utf8"); + } else if (format === "csv") { + await writeFile(safeOutput, localCampaignCsv(leads), "utf8"); + } else { + throw new Error("Export format must be csv or jsonl."); + } + console.log( + JSON.stringify( + { exported: leads.length, output: safeOutput, format }, + null, + 2, + ), + ); + return; + } + throw new Error("Use validate, run, list, or export."); +} + +main().catch((error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; +}); diff --git a/apps/agent/agent/lib/local-campaign-cache.ts b/apps/agent/agent/lib/local-campaign-cache.ts new file mode 100644 index 000000000..0858988d4 --- /dev/null +++ b/apps/agent/agent/lib/local-campaign-cache.ts @@ -0,0 +1,55 @@ +import { createHash } from "node:crypto"; +import { appendFile, mkdir, readFile } from "node:fs/promises"; +import { dirname } from "node:path"; +import { fetchLocalLeadSource } from "./local-lead-extraction"; +import { defaultLocalLeadStagingPath } from "./local-lead-staging"; + +export type CachedPage = { + url: string; + content_hash: string; + text: string; + fetched_at: string; +}; + +const cachePath = () => + defaultLocalLeadStagingPath().replace(/leads\.jsonl$/, "pages.jsonl"); + +export async function readCachedPage( + url: string, + path = cachePath(), +): Promise { + let raw = ""; + try { + raw = await readFile(path, "utf8"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; + throw error; + } + for (const line of raw.split("\n")) { + if (!line.trim()) continue; + const row = JSON.parse(line) as CachedPage; + if (row.url === url) return row; + } + return null; +} + +export async function fetchCachedPage( + url: string, + fetchImpl: typeof fetch = fetch, + path = cachePath(), +): Promise<{ page: CachedPage; cached: boolean }> { + const existing = await readCachedPage(url, path); + if (existing) return { page: existing, cached: true }; + const source = await fetchLocalLeadSource(url, fetchImpl); + const page: CachedPage = { + url, + content_hash: createHash("sha256") + .update(source.text, "utf8") + .digest("hex"), + text: source.text, + fetched_at: new Date().toISOString(), + }; + await mkdir(dirname(path), { recursive: true }); + await appendFile(path, `${JSON.stringify(page)}\n`, "utf8"); + return { page, cached: false }; +} diff --git a/apps/agent/agent/lib/local-campaign-export.ts b/apps/agent/agent/lib/local-campaign-export.ts new file mode 100644 index 000000000..b13b74c1b --- /dev/null +++ b/apps/agent/agent/lib/local-campaign-export.ts @@ -0,0 +1,23 @@ +import type { LocalStagedLead } from "./local-campaign-staging"; + +type CsvValue = string | string[] | number | boolean | null | undefined; + +function csvValue(value: CsvValue): string { + const text = value === null || value === undefined ? "" : String(value); + return /[,"\n]/.test(text) ? `"${text.replaceAll('"', '""')}"` : text; +} + +export function localCampaignJsonl(leads: LocalStagedLead[]): string { + return `${leads.map((lead) => JSON.stringify(lead)).join("\n")}\n`; +} + +export function localCampaignCsv(leads: LocalStagedLead[]): string { + const headers = leads[0] ? Object.keys(leads[0]) : []; + const rows = [ + headers, + ...leads.map((lead) => + headers.map((header) => lead[header as keyof typeof lead]), + ), + ]; + return `${rows.map((row) => row.map(csvValue).join(",")).join("\n")}\n`; +} diff --git a/apps/agent/agent/lib/local-campaign-extraction.ts b/apps/agent/agent/lib/local-campaign-extraction.ts new file mode 100644 index 000000000..27e40e2c3 --- /dev/null +++ b/apps/agent/agent/lib/local-campaign-extraction.ts @@ -0,0 +1,93 @@ +import { createHash } from "node:crypto"; +import { + LOCAL_CAMPAIGN_SCALAR_FIELDS, + type LocalCampaignLead, + localCampaignLeadSchema, +} from "./local-campaign-lead-schema"; +import { LOCAL_LEAD_MODEL } from "./local-lead-extraction"; +import { createLocalOllamaClient, type OllamaClient } from "./local-ollama"; + +export const LOCAL_CAMPAIGN_MODEL = LOCAL_LEAD_MODEL; + +export type LocalCampaignLeadRecord = LocalCampaignLead & { + content_hash: string; + model_name: string; + latency_ms: number; + attempt_count: number; + created_at: string; +}; + +export function parseLocalCampaignResponse( + raw: string, + sourceUrl: string, +): LocalCampaignLead { + const trimmed = raw.trim(); + if ( + trimmed.includes("```") || + !trimmed.startsWith("{") || + !trimmed.endsWith("}") + ) { + throw new Error("Campaign model response is not plain JSON."); + } + + let value: unknown; + try { + value = JSON.parse(trimmed); + } catch { + throw new Error("Campaign model response is invalid JSON."); + } + + const lead = localCampaignLeadSchema.parse(value); + const expectedMissing = LOCAL_CAMPAIGN_SCALAR_FIELDS.filter( + (field) => lead[field] === null, + ).sort(); + const actualMissing = [...lead.missing_fields].sort(); + if (JSON.stringify(expectedMissing) !== JSON.stringify(actualMissing)) { + throw new Error("missing_fields does not match null fields."); + } + if (lead.source_url !== sourceUrl) { + throw new Error("source_url does not match the requested source."); + } + + for (const field of LOCAL_CAMPAIGN_SCALAR_FIELDS) { + const fieldValue = lead[field]; + if ( + fieldValue && + !lead.evidence_excerpt + .toLocaleLowerCase() + .includes(fieldValue.toLocaleLowerCase()) + ) { + throw new Error(`${field} is not present in evidence_excerpt.`); + } + } + return lead; +} + +export async function extractLocalCampaignLead( + text: string, + sourceUrl: string, + client: OllamaClient = createLocalOllamaClient(), +): Promise { + if (!text.trim()) throw new Error("Source text is empty."); + const prompt = [ + "Return exactly one plain JSON object. Do not use markdown fences.", + "Use exactly these keys: company_name, website, contact_name, role, email, phone, location, source_url, evidence_excerpt, project_fit_reason, missing_fields.", + "Use null for every unknown scalar field.", + "Use missing_fields for exactly the scalar fields that are null.", + "Never invent a company, contact, role, email, phone, website, or location.", + "Every non-null extracted scalar value must appear in evidence_excerpt.", + `The source URL is ${sourceUrl}. Return that exact value as source_url.`, + "Page text:", + text, + ].join("\n"); + const result = await client.generate({ model: LOCAL_CAMPAIGN_MODEL, prompt }); + const lead = parseLocalCampaignResponse(result.text, sourceUrl); + return { + ...lead, + content_hash: createHash("sha256").update(text, "utf8").digest("hex"), + model_name: LOCAL_CAMPAIGN_MODEL, + latency_ms: result.latencyMs, + attempt_count: 1, + created_at: new Date().toISOString(), + }; +} diff --git a/apps/agent/agent/lib/local-campaign-lead-schema.ts b/apps/agent/agent/lib/local-campaign-lead-schema.ts new file mode 100644 index 000000000..757170050 --- /dev/null +++ b/apps/agent/agent/lib/local-campaign-lead-schema.ts @@ -0,0 +1,29 @@ +import { z } from "zod"; + +export const localCampaignLeadSchema = z + .object({ + company_name: z.string().trim().min(1).nullable(), + website: z.string().trim().url().nullable(), + contact_name: z.string().trim().min(1).nullable(), + role: z.string().trim().min(1).nullable(), + email: z.string().trim().email().nullable(), + phone: z.string().trim().min(1).nullable(), + location: z.string().trim().min(1).nullable(), + source_url: z.string().trim().url(), + evidence_excerpt: z.string().trim().min(1).max(2_000), + project_fit_reason: z.string().trim().min(1).max(1_000), + missing_fields: z.array(z.string().trim().min(1)), + }) + .strict(); + +export type LocalCampaignLead = z.infer; + +export const LOCAL_CAMPAIGN_SCALAR_FIELDS = [ + "company_name", + "website", + "contact_name", + "role", + "email", + "phone", + "location", +] as const; diff --git a/apps/agent/agent/lib/local-campaign-runner.ts b/apps/agent/agent/lib/local-campaign-runner.ts new file mode 100644 index 000000000..d311eaf9b --- /dev/null +++ b/apps/agent/agent/lib/local-campaign-runner.ts @@ -0,0 +1,153 @@ +import { createHash } from "node:crypto"; +import { readFile } from "node:fs/promises"; +import { type CachedPage, fetchCachedPage } from "./local-campaign-cache"; +import { extractLocalCampaignLead } from "./local-campaign-extraction"; +import { + type LocalCampaign, + localCampaignSchema, + localCampaignSeedSchema, +} from "./local-campaign-schema"; +import { scoreLocalCampaignLead } from "./local-campaign-scoring"; +import { + type LocalStagedLead, + normalizeDomain, + stageCampaignLead, +} from "./local-campaign-staging"; +import type { OllamaClient } from "./local-ollama"; + +export function assertLocalCampaignOnly(args: string[]): void { + const forbidden = args.filter((arg) => + ["--sync", "--sync-crm", "--production", "--neon"].includes( + arg.split("=", 1)[0] ?? "", + ), + ); + if (forbidden.length > 0) { + throw new Error( + `CRM synchronization is not implemented. Refused: ${forbidden.join(", ")}`, + ); + } +} + +export type LocalCampaignRunDependencies = { + fetchImpl?: typeof fetch; + client?: OllamaClient; + pageLoader?: ( + url: string, + fetchImpl?: typeof fetch, + ) => Promise<{ + page: CachedPage; + cached: boolean; + }>; + stager?: typeof stageCampaignLead; + stagingPath?: string; +}; + +export type LocalCampaignRunResult = { + stagingPath: string; + processed: number; + cached: number; + staged: number; + duplicates: number; + leads: LocalStagedLead[]; +}; + +export async function loadCampaignFile(path: string): Promise { + return localCampaignSchema.parse(JSON.parse(await readFile(path, "utf8"))); +} + +export async function loadCampaignSeeds(path: string): Promise { + const raw = await readFile(path, "utf8"); + const urls: string[] = []; + if (path.toLocaleLowerCase().endsWith(".csv")) { + const lines = raw.split(/\r?\n/).filter((line) => line.trim()); + if (lines.some((line) => line.includes('"'))) { + throw new Error( + "Quoted CSV fields are not supported. Use JSONL for complex seed data.", + ); + } + const header = + lines + .shift() + ?.split(",") + .map((value) => value.trim()) ?? []; + const urlIndex = header.findIndex((value) => + ["url", "source_url"].includes(value), + ); + if (urlIndex < 0) + throw new Error("Seed CSV needs a url or source_url column."); + for (const line of lines) { + const value = line.split(",")[urlIndex]?.trim(); + if (value) urls.push(localCampaignSeedSchema.parse({ url: value }).url); + } + return urls; + } + for (const line of raw.split(/\r?\n/).filter((value) => value.trim())) { + const parsed: unknown = JSON.parse(line); + urls.push(localCampaignSeedSchema.parse(parsed).url); + } + return urls; +} + +export async function runLocalCampaign( + campaign: LocalCampaign, + seedUrls: string[] = campaign.seed_urls, + dependencies: LocalCampaignRunDependencies = {}, +): Promise { + const uniqueSeeds = [...new Set(seedUrls)]; + const domains = new Map(); + const leads: LocalStagedLead[] = []; + let cached = 0; + let staged = 0; + let duplicates = 0; + for (const url of uniqueSeeds) { + if ( + leads.filter((lead) => lead.status !== "duplicate").length >= + campaign.max_companies + ) + break; + const domain = normalizeDomain(new URL(url).hostname); + const pageCount = domains.get(domain) ?? 0; + if (pageCount >= campaign.max_pages_per_domain) continue; + domains.set(domain, pageCount + 1); + const loaded = await (dependencies.pageLoader ?? fetchCachedPage)( + url, + dependencies.fetchImpl, + ); + if (loaded.cached) cached += 1; + const extracted = await extractLocalCampaignLead( + loaded.page.text, + url, + dependencies.client, + ); + const scores = scoreLocalCampaignLead(extracted, campaign); + const missingRequired = campaign.required_fields.filter( + (field) => extracted[field as keyof typeof extracted] === null, + ); + const record: LocalStagedLead = { + ...extracted, + ...scores, + campaign_id: campaign.campaign_id, + lead_id: createHash("sha256") + .update(`${campaign.campaign_id}:${extracted.content_hash}`, "utf8") + .digest("hex"), + source_domain: domain, + status: missingRequired.length > 0 ? "needs_review" : "new", + }; + const result = await (dependencies.stager ?? stageCampaignLead)( + record, + dependencies.stagingPath, + ); + leads.push(result.staged); + if (result.duplicate) duplicates += 1; + else staged += 1; + } + return { + stagingPath: + dependencies.stagingPath ?? "var/local-leads/campaign-leads.jsonl", + processed: leads.length, + cached, + staged, + duplicates, + leads, + }; +} diff --git a/apps/agent/agent/lib/local-campaign-schema.ts b/apps/agent/agent/lib/local-campaign-schema.ts new file mode 100644 index 000000000..e61c8a299 --- /dev/null +++ b/apps/agent/agent/lib/local-campaign-schema.ts @@ -0,0 +1,34 @@ +import { z } from "zod"; + +export const localCampaignSchema = z + .object({ + campaign_id: z.string().trim().min(1), + project: z.string().trim().min(1), + target_customer_type: z.string().trim().min(1), + geography: z.array(z.string().trim().min(1)), + event_window: z + .object({ start: z.string().date(), end: z.string().date() }) + .strict() + .optional(), + source_types: z.array(z.enum(["seed_url", "manual_list", "search"])), + seed_urls: z.array(z.string().trim().url()), + keywords: z.array(z.string().trim().min(1)), + exclusions: z.array(z.string().trim().min(1)), + max_companies: z.number().int().positive().max(10_000), + max_pages_per_domain: z.number().int().positive().max(100), + required_fields: z.array(z.string().trim().min(1)), + scoring_rules: z + .object({ + fit_keyword_points: z.number().int().nonnegative().max(100), + evidence_points: z.number().int().nonnegative().max(100), + contact_field_points: z.number().int().nonnegative().max(100), + }) + .strict(), + }) + .strict(); + +export type LocalCampaign = z.infer; + +export const localCampaignSeedSchema = z.object({ + url: z.string().trim().url(), +}); diff --git a/apps/agent/agent/lib/local-campaign-scoring.ts b/apps/agent/agent/lib/local-campaign-scoring.ts new file mode 100644 index 000000000..7c81c63a8 --- /dev/null +++ b/apps/agent/agent/lib/local-campaign-scoring.ts @@ -0,0 +1,75 @@ +import type { LocalCampaignLead } from "./local-campaign-lead-schema"; +import type { LocalCampaign } from "./local-campaign-schema"; + +export type LocalCampaignScores = { + fit_score: number; + evidence_score: number; + contact_completeness_score: number; + outreach_readiness_score: number; + review_required_reason: string; +}; + +export function scoreLocalCampaignLead( + lead: LocalCampaignLead, + campaign: LocalCampaign, +): LocalCampaignScores { + const haystack = [ + lead.company_name, + lead.website, + lead.location, + lead.project_fit_reason, + lead.evidence_excerpt, + ] + .filter(Boolean) + .join(" ") + .toLocaleLowerCase(); + const matched = campaign.keywords.filter((keyword) => + haystack.includes(keyword.toLocaleLowerCase()), + ); + const excluded = campaign.exclusions.filter((term) => + haystack.includes(term.toLocaleLowerCase()), + ); + const fitScore = Math.max( + 0, + Math.min( + 100, + matched.length * campaign.scoring_rules.fit_keyword_points - + excluded.length * campaign.scoring_rules.fit_keyword_points, + ), + ); + const evidenceScore = Math.min( + 100, + campaign.scoring_rules.evidence_points + + (lead.evidence_excerpt.length >= 40 ? 20 : 0), + ); + const contactFields = [ + lead.contact_name, + lead.role, + lead.email, + lead.phone, + ].filter((value) => value !== null).length; + const contactCompletenessScore = Math.min( + 100, + contactFields * campaign.scoring_rules.contact_field_points, + ); + const outreachReadinessScore = Math.round( + (contactCompletenessScore + evidenceScore) / 2, + ); + const reasons = [ + matched.length === 0 ? "No campaign keyword appears in evidence." : null, + excluded.length > 0 + ? `Excluded term appears: ${excluded.join(", ")}.` + : null, + lead.email === null ? "Email is missing." : null, + lead.phone === null ? "Phone is missing." : null, + lead.contact_name === null ? "Contact name is missing." : null, + ].filter((reason): reason is string => reason !== null); + return { + fit_score: fitScore, + evidence_score: evidenceScore, + contact_completeness_score: contactCompletenessScore, + outreach_readiness_score: outreachReadinessScore, + review_required_reason: + reasons.join(" ") || "Evidence supports campaign fit.", + }; +} diff --git a/apps/agent/agent/lib/local-campaign-staging.ts b/apps/agent/agent/lib/local-campaign-staging.ts new file mode 100644 index 000000000..fd32b1355 --- /dev/null +++ b/apps/agent/agent/lib/local-campaign-staging.ts @@ -0,0 +1,113 @@ +import { appendFile, mkdir, readFile } from "node:fs/promises"; +import { basename, dirname, isAbsolute, relative, resolve } from "node:path"; +import type { LocalCampaignLeadRecord } from "./local-campaign-extraction"; +import type { LocalCampaignScores } from "./local-campaign-scoring"; + +export type LocalCampaignStatus = + | "new" + | "duplicate" + | "needs_review" + | "approved" + | "rejected"; + +export type LocalStagedLead = LocalCampaignLeadRecord & + LocalCampaignScores & { + campaign_id: string; + lead_id: string; + source_domain: string; + status: LocalCampaignStatus; + }; + +const AGENT_ROOT = resolve(dirname(new URL(import.meta.url).pathname), "../.."); +const DEFAULT_PATH = "var/local-leads/campaign-leads.jsonl"; + +export function defaultLocalCampaignStagingPath(): string { + return resolve(AGENT_ROOT, DEFAULT_PATH); +} + +export function assertLocalCampaignPath(path: string): string { + const root = resolve(AGENT_ROOT, "var/local-leads"); + const resolved = resolve(AGENT_ROOT, path); + const relativePath = relative(root, resolved); + if ( + !relativePath || + relativePath.startsWith("..") || + isAbsolute(relativePath) + ) { + throw new Error("Campaign path must stay under var/local-leads."); + } + if (basename(resolved).startsWith(".env") || resolved.includes("/.git/")) { + throw new Error("Campaign path cannot target configuration or Git files."); + } + return resolved; +} + +export async function readStagedLeads( + path = defaultLocalCampaignStagingPath(), +): Promise { + const safePath = assertLocalCampaignPath(path); + let raw = ""; + try { + raw = await readFile(safePath, "utf8"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return []; + throw error; + } + return raw + .split("\n") + .filter((line) => line.trim()) + .map((line) => JSON.parse(line) as LocalStagedLead); +} + +function dedupeKeys(lead: LocalStagedLead): string[] { + return [ + `domain:${normalizeDomain(lead.source_domain)}`, + lead.company_name ? `company:${normalizeText(lead.company_name)}` : null, + lead.email ? `email:${normalizeText(lead.email)}` : null, + lead.phone ? `phone:${normalizePhone(lead.phone)}` : null, + `hash:${lead.content_hash}`, + ].filter((key): key is string => key !== null); +} + +export function normalizeDomain(value: string): string { + return value + .toLocaleLowerCase() + .replace(/^www\./, "") + .replace(/\.$/, ""); +} + +function normalizeText(value: string): string { + return value + .toLocaleLowerCase() + .replace(/[^a-z0-9]+/g, " ") + .trim(); +} + +function normalizePhone(value: string): string { + return value.replace(/\D/g, ""); +} + +export function dedupeAgainst( + lead: LocalStagedLead, + existing: LocalStagedLead[], +): LocalStagedLead | null { + const keys = new Set(dedupeKeys(lead)); + return ( + existing.find((candidate) => + dedupeKeys(candidate).some((key) => keys.has(key)), + ) ?? null + ); +} + +export async function stageCampaignLead( + lead: LocalStagedLead, + path = defaultLocalCampaignStagingPath(), +): Promise<{ path: string; staged: LocalStagedLead; duplicate: boolean }> { + const safePath = assertLocalCampaignPath(path); + const existing = await readStagedLeads(safePath); + const duplicate = dedupeAgainst(lead, existing); + const staged = duplicate ? { ...lead, status: "duplicate" as const } : lead; + await mkdir(dirname(safePath), { recursive: true }); + await appendFile(safePath, `${JSON.stringify(staged)}\n`, "utf8"); + return { path: safePath, staged, duplicate: duplicate !== null }; +} diff --git a/apps/agent/campaigns/README.md b/apps/agent/campaigns/README.md new file mode 100644 index 000000000..27e8c8454 --- /dev/null +++ b/apps/agent/campaigns/README.md @@ -0,0 +1,9 @@ +# Local campaign files + +Campaign files use JSON. Seed files use JSONL with one object per line. + +Simple CSV files also work when they contain a `url` or `source_url` column. +Quoted CSV fields are rejected. JSONL is the preferred format for complex seed data. + +The campaign runner stores pages, leads, and exports under `var/local-leads`. +It never writes CRM or Neon data. diff --git a/apps/agent/campaigns/partwall-2027.json b/apps/agent/campaigns/partwall-2027.json new file mode 100644 index 000000000..93cab49a5 --- /dev/null +++ b/apps/agent/campaigns/partwall-2027.json @@ -0,0 +1,29 @@ +{ + "campaign_id": "partwall-trade-show-2027", + "project": "PartWall", + "target_customer_type": "trade show exhibitors and industrial display buyers", + "geography": ["United States", "Canada"], + "event_window": { + "start": "2027-03-01", + "end": "2027-06-30" + }, + "source_types": ["seed_url", "manual_list"], + "seed_urls": [], + "keywords": [ + "exhibitor", + "trade show", + "booth", + "display", + "industrial", + "manufacturer" + ], + "exclusions": ["job board", "press release only", "consumer retail"], + "max_companies": 25, + "max_pages_per_domain": 3, + "required_fields": ["company_name", "website", "evidence_excerpt"], + "scoring_rules": { + "fit_keyword_points": 15, + "evidence_points": 40, + "contact_field_points": 15 + } +} diff --git a/apps/agent/package.json b/apps/agent/package.json index 16ea35377..5c6ee54fd 100644 --- a/apps/agent/package.json +++ b/apps/agent/package.json @@ -9,6 +9,7 @@ "backfill:images": "bun scripts/backfill-brand-images.ts", "dev": "eve dev", "dev:headless": "eve dev --no-ui", + "local-campaign:dry-run": "bun agent/cli/local-campaign-worker.ts", "local-lead:dry-run": "bun agent/cli/local-lead-worker.ts", "dispatch": "curl -fsS -X POST \"${AGENT_URL:-http://127.0.0.1:2000}/eve/v1/dev/schedules/dispatch\"", "build": "eve build", diff --git a/apps/agent/test/local-campaign.spec.ts b/apps/agent/test/local-campaign.spec.ts new file mode 100644 index 000000000..2043e93f7 --- /dev/null +++ b/apps/agent/test/local-campaign.spec.ts @@ -0,0 +1,205 @@ +import { describe, expect, it } from "bun:test"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fetchCachedPage } from "../agent/lib/local-campaign-cache"; +import { + localCampaignCsv, + localCampaignJsonl, +} from "../agent/lib/local-campaign-export"; +import { parseLocalCampaignResponse } from "../agent/lib/local-campaign-extraction"; +import { + assertLocalCampaignOnly, + loadCampaignSeeds, + runLocalCampaign, +} from "../agent/lib/local-campaign-runner"; +import { localCampaignSchema } from "../agent/lib/local-campaign-schema"; +import { scoreLocalCampaignLead } from "../agent/lib/local-campaign-scoring"; +import { + assertLocalCampaignPath, + dedupeAgainst, + normalizeDomain, +} from "../agent/lib/local-campaign-staging"; + +const campaign = localCampaignSchema.parse({ + campaign_id: "test-campaign", + project: "PartWall", + target_customer_type: "exhibitor", + geography: ["United States"], + source_types: ["seed_url", "manual_list"], + seed_urls: ["https://example.com/one"], + keywords: ["exhibitor", "display"], + exclusions: ["consumer retail"], + max_companies: 2, + max_pages_per_domain: 1, + required_fields: ["company_name", "website", "evidence_excerpt"], + scoring_rules: { + fit_keyword_points: 20, + evidence_points: 40, + contact_field_points: 15, + }, +}); + +const leadJson = JSON.stringify({ + company_name: "Acme Displays", + website: "https://acme.example", + contact_name: null, + role: null, + email: null, + phone: null, + location: "Chicago", + source_url: "https://example.com/one", + evidence_excerpt: + "Acme Displays, https://acme.example, is an exhibitor display company in Chicago.", + project_fit_reason: "The exhibitor needs display equipment.", + missing_fields: ["contact_name", "role", "email", "phone"], +}); + +describe("local campaign runner", () => { + it("validates campaign definitions", () => { + expect(localCampaignSchema.safeParse(campaign).success).toBe(true); + }); + + it("loads CSV and JSONL seed lists", async () => { + const directory = await mkdtemp(join(tmpdir(), "local-campaign-")); + try { + const csv = join(directory, "seeds.csv"); + const jsonl = join(directory, "seeds.jsonl"); + await Bun.write(csv, "url\nhttps://example.com/a\n"); + await Bun.write(jsonl, '{"url":"https://example.com/b"}\n'); + expect(await loadCampaignSeeds(csv)).toEqual(["https://example.com/a"]); + expect(await loadCampaignSeeds(jsonl)).toEqual(["https://example.com/b"]); + } finally { + await rm(directory, { recursive: true, force: true }); + } + }); + + it("rejects quoted CSV fields and recommends JSONL", async () => { + const directory = await mkdtemp(join(tmpdir(), "local-campaign-csv-")); + try { + const csv = join(directory, "quoted.csv"); + await Bun.write(csv, 'url\n"https://example.com/a,b"\n'); + await expect(loadCampaignSeeds(csv)).rejects.toThrow("Use JSONL"); + } finally { + await rm(directory, { recursive: true, force: true }); + } + }); + + it("reuses cached pages and respects domain caps", async () => { + let loads = 0; + const result = await runLocalCampaign( + campaign, + ["https://example.com/one", "https://example.com/two"], + { + pageLoader: async () => { + loads += 1; + return { + cached: loads === 1, + page: { + url: "https://example.com/one", + content_hash: "hash", + text: "Acme Displays is an exhibitor display company in Chicago.", + fetched_at: new Date().toISOString(), + }, + }; + }, + client: { generate: async () => ({ text: leadJson, latencyMs: 4 }) }, + stager: async (lead, path) => ({ + path: path ?? "local", + staged: lead, + duplicate: false, + }), + }, + ); + expect(loads).toBe(1); + expect(result.cached).toBe(1); + }); + + it("caches fetched page content by URL", async () => { + const directory = await mkdtemp(join(tmpdir(), "local-campaign-cache-")); + try { + const path = join(directory, "pages.jsonl"); + let calls = 0; + const fetchImpl: typeof fetch = async () => { + calls += 1; + return new Response("cached page", { status: 200 }); + }; + const first = await fetchCachedPage( + "https://example.com/page", + fetchImpl, + path, + ); + const second = await fetchCachedPage( + "https://example.com/page", + fetchImpl, + path, + ); + expect(first.cached).toBe(false); + expect(second.cached).toBe(true); + expect(calls).toBe(1); + expect(second.page.content_hash).toBe(first.page.content_hash); + } finally { + await rm(directory, { recursive: true, force: true }); + } + }); + + it("rejects unsupported model output and preserves evidence rules", () => { + expect(() => + parseLocalCampaignResponse( + `\`\`\`json\n${leadJson}\n\`\`\``, + "https://example.com/one", + ), + ).toThrow(); + expect(() => + parseLocalCampaignResponse( + leadJson.replace("Acme Displays", "Invented Company"), + "https://example.com/one", + ), + ).toThrow("company_name is not present"); + }); + + it("scores from keywords, evidence, and fields", () => { + const lead = parseLocalCampaignResponse( + leadJson, + "https://example.com/one", + ); + const scores = scoreLocalCampaignLead(lead, campaign); + expect(scores.fit_score).toBeGreaterThan(0); + expect(scores.evidence_score).toBe(60); + expect(scores.contact_completeness_score).toBe(0); + expect(scores.review_required_reason).toContain("Email is missing."); + }); + + it("deduplicates normalized domain and exports JSONL and CSV", async () => { + expect(normalizeDomain("WWW.Example.com.")).toBe("example.com"); + const lead = { + ...parseLocalCampaignResponse(leadJson, "https://example.com/one"), + ...scoreLocalCampaignLead( + parseLocalCampaignResponse(leadJson, "https://example.com/one"), + campaign, + ), + campaign_id: campaign.campaign_id, + lead_id: "lead", + source_domain: "example.com", + status: "new" as const, + }; + expect(dedupeAgainst(lead, [lead])).not.toBeNull(); + const jsonl = localCampaignJsonl([lead]); + const csv = localCampaignCsv([lead]); + expect(JSON.parse(jsonl).lead_id).toBe("lead"); + expect(csv).toContain("campaign_id"); + }); + + it("refuses CRM synchronization flags before work", () => { + expect(() => assertLocalCampaignOnly(["run", "--sync-crm"])).toThrow(); + }); + + it("keeps staging and export paths under local lead storage", () => { + expect(assertLocalCampaignPath("var/local-leads/export.csv")).toEndWith( + "/apps/agent/var/local-leads/export.csv", + ); + expect(() => assertLocalCampaignPath("../../outside.csv")).toThrow( + "var/local-leads", + ); + }); +}); From 8448b63f87a70dd05992fab99b6794a53e366ee5 Mon Sep 17 00:00:00 2001 From: Danny Date: Wed, 16 Sep 2026 17:07:25 -0400 Subject: [PATCH 29/33] Fix local campaign failure handling --- apps/agent/agent/lib/local-campaign-runner.ts | 101 ++++++++++++++---- apps/agent/test/local-campaign.spec.ts | 39 +++++++ 2 files changed, 122 insertions(+), 18 deletions(-) diff --git a/apps/agent/agent/lib/local-campaign-runner.ts b/apps/agent/agent/lib/local-campaign-runner.ts index d311eaf9b..06b59e249 100644 --- a/apps/agent/agent/lib/local-campaign-runner.ts +++ b/apps/agent/agent/lib/local-campaign-runner.ts @@ -1,6 +1,7 @@ import { createHash } from "node:crypto"; import { readFile } from "node:fs/promises"; import { type CachedPage, fetchCachedPage } from "./local-campaign-cache"; +import type { LocalCampaignLeadRecord } from "./local-campaign-extraction"; import { extractLocalCampaignLead } from "./local-campaign-extraction"; import { type LocalCampaign, @@ -45,12 +46,36 @@ export type LocalCampaignRunDependencies = { export type LocalCampaignRunResult = { stagingPath: string; processed: number; + fetched: number; cached: number; + rejected_by_url_safety: number; + failed_fetch: number; + extracted: number; + extraction_failures: number; staged: number; duplicates: number; leads: LocalStagedLead[]; + failures: Array<{ + url: string; + stage: "url_safety" | "fetch" | "extraction" | "staging"; + reason: string; + }>; }; +function errorReason(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +function isUrlSafetyError(error: unknown): boolean { + const reason = errorReason(error); + return [ + "Local source URLs are blocked", + "local address", + "DNS lookup failed", + "Source URL must use HTTP or HTTPS", + ].some((marker) => reason.includes(marker)); +} + export async function loadCampaignFile(path: string): Promise { return localCampaignSchema.parse(JSON.parse(await readFile(path, "utf8"))); } @@ -96,9 +121,16 @@ export async function runLocalCampaign( const uniqueSeeds = [...new Set(seedUrls)]; const domains = new Map(); const leads: LocalStagedLead[] = []; + const failures: LocalCampaignRunResult["failures"] = []; + let fetched = 0; let cached = 0; + let rejectedByUrlSafety = 0; + let failedFetch = 0; + let extracted = 0; + let extractionFailures = 0; let staged = 0; let duplicates = 0; + let processed = 0; for (const url of uniqueSeeds) { if ( leads.filter((lead) => lead.status !== "duplicate").length >= @@ -109,34 +141,61 @@ export async function runLocalCampaign( const pageCount = domains.get(domain) ?? 0; if (pageCount >= campaign.max_pages_per_domain) continue; domains.set(domain, pageCount + 1); - const loaded = await (dependencies.pageLoader ?? fetchCachedPage)( - url, - dependencies.fetchImpl, - ); + processed += 1; + let loaded: Awaited< + ReturnType> + >; + try { + loaded = await (dependencies.pageLoader ?? fetchCachedPage)( + url, + dependencies.fetchImpl, + ); + } catch (error) { + const stage = isUrlSafetyError(error) ? "url_safety" : "fetch"; + if (stage === "url_safety") rejectedByUrlSafety += 1; + else failedFetch += 1; + failures.push({ url, stage, reason: errorReason(error) }); + continue; + } if (loaded.cached) cached += 1; - const extracted = await extractLocalCampaignLead( - loaded.page.text, - url, - dependencies.client, - ); - const scores = scoreLocalCampaignLead(extracted, campaign); + else fetched += 1; + let lead: LocalCampaignLeadRecord; + try { + lead = await extractLocalCampaignLead( + loaded.page.text, + url, + dependencies.client, + ); + extracted += 1; + } catch (error) { + extractionFailures += 1; + failures.push({ url, stage: "extraction", reason: errorReason(error) }); + continue; + } + const scores = scoreLocalCampaignLead(lead, campaign); const missingRequired = campaign.required_fields.filter( - (field) => extracted[field as keyof typeof extracted] === null, + (field) => lead[field as keyof typeof lead] === null, ); const record: LocalStagedLead = { - ...extracted, + ...lead, ...scores, campaign_id: campaign.campaign_id, lead_id: createHash("sha256") - .update(`${campaign.campaign_id}:${extracted.content_hash}`, "utf8") + .update(`${campaign.campaign_id}:${lead.content_hash}`, "utf8") .digest("hex"), source_domain: domain, status: missingRequired.length > 0 ? "needs_review" : "new", }; - const result = await (dependencies.stager ?? stageCampaignLead)( - record, - dependencies.stagingPath, - ); + let result: Awaited>; + try { + result = await (dependencies.stager ?? stageCampaignLead)( + record, + dependencies.stagingPath, + ); + } catch (error) { + failures.push({ url, stage: "staging", reason: errorReason(error) }); + continue; + } leads.push(result.staged); if (result.duplicate) duplicates += 1; else staged += 1; @@ -144,10 +203,16 @@ export async function runLocalCampaign( return { stagingPath: dependencies.stagingPath ?? "var/local-leads/campaign-leads.jsonl", - processed: leads.length, + processed, + fetched, cached, + rejected_by_url_safety: rejectedByUrlSafety, + failed_fetch: failedFetch, + extracted, + extraction_failures: extractionFailures, staged, duplicates, leads, + failures, }; } diff --git a/apps/agent/test/local-campaign.spec.ts b/apps/agent/test/local-campaign.spec.ts index 2043e93f7..f55ca19be 100644 --- a/apps/agent/test/local-campaign.spec.ts +++ b/apps/agent/test/local-campaign.spec.ts @@ -115,6 +115,45 @@ describe("local campaign runner", () => { expect(result.cached).toBe(1); }); + it("continues after an invalid extraction and reports the failure", async () => { + const secondUrl = "https://example.org/two"; + let calls = 0; + const result = await runLocalCampaign( + campaign, + ["https://example.com/one", secondUrl], + { + pageLoader: async (url) => ({ + cached: false, + page: { + url, + content_hash: url, + text: "A public exhibitor page.", + fetched_at: new Date().toISOString(), + }, + }), + client: { + generate: async () => ({ + text: + calls++ === 0 + ? '{"invalid":true}' + : leadJson.replace("https://example.com/one", secondUrl), + latencyMs: 4, + }), + }, + stager: async (lead, path) => ({ + path: path ?? "local", + staged: lead, + duplicate: false, + }), + }, + ); + expect(result.processed).toBe(2); + expect(result.extracted).toBe(1); + expect(result.extraction_failures).toBe(1); + expect(result.staged).toBe(1); + expect(result.failures[0]?.stage).toBe("extraction"); + }); + it("caches fetched page content by URL", async () => { const directory = await mkdtemp(join(tmpdir(), "local-campaign-cache-")); try { From e08d24eeb5496fb426e87795781121fa721c7d56 Mon Sep 17 00:00:00 2001 From: Danny Date: Wed, 16 Sep 2026 19:49:44 -0400 Subject: [PATCH 30/33] Add local campaign progress instrumentation --- apps/agent/agent/cli/local-campaign-worker.ts | 38 +- .../agent/lib/local-campaign-extraction.ts | 7 +- apps/agent/agent/lib/local-campaign-runner.ts | 358 ++++++++++++++++-- apps/agent/agent/lib/local-campaign-schema.ts | 25 +- .../agent/agent/lib/local-campaign-scoring.ts | 24 +- .../agent/lib/local-campaign-source-units.ts | 209 ++++++++++ .../agent/agent/lib/local-campaign-staging.ts | 3 + apps/agent/agent/lib/local-lead-extraction.ts | 44 ++- apps/agent/agent/lib/local-ollama.ts | 43 ++- apps/agent/test/local-campaign.spec.ts | 271 ++++++++++++- apps/agent/test/local-lead-extraction.spec.ts | 27 ++ 11 files changed, 982 insertions(+), 67 deletions(-) create mode 100644 apps/agent/agent/lib/local-campaign-source-units.ts diff --git a/apps/agent/agent/cli/local-campaign-worker.ts b/apps/agent/agent/cli/local-campaign-worker.ts index b51737910..408bab0ae 100644 --- a/apps/agent/agent/cli/local-campaign-worker.ts +++ b/apps/agent/agent/cli/local-campaign-worker.ts @@ -24,6 +24,16 @@ function argument(args: string[], flag: string): string | undefined { ); } +function numericArgument(args: string[], flag: string): number | undefined { + const value = argument(args, flag); + if (value === undefined) return undefined; + const parsed = Number(value); + if (!Number.isFinite(parsed) || parsed <= 0) { + throw new Error(`${flag} must be a positive number.`); + } + return Math.round(parsed); +} + async function main(): Promise { const args = process.argv.slice(2); assertLocalCampaignOnly(args); @@ -40,13 +50,39 @@ async function main(): Promise { return; } if (command === "run") { - const campaign = await loadCampaignFile(argument(args, "--campaign") ?? ""); + const baseCampaign = await loadCampaignFile( + argument(args, "--campaign") ?? "", + ); + const campaign = { + ...baseCampaign, + max_source_units_per_page: + numericArgument(args, "--max-source-units-per-page") ?? + baseCampaign.max_source_units_per_page, + max_source_unit_chars: + numericArgument(args, "--max-source-unit-chars") ?? + baseCampaign.max_source_unit_chars, + max_ollama_calls_per_seed: + numericArgument(args, "--max-ollama-calls-per-seed") ?? + baseCampaign.max_ollama_calls_per_seed, + max_total_ollama_calls: + numericArgument(args, "--max-total-ollama-calls") ?? + baseCampaign.max_total_ollama_calls, + per_ollama_call_timeout_ms: + numericArgument(args, "--per-ollama-call-timeout-ms") ?? + baseCampaign.per_ollama_call_timeout_ms, + }; const seedFile = argument(args, "--seed-file"); const seeds = seedFile ? await loadCampaignSeeds(seedFile) : campaign.seed_urls; + const progress = args.includes("--progress"); const result = await runLocalCampaign(campaign, seeds, { stagingPath: argument(args, "--staging-path"), + seedStatusPath: argument(args, "--seed-status-path"), + perSeedTimeoutMs: numericArgument(args, "--per-seed-timeout-ms"), + onProgress: progress + ? (event) => console.error(JSON.stringify(event)) + : undefined, }); console.log(JSON.stringify({ mode: "dry-run", ...result }, null, 2)); return; diff --git a/apps/agent/agent/lib/local-campaign-extraction.ts b/apps/agent/agent/lib/local-campaign-extraction.ts index 27e40e2c3..2a2e8f95f 100644 --- a/apps/agent/agent/lib/local-campaign-extraction.ts +++ b/apps/agent/agent/lib/local-campaign-extraction.ts @@ -67,6 +67,7 @@ export async function extractLocalCampaignLead( text: string, sourceUrl: string, client: OllamaClient = createLocalOllamaClient(), + options: { timeoutMs?: number } = {}, ): Promise { if (!text.trim()) throw new Error("Source text is empty."); const prompt = [ @@ -80,7 +81,11 @@ export async function extractLocalCampaignLead( "Page text:", text, ].join("\n"); - const result = await client.generate({ model: LOCAL_CAMPAIGN_MODEL, prompt }); + const result = await client.generate({ + model: LOCAL_CAMPAIGN_MODEL, + prompt, + timeoutMs: options.timeoutMs, + }); const lead = parseLocalCampaignResponse(result.text, sourceUrl); return { ...lead, diff --git a/apps/agent/agent/lib/local-campaign-runner.ts b/apps/agent/agent/lib/local-campaign-runner.ts index 06b59e249..02cb3dcac 100644 --- a/apps/agent/agent/lib/local-campaign-runner.ts +++ b/apps/agent/agent/lib/local-campaign-runner.ts @@ -1,5 +1,6 @@ import { createHash } from "node:crypto"; -import { readFile } from "node:fs/promises"; +import { appendFile, mkdir, readFile } from "node:fs/promises"; +import { dirname } from "node:path"; import { type CachedPage, fetchCachedPage } from "./local-campaign-cache"; import type { LocalCampaignLeadRecord } from "./local-campaign-extraction"; import { extractLocalCampaignLead } from "./local-campaign-extraction"; @@ -10,6 +11,13 @@ import { } from "./local-campaign-schema"; import { scoreLocalCampaignLead } from "./local-campaign-scoring"; import { + DEFAULT_LOCAL_CAMPAIGN_SOURCE_UNIT_CAPS, + extractLocalCampaignSourceUnits, + hasUsefulPageEvidence, + type LocalCampaignSourceUnit, +} from "./local-campaign-source-units"; +import { + assertLocalCampaignPath, type LocalStagedLead, normalizeDomain, stageCampaignLead, @@ -41,6 +49,44 @@ export type LocalCampaignRunDependencies = { }>; stager?: typeof stageCampaignLead; stagingPath?: string; + seedStatusPath?: string; + perSeedTimeoutMs?: number; + onProgress?: (event: LocalCampaignProgressEvent) => void; +}; + +export type LocalCampaignProgressEvent = { + event: + | "seed_started" + | "fetch_started" + | "fetch_completed" + | "fetch_failed" + | "cache_hit" + | "cache_miss" + | "source_units_found" + | "source_units_skipped" + | "ollama_call_started" + | "ollama_call_completed" + | "ollama_call_failed" + | "ollama_call_timed_out" + | "staged_lead_written" + | "seed_completed" + | "seed_failed"; + url: string; + elapsed_ms: number; + reason?: string; + source_unit_count?: number; + ollama_call_count?: number; +}; + +export type LocalCampaignSeedStatus = { + url: string; + status: "completed" | "failed" | "skipped"; + failure_reason: string | null; + elapsed_ms: number; + source_unit_count: number; + ollama_call_count: number; + staged_count: number; + created_at: string; }; export type LocalCampaignRunResult = { @@ -48,6 +94,8 @@ export type LocalCampaignRunResult = { processed: number; fetched: number; cached: number; + source_units: number; + ollama_calls: number; rejected_by_url_safety: number; failed_fetch: number; extracted: number; @@ -60,6 +108,7 @@ export type LocalCampaignRunResult = { stage: "url_safety" | "fetch" | "extraction" | "staging"; reason: string; }>; + seed_statuses: LocalCampaignSeedStatus[]; }; function errorReason(error: unknown): string { @@ -76,6 +125,61 @@ function isUrlSafetyError(error: unknown): boolean { ].some((marker) => reason.includes(marker)); } +function elapsedSince(startedAt: number): number { + return Math.round(performance.now() - startedAt); +} + +function isTimeoutReason(reason: string): boolean { + return reason.toLocaleLowerCase().includes("timed out"); +} + +async function writeSeedStatus( + status: LocalCampaignSeedStatus, + path?: string, +): Promise { + if (!path) return; + const safePath = assertLocalCampaignPath(path); + await mkdir(dirname(safePath), { recursive: true }); + await appendFile(safePath, `${JSON.stringify(status)}\n`, "utf8"); +} + +async function flushSeedStatus( + seedStatuses: LocalCampaignSeedStatus[], + status: LocalCampaignSeedStatus, + path?: string, +): Promise { + seedStatuses.push(status); + await writeSeedStatus(status, path); +} + +function extractionCandidates( + units: LocalCampaignSourceUnit[], + text: string, + url: string, + maxOllamaCallsPerSeed: number, + maxSourceUnitChars: number, +): Array<{ + text: string; + sourceUrl: string; + sourceUnit: LocalCampaignSourceUnit | null; +}> { + if (units.length > 0) { + return units.slice(0, maxOllamaCallsPerSeed).map((unit) => ({ + text: unit.candidate_text, + sourceUrl: unit.parent_source_url, + sourceUnit: unit, + })); + } + if (!hasUsefulPageEvidence(text)) return []; + return [ + { + text: text.slice(0, maxSourceUnitChars), + sourceUrl: url, + sourceUnit: null, + }, + ]; +} + export async function loadCampaignFile(path: string): Promise { return localCampaignSchema.parse(JSON.parse(await readFile(path, "utf8"))); } @@ -122,8 +226,11 @@ export async function runLocalCampaign( const domains = new Map(); const leads: LocalStagedLead[] = []; const failures: LocalCampaignRunResult["failures"] = []; + const seedStatuses: LocalCampaignSeedStatus[] = []; let fetched = 0; let cached = 0; + let sourceUnits = 0; + let ollamaCalls = 0; let rejectedByUrlSafety = 0; let failedFetch = 0; let extracted = 0; @@ -131,7 +238,52 @@ export async function runLocalCampaign( let staged = 0; let duplicates = 0; let processed = 0; + const maxSourceUnitsPerPage = + campaign.max_source_units_per_page ?? + DEFAULT_LOCAL_CAMPAIGN_SOURCE_UNIT_CAPS.max_source_units_per_page; + const maxSourceUnitChars = + campaign.max_source_unit_chars ?? + DEFAULT_LOCAL_CAMPAIGN_SOURCE_UNIT_CAPS.max_source_unit_chars; + const maxOllamaCallsPerSeed = campaign.max_ollama_calls_per_seed ?? 5; + const maxTotalOllamaCalls = campaign.max_total_ollama_calls ?? 50; + const perOllamaCallTimeoutMs = campaign.per_ollama_call_timeout_ms ?? 45_000; + const perSeedTimeoutMs = dependencies.perSeedTimeoutMs ?? 90_000; + const progress = dependencies.onProgress; for (const url of uniqueSeeds) { + const seedStartedAt = performance.now(); + let seedSourceUnitCount = 0; + let seedOllamaCalls = 0; + let seedStagedCount = 0; + const emit = ( + event: LocalCampaignProgressEvent["event"], + details: Omit< + LocalCampaignProgressEvent, + "event" | "url" | "elapsed_ms" + > = {}, + ) => { + progress?.({ + event, + url, + elapsed_ms: elapsedSince(seedStartedAt), + source_unit_count: seedSourceUnitCount, + ollama_call_count: seedOllamaCalls, + ...details, + }); + }; + const status = ( + value: LocalCampaignSeedStatus["status"], + failureReason: string | null, + ): LocalCampaignSeedStatus => ({ + url, + status: value, + failure_reason: failureReason, + elapsed_ms: elapsedSince(seedStartedAt), + source_unit_count: seedSourceUnitCount, + ollama_call_count: seedOllamaCalls, + staged_count: seedStagedCount, + created_at: new Date().toISOString(), + }); + emit("seed_started"); if ( leads.filter((lead) => lead.status !== "duplicate").length >= campaign.max_companies @@ -139,66 +291,189 @@ export async function runLocalCampaign( break; const domain = normalizeDomain(new URL(url).hostname); const pageCount = domains.get(domain) ?? 0; - if (pageCount >= campaign.max_pages_per_domain) continue; + if (pageCount >= campaign.max_pages_per_domain) { + const reason = "Domain page cap reached."; + emit("seed_failed", { reason }); + await flushSeedStatus( + seedStatuses, + status("skipped", reason), + dependencies.seedStatusPath, + ); + continue; + } domains.set(domain, pageCount + 1); processed += 1; let loaded: Awaited< ReturnType> >; try { + emit("fetch_started"); loaded = await (dependencies.pageLoader ?? fetchCachedPage)( url, dependencies.fetchImpl, ); + emit("fetch_completed"); } catch (error) { const stage = isUrlSafetyError(error) ? "url_safety" : "fetch"; + const reason = errorReason(error); if (stage === "url_safety") rejectedByUrlSafety += 1; else failedFetch += 1; - failures.push({ url, stage, reason: errorReason(error) }); - continue; - } - if (loaded.cached) cached += 1; - else fetched += 1; - let lead: LocalCampaignLeadRecord; - try { - lead = await extractLocalCampaignLead( - loaded.page.text, - url, - dependencies.client, + failures.push({ url, stage, reason }); + emit("fetch_failed", { reason }); + emit("seed_failed", { reason }); + await flushSeedStatus( + seedStatuses, + status("failed", reason), + dependencies.seedStatusPath, ); - extracted += 1; - } catch (error) { - extractionFailures += 1; - failures.push({ url, stage: "extraction", reason: errorReason(error) }); continue; } - const scores = scoreLocalCampaignLead(lead, campaign); - const missingRequired = campaign.required_fields.filter( - (field) => lead[field as keyof typeof lead] === null, + if (loaded.cached) { + cached += 1; + emit("cache_hit"); + } else { + fetched += 1; + emit("cache_miss"); + } + const units = extractLocalCampaignSourceUnits(loaded.page.text, url, { + max_source_units_per_page: maxSourceUnitsPerPage, + max_source_unit_chars: maxSourceUnitChars, + }); + seedSourceUnitCount = units.length; + sourceUnits += units.length; + emit("source_units_found", { source_unit_count: units.length }); + const candidates = extractionCandidates( + units, + loaded.page.text, + url, + maxOllamaCallsPerSeed, + maxSourceUnitChars, ); - const record: LocalStagedLead = { - ...lead, - ...scores, - campaign_id: campaign.campaign_id, - lead_id: createHash("sha256") - .update(`${campaign.campaign_id}:${lead.content_hash}`, "utf8") - .digest("hex"), - source_domain: domain, - status: missingRequired.length > 0 ? "needs_review" : "new", - }; - let result: Awaited>; - try { - result = await (dependencies.stager ?? stageCampaignLead)( - record, - dependencies.stagingPath, + if (units.length === 0 && candidates.length === 0) { + const reason = "No company-level source unit or useful page evidence."; + emit("source_units_skipped", { reason }); + emit("seed_completed", { reason }); + await flushSeedStatus( + seedStatuses, + status("completed", reason), + dependencies.seedStatusPath, ); - } catch (error) { - failures.push({ url, stage: "staging", reason: errorReason(error) }); continue; } - leads.push(result.staged); - if (result.duplicate) duplicates += 1; - else staged += 1; + if (units.length > candidates.length) { + emit("source_units_skipped", { + reason: "Source unit cap limited model calls.", + }); + } + let seedTimedOut = false; + for (const candidate of candidates) { + if (ollamaCalls >= maxTotalOllamaCalls) break; + if (elapsedSince(seedStartedAt) >= perSeedTimeoutMs) { + const reason = "Seed timed out."; + emit("seed_failed", { reason }); + await flushSeedStatus( + seedStatuses, + status("failed", reason), + dependencies.seedStatusPath, + ); + seedTimedOut = true; + break; + } + let lead: LocalCampaignLeadRecord; + try { + ollamaCalls += 1; + seedOllamaCalls += 1; + emit("ollama_call_started"); + lead = await extractLocalCampaignLead( + candidate.text, + candidate.sourceUrl, + dependencies.client, + { timeoutMs: perOllamaCallTimeoutMs }, + ); + emit("ollama_call_completed"); + extracted += 1; + } catch (error) { + const reason = errorReason(error); + extractionFailures += 1; + emit( + isTimeoutReason(reason) + ? "ollama_call_timed_out" + : "ollama_call_failed", + { + reason, + }, + ); + failures.push({ + url: candidate.sourceUrl, + stage: "extraction", + reason, + }); + continue; + } + const scores = scoreLocalCampaignLead(lead, campaign, { + sourceUnit: candidate.sourceUnit !== null, + extractionMethod: candidate.sourceUnit?.extraction_method ?? null, + }); + const missingRequired = campaign.required_fields.filter( + (field) => lead[field as keyof typeof lead] === null, + ); + const record: LocalStagedLead = { + ...lead, + ...scores, + campaign_id: campaign.campaign_id, + lead_id: createHash("sha256") + .update(`${campaign.campaign_id}:${lead.content_hash}`, "utf8") + .digest("hex"), + source_domain: domain, + source_unit_id: candidate.sourceUnit?.unit_id, + parent_source_url: candidate.sourceUnit?.parent_source_url, + extraction_method: candidate.sourceUnit?.extraction_method, + status: missingRequired.length > 0 ? "needs_review" : "new", + }; + let result: Awaited>; + try { + result = await (dependencies.stager ?? stageCampaignLead)( + record, + dependencies.stagingPath, + ); + } catch (error) { + const reason = errorReason(error); + failures.push({ + url: candidate.sourceUrl, + stage: "staging", + reason, + }); + continue; + } + seedStagedCount += 1; + emit("staged_lead_written"); + leads.push(result.staged); + if (result.duplicate) duplicates += 1; + else staged += 1; + if ( + leads.filter((lead) => lead.status !== "duplicate").length >= + campaign.max_companies + ) { + break; + } + } + if (!seedTimedOut && !seedStatuses.some((entry) => entry.url === url)) { + const reason = + seedStagedCount > 0 + ? null + : candidates.length > 0 + ? "No source unit produced an accepted staged lead." + : "No extraction candidates."; + emit(reason ? "seed_failed" : "seed_completed", { + reason: reason ?? undefined, + }); + await flushSeedStatus( + seedStatuses, + status(reason ? "failed" : "completed", reason), + dependencies.seedStatusPath, + ); + } + if (ollamaCalls >= maxTotalOllamaCalls) break; } return { stagingPath: @@ -206,6 +481,8 @@ export async function runLocalCampaign( processed, fetched, cached, + source_units: sourceUnits, + ollama_calls: ollamaCalls, rejected_by_url_safety: rejectedByUrlSafety, failed_fetch: failedFetch, extracted, @@ -214,5 +491,6 @@ export async function runLocalCampaign( duplicates, leads, failures, + seed_statuses: seedStatuses, }; } diff --git a/apps/agent/agent/lib/local-campaign-schema.ts b/apps/agent/agent/lib/local-campaign-schema.ts index e61c8a299..c372d4e24 100644 --- a/apps/agent/agent/lib/local-campaign-schema.ts +++ b/apps/agent/agent/lib/local-campaign-schema.ts @@ -16,6 +16,16 @@ export const localCampaignSchema = z exclusions: z.array(z.string().trim().min(1)), max_companies: z.number().int().positive().max(10_000), max_pages_per_domain: z.number().int().positive().max(100), + max_source_units_per_page: z.number().int().positive().max(500).optional(), + max_source_unit_chars: z.number().int().positive().max(10_000).optional(), + max_ollama_calls_per_seed: z.number().int().positive().max(100).optional(), + max_total_ollama_calls: z.number().int().positive().max(10_000).optional(), + per_ollama_call_timeout_ms: z + .number() + .int() + .positive() + .max(120_000) + .optional(), required_fields: z.array(z.string().trim().min(1)), scoring_rules: z .object({ @@ -29,6 +39,15 @@ export const localCampaignSchema = z export type LocalCampaign = z.infer; -export const localCampaignSeedSchema = z.object({ - url: z.string().trim().url(), -}); +export const localCampaignSeedSchema = z + .object({ + url: z.string().trim().url().optional(), + source_url: z.string().trim().url().optional(), + }) + .passthrough() + .refine((seed) => seed.url ?? seed.source_url, { + message: "Seed needs url or source_url.", + }) + .transform((seed): { url: string } => { + return { url: seed.url ?? seed.source_url ?? "" }; + }); diff --git a/apps/agent/agent/lib/local-campaign-scoring.ts b/apps/agent/agent/lib/local-campaign-scoring.ts index 7c81c63a8..6d25531a7 100644 --- a/apps/agent/agent/lib/local-campaign-scoring.ts +++ b/apps/agent/agent/lib/local-campaign-scoring.ts @@ -12,6 +12,7 @@ export type LocalCampaignScores = { export function scoreLocalCampaignLead( lead: LocalCampaignLead, campaign: LocalCampaign, + options: { sourceUnit?: boolean; extractionMethod?: string | null } = {}, ): LocalCampaignScores { const haystack = [ lead.company_name, @@ -37,10 +38,24 @@ export function scoreLocalCampaignLead( excluded.length * campaign.scoring_rules.fit_keyword_points, ), ); - const evidenceScore = Math.min( - 100, - campaign.scoring_rules.evidence_points + - (lead.evidence_excerpt.length >= 40 ? 20 : 0), + const sourceUnitBonus = options.sourceUnit ? 20 : 0; + const methodBonus = + options.extractionMethod === "table_row" || + options.extractionMethod === "card" + ? 10 + : 0; + const titleOnlyPenalty = + lead.company_name === null && lead.website === null ? 30 : 0; + const evidenceScore = Math.max( + 0, + Math.min( + 100, + campaign.scoring_rules.evidence_points + + (lead.evidence_excerpt.length >= 40 ? 20 : 0) + + sourceUnitBonus + + methodBonus - + titleOnlyPenalty, + ), ); const contactFields = [ lead.contact_name, @@ -63,6 +78,7 @@ export function scoreLocalCampaignLead( lead.email === null ? "Email is missing." : null, lead.phone === null ? "Phone is missing." : null, lead.contact_name === null ? "Contact name is missing." : null, + options.sourceUnit ? null : "Evidence is page-level.", ].filter((reason): reason is string => reason !== null); return { fit_score: fitScore, diff --git a/apps/agent/agent/lib/local-campaign-source-units.ts b/apps/agent/agent/lib/local-campaign-source-units.ts new file mode 100644 index 000000000..4be5ccd6d --- /dev/null +++ b/apps/agent/agent/lib/local-campaign-source-units.ts @@ -0,0 +1,209 @@ +import { createHash } from "node:crypto"; +import { normalizeDomain } from "./local-campaign-staging"; + +export type LocalCampaignSourceUnit = { + unit_id: string; + parent_source_url: string; + source_domain: string; + candidate_company_name: string | null; + candidate_website: string | null; + candidate_location: string | null; + candidate_text: string; + evidence_excerpt: string; + extraction_method: "table_row" | "list_item" | "card" | "link_context"; + content_hash: string; +}; + +export type LocalCampaignSourceUnitCaps = { + max_source_units_per_page: number; + max_source_unit_chars: number; +}; + +const COMPANY_SUFFIX = + /\b(inc|llc|ltd|limited|corp|corporation|co|company|group|systems|solutions|technologies|technology|manufacturing|industrial|industries|machinery|automation|electric|electronics|displays|fixtures|metal|plastics|tooling|logistics|labs)\b/i; +const WEBSITE_PATTERN = /\bhttps?:\/\/[^\s"'<>]+|\bwww\.[^\s"'<>]+/i; +const EMAIL_PATTERN = /\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/i; +const PHONE_PATTERN = /(?:\+?1[\s.-]?)?(?:\(?\d{3}\)?[\s.-]?)\d{3}[\s.-]?\d{4}/; +const BOILERPLATE = + /\b(cookie|privacy|terms|register now|buy tickets|subscribe|newsletter|sponsor|navigation|menu|login|sign in|venue|floor plan|organizer|copyright|all rights reserved)\b/i; +const EVENT_LEVEL = + /\b(exhibitor directory|exhibitor list|trade show|conference|expo|event|venue|registration|attendee|speaker)\b/i; +const BLOCK_PATTERNS = [ + { pattern: /]*>([\s\S]*?)<\/tr>/gi, method: "table_row" as const }, + { pattern: /]*>([\s\S]*?)<\/li>/gi, method: "list_item" as const }, + { + pattern: + /<(?:article|section|div)\b[^>]*(?:class|id)=["'][^"']*(?:card|exhibitor|company|profile|listing|result)[^"']*["'][^>]*>([\s\S]*?)<\/(?:article|section|div)>/gi, + method: "card" as const, + }, +]; + +export const DEFAULT_LOCAL_CAMPAIGN_SOURCE_UNIT_CAPS: LocalCampaignSourceUnitCaps = + { + max_source_units_per_page: 20, + max_source_unit_chars: 1800, + }; + +export function extractLocalCampaignSourceUnits( + text: string, + parentSourceUrl: string, + caps: LocalCampaignSourceUnitCaps = DEFAULT_LOCAL_CAMPAIGN_SOURCE_UNIT_CAPS, +): LocalCampaignSourceUnit[] { + const sourceDomain = normalizeDomain(new URL(parentSourceUrl).hostname); + const units: LocalCampaignSourceUnit[] = []; + for (const entry of BLOCK_PATTERNS) { + for (const match of text.matchAll(entry.pattern)) { + const unit = buildSourceUnit( + match[1] ?? "", + parentSourceUrl, + sourceDomain, + entry.method, + caps.max_source_unit_chars, + ); + if (unit) units.push(unit); + } + } + for (const match of text.matchAll( + /]*href=["']([^"']+)["'][^>]*>([\s\S]*?)<\/a>/gi, + )) { + const href = normalizeWebsite(match[1] ?? "", parentSourceUrl); + const label = cleanText(match[2] ?? ""); + const block = [label, href].filter(Boolean).join(" "); + const unit = buildSourceUnit( + block, + parentSourceUrl, + sourceDomain, + "link_context", + caps.max_source_unit_chars, + href, + ); + if (unit) units.push(unit); + } + return dedupeSourceUnits(units).slice(0, caps.max_source_units_per_page); +} + +export function hasUsefulPageEvidence(text: string): boolean { + const cleaned = cleanText(text); + return looksCompanyLevel(cleaned) && !isOnlyEventLevel(cleaned); +} + +export function dedupeSourceUnits( + units: LocalCampaignSourceUnit[], +): LocalCampaignSourceUnit[] { + const seen = new Set(); + const result: LocalCampaignSourceUnit[] = []; + for (const unit of units) { + const key = [ + unit.candidate_website ? `site:${unit.candidate_website}` : null, + unit.candidate_company_name + ? `company:${unit.candidate_company_name.toLocaleLowerCase()}` + : null, + `hash:${unit.content_hash}`, + ] + .filter(Boolean) + .join("|"); + if (seen.has(key)) continue; + seen.add(key); + result.push(unit); + } + return result; +} + +function buildSourceUnit( + raw: string, + parentSourceUrl: string, + sourceDomain: string, + extractionMethod: LocalCampaignSourceUnit["extraction_method"], + maxChars: number, + websiteHint: string | null = null, +): LocalCampaignSourceUnit | null { + const cleaned = cleanText(raw); + if (!looksCompanyLevel(cleaned) || isOnlyEventLevel(cleaned)) return null; + const candidateText = cleaned.slice(0, maxChars); + const contentHash = createHash("sha256") + .update(`${parentSourceUrl}:${candidateText}`, "utf8") + .digest("hex"); + const company = candidateCompanyName(candidateText); + const website = + websiteHint ?? + candidateText.match(WEBSITE_PATTERN)?.[0]?.replace(/[),.]+$/, "") ?? + null; + const location = candidateLocation(candidateText); + return { + unit_id: contentHash.slice(0, 24), + parent_source_url: parentSourceUrl, + source_domain: sourceDomain, + candidate_company_name: company, + candidate_website: website, + candidate_location: location, + candidate_text: candidateText, + evidence_excerpt: candidateText.slice(0, 500), + extraction_method: extractionMethod, + content_hash: contentHash, + }; +} + +function looksCompanyLevel(text: string): boolean { + if (text.length < 20 || BOILERPLATE.test(text)) return false; + return ( + COMPANY_SUFFIX.test(text) || + WEBSITE_PATTERN.test(text) || + EMAIL_PATTERN.test(text) || + PHONE_PATTERN.test(text) + ); +} + +function isOnlyEventLevel(text: string): boolean { + return ( + EVENT_LEVEL.test(text) && + !COMPANY_SUFFIX.test(text) && + !WEBSITE_PATTERN.test(text) + ); +} + +function candidateCompanyName(text: string): string | null { + const segments = text + .split(/\s{2,}|[|•]/) + .map((value) => value.trim()) + .filter(Boolean); + return ( + segments.find( + (segment) => COMPANY_SUFFIX.test(segment) && segment.length <= 120, + ) ?? null + ); +} + +function candidateLocation(text: string): string | null { + const match = text.match( + /\b(?:[A-Z][a-z]+(?:\s+[A-Z][a-z]+)*,\s*)?[A-Z]{2}\b/, + ); + return match?.[0] ?? null; +} + +function normalizeWebsite( + value: string, + parentSourceUrl: string, +): string | null { + if (!value || value.startsWith("#") || value.startsWith("mailto:")) + return null; + try { + return new URL(value, parentSourceUrl).toString(); + } catch { + return null; + } +} + +function cleanText(value: string): string { + return value + .replace(//gi, " ") + .replace(//gi, " ") + .replace(/<[^>]+>/g, " ") + .replace(/ /g, " ") + .replace(/&/g, "&") + .replace(/</g, "<") + .replace(/>/g, ">") + .replace(/'/g, "'") + .replace(/"/g, '"') + .replace(/\s+/g, " ") + .trim(); +} diff --git a/apps/agent/agent/lib/local-campaign-staging.ts b/apps/agent/agent/lib/local-campaign-staging.ts index fd32b1355..20607a4b6 100644 --- a/apps/agent/agent/lib/local-campaign-staging.ts +++ b/apps/agent/agent/lib/local-campaign-staging.ts @@ -15,6 +15,9 @@ export type LocalStagedLead = LocalCampaignLeadRecord & campaign_id: string; lead_id: string; source_domain: string; + source_unit_id?: string; + parent_source_url?: string; + extraction_method?: string; status: LocalCampaignStatus; }; diff --git a/apps/agent/agent/lib/local-lead-extraction.ts b/apps/agent/agent/lib/local-lead-extraction.ts index 629ff4776..0d6bf0e47 100644 --- a/apps/agent/agent/lib/local-lead-extraction.ts +++ b/apps/agent/agent/lib/local-lead-extraction.ts @@ -6,6 +6,7 @@ import { createLocalOllamaClient, type OllamaClient } from "./local-ollama"; export const LOCAL_LEAD_MODEL = "qwen2.5:14b"; const DEFAULT_FETCH_TIMEOUT_MS = 15_000; const DEFAULT_MAX_SOURCE_BYTES = 512_000; +const DEFAULT_DNS_TIMEOUT_MS = 5_000; export type DnsLookup = ( hostname: string, @@ -121,6 +122,7 @@ export async function fetchLocalLeadSource( options: { timeoutMs?: number; maxBytes?: number; + dnsTimeoutMs?: number; lookup?: DnsLookup; } = {}, ): Promise<{ sourceUrl: string; text: string }> { @@ -134,12 +136,33 @@ export async function fetchLocalLeadSource( await assertPublicHostname( parsed.hostname, options.lookup ?? resolveHostname, + options.dnsTimeoutMs ?? DEFAULT_DNS_TIMEOUT_MS, ); - const response = await fetchImpl(url, { - signal: AbortSignal.timeout(options.timeoutMs ?? DEFAULT_FETCH_TIMEOUT_MS), - headers: { accept: "text/html,text/plain;q=0.9" }, + const controller = new AbortController(); + const timeoutMs = options.timeoutMs ?? DEFAULT_FETCH_TIMEOUT_MS; + let timeout: ReturnType | undefined; + const timeoutPromise = new Promise((_resolve, reject) => { + timeout = setTimeout(() => { + controller.abort(); + reject(new Error("Source fetch timed out.")); + }, timeoutMs); }); + let response: Response; + try { + response = await Promise.race([ + fetchImpl(url, { + signal: controller.signal, + headers: { accept: "text/html,text/plain;q=0.9" }, + }), + timeoutPromise, + ]); + } catch (error) { + if (controller.signal.aborted) throw new Error("Source fetch timed out."); + throw error; + } finally { + if (timeout) clearTimeout(timeout); + } if (!response.ok) throw new Error(`Source returned HTTP ${response.status}.`); const maxBytes = options.maxBytes ?? DEFAULT_MAX_SOURCE_BYTES; const contentLength = Number(response.headers.get("content-length")); @@ -178,9 +201,22 @@ export async function fetchLocalLeadSource( async function assertPublicHostname( hostname: string, lookup: DnsLookup, + timeoutMs: number, ): Promise { try { - const addresses = await lookup(hostname, { all: true, verbatim: true }); + let timeout: ReturnType | undefined; + const timeoutPromise = new Promise((_resolve, reject) => { + timeout = setTimeout( + () => reject(new Error("DNS lookup timed out.")), + timeoutMs, + ); + }); + const addresses = await Promise.race([ + lookup(hostname, { all: true, verbatim: true }), + timeoutPromise, + ]).finally(() => { + if (timeout) clearTimeout(timeout); + }); if ( addresses.length === 0 || addresses.some((entry) => isBlockedHostname(entry.address)) diff --git a/apps/agent/agent/lib/local-ollama.ts b/apps/agent/agent/lib/local-ollama.ts index b29ef9f23..8cd37d2d6 100644 --- a/apps/agent/agent/lib/local-ollama.ts +++ b/apps/agent/agent/lib/local-ollama.ts @@ -21,18 +21,39 @@ export function createLocalOllamaClient( return { async generate({ model, prompt, timeoutMs = DEFAULT_TIMEOUT_MS }) { const startedAt = performance.now(); - const response = await fetchImpl(endpoint, { - method: "POST", - headers: { "content-type": "application/json" }, - body: JSON.stringify({ - model, - prompt, - format: "json", - stream: false, - options: { temperature: 0 }, - }), - signal: AbortSignal.timeout(timeoutMs), + const controller = new AbortController(); + let timeout: ReturnType | undefined; + const timeoutPromise = new Promise((_resolve, reject) => { + timeout = setTimeout(() => { + controller.abort(); + reject(new Error("Ollama request timed out.")); + }, timeoutMs); }); + let response: Response; + try { + response = await Promise.race([ + fetchImpl(endpoint, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model, + prompt, + format: "json", + stream: false, + options: { temperature: 0 }, + }), + signal: controller.signal, + }), + timeoutPromise, + ]); + } catch (error) { + if (controller.signal.aborted) { + throw new Error("Ollama request timed out."); + } + throw error; + } finally { + if (timeout) clearTimeout(timeout); + } if (!response.ok) { throw new Error(`Ollama returned HTTP ${response.status}.`); diff --git a/apps/agent/test/local-campaign.spec.ts b/apps/agent/test/local-campaign.spec.ts index f55ca19be..5eeb0c60c 100644 --- a/apps/agent/test/local-campaign.spec.ts +++ b/apps/agent/test/local-campaign.spec.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "bun:test"; -import { mkdtemp, rm } from "node:fs/promises"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { fetchCachedPage } from "../agent/lib/local-campaign-cache"; @@ -15,11 +15,16 @@ import { } from "../agent/lib/local-campaign-runner"; import { localCampaignSchema } from "../agent/lib/local-campaign-schema"; import { scoreLocalCampaignLead } from "../agent/lib/local-campaign-scoring"; +import { + extractLocalCampaignSourceUnits, + hasUsefulPageEvidence, +} from "../agent/lib/local-campaign-source-units"; import { assertLocalCampaignPath, dedupeAgainst, normalizeDomain, } from "../agent/lib/local-campaign-staging"; +import { createLocalOllamaClient } from "../agent/lib/local-ollama"; const campaign = localCampaignSchema.parse({ campaign_id: "test-campaign", @@ -66,7 +71,7 @@ describe("local campaign runner", () => { const csv = join(directory, "seeds.csv"); const jsonl = join(directory, "seeds.jsonl"); await Bun.write(csv, "url\nhttps://example.com/a\n"); - await Bun.write(jsonl, '{"url":"https://example.com/b"}\n'); + await Bun.write(jsonl, '{"source_url":"https://example.com/b"}\n'); expect(await loadCampaignSeeds(csv)).toEqual(["https://example.com/a"]); expect(await loadCampaignSeeds(jsonl)).toEqual(["https://example.com/b"]); } finally { @@ -127,7 +132,7 @@ describe("local campaign runner", () => { page: { url, content_hash: url, - text: "A public exhibitor page.", + text: "Acme Displays LLC is an exhibitor display company in Chicago.", fetched_at: new Date().toISOString(), }, }), @@ -209,6 +214,228 @@ describe("local campaign runner", () => { expect(scores.review_required_reason).toContain("Email is missing."); }); + it("extracts company source units from table rows", () => { + const units = extractLocalCampaignSourceUnits( + "
Acme Displays LLChttps://acme.exampleChicago, IL
", + "https://example.com/exhibitors", + ); + expect(units).toHaveLength(1); + expect(units[0]?.extraction_method).toBe("table_row"); + expect(units[0]?.candidate_company_name).toContain("Acme Displays LLC"); + expect(units[0]?.candidate_website).toBe("https://acme.example"); + }); + + it("extracts company source units from cards and list items", () => { + const units = extractLocalCampaignSourceUnits( + [ + "

Fixture Systems Inc

Website

Display fixtures in Dallas, TX

", + "
  • Panel Technology Ltd https://panel.example exhibit walls
", + ].join(""), + "https://example.com/exhibitors", + ); + expect(units.map((unit) => unit.extraction_method)).toContain("card"); + expect(units.map((unit) => unit.extraction_method)).toContain("list_item"); + }); + + it("extracts company source units from links with surrounding text", () => { + const units = extractLocalCampaignSourceUnits( + "Booth Systems Corp", + "https://example.com/exhibitors", + ); + expect(units).toHaveLength(1); + expect(units[0]?.extraction_method).toBe("link_context"); + }); + + it("rejects title or venue only evidence and boilerplate", () => { + expect( + hasUsefulPageEvidence("Automate 2027 exhibitor directory at JI Expo"), + ).toBe(false); + expect( + extractLocalCampaignSourceUnits( + "
  • Cookie settings privacy navigation menu
    • ", + "https://example.com/exhibitors", + ), + ).toHaveLength(0); + }); + + it("caps and trims source units", () => { + const html = [ + "Acme Displays LLChttps://acme.exampleLong text ".concat( + "x".repeat(200), + "", + ), + "Beta Fixtures Inchttps://beta.example", + ].join(""); + const units = extractLocalCampaignSourceUnits( + html, + "https://example.com/exhibitors", + { max_source_units_per_page: 1, max_source_unit_chars: 80 }, + ); + expect(units).toHaveLength(1); + expect(units[0]?.candidate_text.length).toBeLessThanOrEqual(80); + }); + + it("dedupes source units before Ollama", () => { + const result = extractLocalCampaignSourceUnits( + [ + "Acme Displays LLChttps://acme.example", + "
  • Acme Displays LLC https://acme.example
    • ", + ].join(""), + "https://example.com/exhibitors", + ); + expect(result).toHaveLength(1); + }); + + it("avoids whole-page model calls when source units exist", async () => { + const calls: string[] = []; + const result = await runLocalCampaign( + { ...campaign, max_ollama_calls_per_seed: 2, max_companies: 5 }, + ["https://example.com/exhibitors"], + { + pageLoader: async (url) => ({ + cached: false, + page: { + url, + content_hash: "hash", + text: [ + "Acme Displays LLChttps://acme.exampleChicago, IL", + "Beta Fixtures Inchttps://beta.exampleAustin, TX", + ].join(""), + fetched_at: new Date().toISOString(), + }, + }), + client: { + generate: async ({ prompt }) => { + calls.push(prompt); + const sourceUrl = "https://example.com/exhibitors"; + return { + text: leadJson.replace("https://example.com/one", sourceUrl), + latencyMs: 4, + }; + }, + }, + stager: async (lead, path) => ({ + path: path ?? "local", + staged: lead, + duplicate: false, + }), + }, + ); + expect(result.source_units).toBe(2); + expect(result.ollama_calls).toBe(2); + expect(calls.every((prompt) => !prompt.includes(""))).toBe(true); + }); + + it("falls back to page-level extraction when no source units are found", async () => { + let calls = 0; + const result = await runLocalCampaign( + campaign, + ["https://example.com/one"], + { + pageLoader: async (url) => ({ + cached: false, + page: { + url, + content_hash: "hash", + text: "Acme Displays LLC is an exhibitor display company in Chicago.", + fetched_at: new Date().toISOString(), + }, + }), + client: { + generate: async () => { + calls += 1; + return { text: leadJson, latencyMs: 4 }; + }, + }, + stager: async (lead, path) => ({ + path: path ?? "local", + staged: lead, + duplicate: false, + }), + }, + ); + expect(calls).toBe(1); + expect(result.source_units).toBe(0); + expect(result.ollama_calls).toBe(1); + }); + + it("flushes per-seed failure status", async () => { + const directory = await mkdtemp(join(tmpdir(), "local-campaign-status-")); + const seedStatusPath = `var/local-leads/${directory.split("/").pop()}-status.jsonl`; + try { + const result = await runLocalCampaign( + campaign, + ["https://example.com/one"], + { + pageLoader: async () => { + throw new Error("Source fetch timed out."); + }, + seedStatusPath, + }, + ); + expect(result.seed_statuses).toHaveLength(1); + expect(result.seed_statuses[0]?.failure_reason).toBe( + "Source fetch timed out.", + ); + expect( + await readFile( + new URL(`../${seedStatusPath}`, import.meta.url), + "utf8", + ), + ).toContain("Source fetch timed out."); + } finally { + await rm(directory, { recursive: true, force: true }); + await rm(new URL(`../${seedStatusPath}`, import.meta.url), { + force: true, + }); + } + }); + + it("stops a seed when the per-seed timeout is reached", async () => { + const result = await runLocalCampaign( + { ...campaign, max_ollama_calls_per_seed: 2 }, + ["https://example.com/one"], + { + perSeedTimeoutMs: -1, + pageLoader: async (url) => ({ + cached: false, + page: { + url, + content_hash: "hash", + text: "Acme Displays LLC is an exhibitor display company in Chicago.", + fetched_at: new Date().toISOString(), + }, + }), + client: { + generate: async () => { + await new Promise((resolve) => setTimeout(resolve, 2)); + return { text: leadJson, latencyMs: 2 }; + }, + }, + stager: async (lead, path) => ({ + path: path ?? "local", + staged: lead, + duplicate: false, + }), + }, + ); + expect(result.seed_statuses[0]?.failure_reason).toBe("Seed timed out."); + expect(result.ollama_calls).toBe(0); + }); + + it("scores company-level source unit evidence above page-level evidence", () => { + const lead = parseLocalCampaignResponse( + leadJson, + "https://example.com/one", + ); + const pageScore = scoreLocalCampaignLead(lead, campaign); + const unitScore = scoreLocalCampaignLead(lead, campaign, { + sourceUnit: true, + extractionMethod: "table_row", + }); + expect(unitScore.evidence_score).toBeGreaterThan(pageScore.evidence_score); + }); + it("deduplicates normalized domain and exports JSONL and CSV", async () => { expect(normalizeDomain("WWW.Example.com.")).toBe("example.com"); const lead = { @@ -241,4 +468,42 @@ describe("local campaign runner", () => { "var/local-leads", ); }); + + it("keeps local campaign modules free of CRM writer imports", async () => { + const files = [ + "agent/lib/local-campaign-cache.ts", + "agent/lib/local-campaign-export.ts", + "agent/lib/local-campaign-extraction.ts", + "agent/lib/local-campaign-lead-schema.ts", + "agent/lib/local-campaign-runner.ts", + "agent/lib/local-campaign-schema.ts", + "agent/lib/local-campaign-scoring.ts", + "agent/lib/local-campaign-source-units.ts", + "agent/lib/local-campaign-staging.ts", + "agent/cli/local-campaign-worker.ts", + ]; + const forbidden = + /from\s+["'](?:@crm\/db|.*prisma.*|.*neon.*|.*database.*|.*outreach.*|.*email.*)["']/i; + for (const file of files) { + const text = await readFile( + new URL(`../${file}`, import.meta.url), + "utf8", + ); + expect(text).not.toMatch(forbidden); + } + }); + + it("times out stalled Ollama requests", async () => { + const client = createLocalOllamaClient( + async (_input, init) => + new Promise((_resolve, reject) => { + init?.signal?.addEventListener("abort", () => { + reject(new Error("aborted")); + }); + }), + ); + await expect( + client.generate({ model: "qwen2.5:14b", prompt: "{}", timeoutMs: 1 }), + ).rejects.toThrow("timed out"); + }); }); diff --git a/apps/agent/test/local-lead-extraction.spec.ts b/apps/agent/test/local-lead-extraction.spec.ts index fc2edc2a1..2cb9fafb7 100644 --- a/apps/agent/test/local-lead-extraction.spec.ts +++ b/apps/agent/test/local-lead-extraction.spec.ts @@ -114,6 +114,24 @@ describe("local lead extraction", () => { expect(source.text).toBe("public page"); }); + it("times out stalled source fetches", async () => { + await expect( + fetchLocalLeadSource( + "https://public.example", + async (_input, init) => + new Promise((_resolve, reject) => { + init?.signal?.addEventListener("abort", () => { + reject(new Error("aborted")); + }); + }), + { + lookup: async () => [{ address: "93.184.216.34", family: 4 }], + timeoutMs: 1, + }, + ), + ).rejects.toThrow("timed out"); + }); + it("rejects DNS lookup failure", async () => { await expect( fetchLocalLeadSource("https://public.example", fetch, { @@ -124,6 +142,15 @@ describe("local lead extraction", () => { ).rejects.toThrow("DNS lookup failed"); }); + it("rejects stalled DNS lookup", async () => { + await expect( + fetchLocalLeadSource("https://public.example", fetch, { + dnsTimeoutMs: 1, + lookup: async () => new Promise(() => {}), + }), + ).rejects.toThrow("DNS lookup failed"); + }); + it("does not require DATABASE_URL or CRM modules", async () => { const result = await extractLocalLead( "Acme Field Systems. Jordan Lee — Head of Partnerships.", From 1e53e61f9358bd57180b98e74a073cffb5bc9625 Mon Sep 17 00:00:00 2001 From: Danny Date: Wed, 16 Sep 2026 20:08:31 -0400 Subject: [PATCH 31/33] Tighten local lead extraction schema compliance --- .../agent/lib/local-campaign-extraction.ts | 98 ++++++++++-- apps/agent/agent/lib/local-campaign-runner.ts | 60 ++++++-- apps/agent/agent/lib/local-lead-extraction.ts | 8 +- apps/agent/test/local-campaign.spec.ts | 145 +++++++++++++++++- 4 files changed, 282 insertions(+), 29 deletions(-) diff --git a/apps/agent/agent/lib/local-campaign-extraction.ts b/apps/agent/agent/lib/local-campaign-extraction.ts index 2a2e8f95f..8541a322d 100644 --- a/apps/agent/agent/lib/local-campaign-extraction.ts +++ b/apps/agent/agent/lib/local-campaign-extraction.ts @@ -1,4 +1,5 @@ import { createHash } from "node:crypto"; +import { ZodError } from "zod"; import { LOCAL_CAMPAIGN_SCALAR_FIELDS, type LocalCampaignLead, @@ -20,6 +21,7 @@ export type LocalCampaignLeadRecord = LocalCampaignLead & { export function parseLocalCampaignResponse( raw: string, sourceUrl: string, + sourceText?: string, ): LocalCampaignLead { const trimmed = raw.trim(); if ( @@ -30,13 +32,23 @@ export function parseLocalCampaignResponse( throw new Error("Campaign model response is not plain JSON."); } - let value: unknown; + const value = parsePlainJsonObject(trimmed, "Campaign model response"); + return parseLocalCampaignValue(value, sourceUrl, sourceText); +} + +function parsePlainJsonObject(raw: string, label: string): unknown { try { - value = JSON.parse(trimmed); + return JSON.parse(raw); } catch { - throw new Error("Campaign model response is invalid JSON."); + throw new Error(`${label} is invalid JSON.`); } +} +function parseLocalCampaignValue( + value: unknown, + sourceUrl: string, + sourceText?: string, +): LocalCampaignLead { const lead = localCampaignLeadSchema.parse(value); const expectedMissing = LOCAL_CAMPAIGN_SCALAR_FIELDS.filter( (field) => lead[field] === null, @@ -48,6 +60,14 @@ export function parseLocalCampaignResponse( if (lead.source_url !== sourceUrl) { throw new Error("source_url does not match the requested source."); } + if ( + sourceText && + !sourceText + .toLocaleLowerCase() + .includes(lead.evidence_excerpt.toLocaleLowerCase()) + ) { + throw new Error("evidence_excerpt is not present in source text."); + } for (const field of LOCAL_CAMPAIGN_SCALAR_FIELDS) { const fieldValue = lead[field]; @@ -63,6 +83,42 @@ export function parseLocalCampaignResponse( return lead; } +function needsSchemaRepair(error: unknown): boolean { + return ( + error instanceof ZodError || + (error instanceof Error && + [ + "missing_fields does not match null fields.", + "evidence_excerpt is not present in source text.", + ].includes(error.message)) + ); +} + +function repairPrompt( + sourceUrl: string, + sourceText: string, + firstResponse: string, + reason: string, +): string { + return [ + "Your previous response was rejected by validation.", + "Return exactly one raw JSON object. Do not use markdown. Do not return an array.", + "Use exactly these keys: company_name, website, contact_name, role, email, phone, location, source_url, evidence_excerpt, project_fit_reason, missing_fields.", + "All unknown scalar fields must be null.", + "project_fit_reason must always be a non-empty string.", + "evidence_excerpt must always be a non-empty exact excerpt copied from the source text below.", + "If there is not enough evidence for company_name, website, contact_name, role, email, phone, or location, use null for that field.", + "Never invent a company, website, contact name, role, email, phone, or location.", + "missing_fields must exactly list scalar fields whose values are null.", + `source_url must be exactly ${sourceUrl}.`, + `Validation failure: ${reason}`, + "Rejected response:", + firstResponse, + "Source text:", + sourceText, + ].join("\n"); +} + export async function extractLocalCampaignLead( text: string, sourceUrl: string, @@ -71,14 +127,17 @@ export async function extractLocalCampaignLead( ): Promise { if (!text.trim()) throw new Error("Source text is empty."); const prompt = [ - "Return exactly one plain JSON object. Do not use markdown fences.", + "Return exactly one raw JSON object. Do not use markdown fences. Do not return an array.", "Use exactly these keys: company_name, website, contact_name, role, email, phone, location, source_url, evidence_excerpt, project_fit_reason, missing_fields.", - "Use null for every unknown scalar field.", - "Use missing_fields for exactly the scalar fields that are null.", + "Use null for every unknown scalar field: company_name, website, contact_name, role, email, phone, location.", + "project_fit_reason must always be a non-empty string.", + "evidence_excerpt must always be a non-empty exact excerpt copied from the source text.", + "Use missing_fields for exactly the scalar fields that are null. Do not include evidence_excerpt, project_fit_reason, source_url, or missing_fields in missing_fields.", "Never invent a company, contact, role, email, phone, website, or location.", "Every non-null extracted scalar value must appear in evidence_excerpt.", + "If there is not enough evidence for a lead, return null for unknown scalar fields, use an exact source excerpt as evidence_excerpt, and explain the missing evidence in project_fit_reason.", `The source URL is ${sourceUrl}. Return that exact value as source_url.`, - "Page text:", + "Source text:", text, ].join("\n"); const result = await client.generate({ @@ -86,13 +145,32 @@ export async function extractLocalCampaignLead( prompt, timeoutMs: options.timeoutMs, }); - const lead = parseLocalCampaignResponse(result.text, sourceUrl); + let lead: LocalCampaignLead; + let latencyMs = result.latencyMs; + let attemptCount = 1; + try { + lead = parseLocalCampaignResponse(result.text, sourceUrl, text); + } catch (error) { + if (!needsSchemaRepair(error)) throw error; + const retry = await client.generate({ + model: LOCAL_CAMPAIGN_MODEL, + prompt: repairPrompt(sourceUrl, text, result.text, errorReason(error)), + timeoutMs: options.timeoutMs, + }); + latencyMs += retry.latencyMs; + attemptCount = 2; + lead = parseLocalCampaignResponse(retry.text, sourceUrl, text); + } return { ...lead, content_hash: createHash("sha256").update(text, "utf8").digest("hex"), model_name: LOCAL_CAMPAIGN_MODEL, - latency_ms: result.latencyMs, - attempt_count: 1, + latency_ms: latencyMs, + attempt_count: attemptCount, created_at: new Date().toISOString(), }; } + +function errorReason(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} diff --git a/apps/agent/agent/lib/local-campaign-runner.ts b/apps/agent/agent/lib/local-campaign-runner.ts index 02cb3dcac..40deaa744 100644 --- a/apps/agent/agent/lib/local-campaign-runner.ts +++ b/apps/agent/agent/lib/local-campaign-runner.ts @@ -180,6 +180,34 @@ function extractionCandidates( ]; } +function lacksCompanyLevelEvidence( + lead: LocalCampaignLeadRecord, + sourceUnit: LocalCampaignSourceUnit | null, +): boolean { + return ( + sourceUnit !== null && lead.company_name === null && lead.website === null + ); +} + +function cacheCounters(wasCached: boolean): { + cached: number; + fetched: number; + event: "cache_hit" | "cache_miss"; +} { + if (wasCached) return { cached: 1, fetched: 0, event: "cache_hit" }; + return { cached: 0, fetched: 1, event: "cache_miss" }; +} + +function seedCompletionFailureReason( + stagedCount: number, + candidateCount: number, +): string | null { + if (stagedCount > 0) return null; + if (candidateCount > 0) + return "No source unit produced an accepted staged lead."; + return "No extraction candidates."; +} + export async function loadCampaignFile(path: string): Promise { return localCampaignSchema.parse(JSON.parse(await readFile(path, "utf8"))); } @@ -328,13 +356,10 @@ export async function runLocalCampaign( ); continue; } - if (loaded.cached) { - cached += 1; - emit("cache_hit"); - } else { - fetched += 1; - emit("cache_miss"); - } + const cache = cacheCounters(loaded.cached); + cached += cache.cached; + fetched += cache.fetched; + emit(cache.event); const units = extractLocalCampaignSourceUnits(loaded.page.text, url, { max_source_units_per_page: maxSourceUnitsPerPage, max_source_unit_chars: maxSourceUnitChars, @@ -410,6 +435,17 @@ export async function runLocalCampaign( }); continue; } + if (lacksCompanyLevelEvidence(lead, candidate.sourceUnit)) { + const reason = "Source unit did not produce company-level evidence."; + extractionFailures += 1; + emit("ollama_call_failed", { reason }); + failures.push({ + url: candidate.sourceUrl, + stage: "extraction", + reason, + }); + continue; + } const scores = scoreLocalCampaignLead(lead, campaign, { sourceUnit: candidate.sourceUnit !== null, extractionMethod: candidate.sourceUnit?.extraction_method ?? null, @@ -458,12 +494,10 @@ export async function runLocalCampaign( } } if (!seedTimedOut && !seedStatuses.some((entry) => entry.url === url)) { - const reason = - seedStagedCount > 0 - ? null - : candidates.length > 0 - ? "No source unit produced an accepted staged lead." - : "No extraction candidates."; + const reason = seedCompletionFailureReason( + seedStagedCount, + candidates.length, + ); emit(reason ? "seed_failed" : "seed_completed", { reason: reason ?? undefined, }); diff --git a/apps/agent/agent/lib/local-lead-extraction.ts b/apps/agent/agent/lib/local-lead-extraction.ts index 0d6bf0e47..cc02757ac 100644 --- a/apps/agent/agent/lib/local-lead-extraction.ts +++ b/apps/agent/agent/lib/local-lead-extraction.ts @@ -167,7 +167,9 @@ export async function fetchLocalLeadSource( const maxBytes = options.maxBytes ?? DEFAULT_MAX_SOURCE_BYTES; const contentLength = Number(response.headers.get("content-length")); if (Number.isFinite(contentLength) && contentLength > maxBytes) { - throw new Error("Source exceeds the maximum content size."); + throw new Error( + `Source exceeds the maximum content size of ${maxBytes} bytes.`, + ); } if (!response.body) throw new Error("Source returned no body."); @@ -181,7 +183,9 @@ export async function fetchLocalLeadSource( size += next.value.byteLength; if (size > maxBytes) { await reader.cancel(); - throw new Error("Source exceeds the maximum content size."); + throw new Error( + `Source exceeds the maximum content size of ${maxBytes} bytes.`, + ); } chunks.push(next.value); } diff --git a/apps/agent/test/local-campaign.spec.ts b/apps/agent/test/local-campaign.spec.ts index 5eeb0c60c..495a09ce2 100644 --- a/apps/agent/test/local-campaign.spec.ts +++ b/apps/agent/test/local-campaign.spec.ts @@ -7,7 +7,10 @@ import { localCampaignCsv, localCampaignJsonl, } from "../agent/lib/local-campaign-export"; -import { parseLocalCampaignResponse } from "../agent/lib/local-campaign-extraction"; +import { + extractLocalCampaignLead, + parseLocalCampaignResponse, +} from "../agent/lib/local-campaign-extraction"; import { assertLocalCampaignOnly, loadCampaignSeeds, @@ -132,14 +135,14 @@ describe("local campaign runner", () => { page: { url, content_hash: url, - text: "Acme Displays LLC is an exhibitor display company in Chicago.", + text: "Acme Displays, https://acme.example, is an exhibitor display company in Chicago.", fetched_at: new Date().toISOString(), }, }), client: { generate: async () => ({ text: - calls++ === 0 + calls++ < 2 ? '{"invalid":true}' : leadJson.replace("https://example.com/one", secondUrl), latencyMs: 4, @@ -202,6 +205,90 @@ describe("local campaign runner", () => { ).toThrow("company_name is not present"); }); + it("requires project_fit_reason to be a string", () => { + const value = JSON.parse(leadJson) as Record; + value.project_fit_reason = null; + expect(() => + parseLocalCampaignResponse( + JSON.stringify(value), + "https://example.com/one", + ), + ).toThrow("project_fit_reason"); + }); + + it("requires non-empty exact source evidence", () => { + const emptyEvidence = JSON.parse(leadJson) as Record; + emptyEvidence.evidence_excerpt = ""; + expect(() => + parseLocalCampaignResponse( + JSON.stringify(emptyEvidence), + "https://example.com/one", + "Acme Displays source text.", + ), + ).toThrow("evidence_excerpt"); + const wrongEvidence = JSON.parse(leadJson) as Record; + wrongEvidence.evidence_excerpt = + "Acme Displays, https://acme.example, is an exhibitor display company in Chicago."; + expect(() => + parseLocalCampaignResponse( + JSON.stringify(wrongEvidence), + "https://example.com/one", + "Different source text.", + ), + ).toThrow("evidence_excerpt is not present"); + }); + + it("retries schema-invalid model output once and accepts a repaired response", async () => { + const sourceText = + "Acme Displays, https://acme.example, is an exhibitor display company in Chicago."; + let calls = 0; + const result = await extractLocalCampaignLead( + sourceText, + "https://example.com/one", + { + generate: async () => { + calls += 1; + return { + text: + calls === 1 + ? JSON.stringify({ + ...JSON.parse(leadJson), + evidence_excerpt: "", + project_fit_reason: null, + }) + : leadJson, + latencyMs: 5, + }; + }, + }, + ); + expect(result.attempt_count).toBe(2); + expect(result.latency_ms).toBe(10); + }); + + it("rejects after one failed repair attempt with the Zod issue", async () => { + let calls = 0; + await expect( + extractLocalCampaignLead( + "Acme Displays, https://acme.example, is an exhibitor display company in Chicago.", + "https://example.com/one", + { + generate: async () => { + calls += 1; + return { + text: JSON.stringify({ + ...JSON.parse(leadJson), + project_fit_reason: null, + }), + latencyMs: 5, + }; + }, + }, + ), + ).rejects.toThrow("project_fit_reason"); + expect(calls).toBe(2); + }); + it("scores from keywords, evidence, and fields", () => { const lead = parseLocalCampaignResponse( leadJson, @@ -326,6 +413,56 @@ describe("local campaign runner", () => { expect(calls.every((prompt) => !prompt.includes(""))).toBe(true); }); + it("rejects source unit output without company-level evidence", async () => { + const thinLead = JSON.stringify({ + company_name: null, + website: null, + contact_name: null, + role: null, + email: null, + phone: null, + location: null, + source_url: "https://example.com/exhibitors", + evidence_excerpt: "Members Company Application", + project_fit_reason: + "The source text does not identify a specific company or website.", + missing_fields: [ + "company_name", + "website", + "contact_name", + "role", + "email", + "phone", + "location", + ], + }); + const result = await runLocalCampaign( + { ...campaign, max_ollama_calls_per_seed: 1, max_companies: 5 }, + ["https://example.com/exhibitors"], + { + pageLoader: async (url) => ({ + cached: false, + page: { + url, + content_hash: "hash", + text: "
  • Members Company Application https://application.example
    • ", + fetched_at: new Date().toISOString(), + }, + }), + client: { generate: async () => ({ text: thinLead, latencyMs: 4 }) }, + stager: async (lead, path) => ({ + path: path ?? "local", + staged: lead, + duplicate: false, + }), + }, + ); + expect(result.staged).toBe(0); + expect(result.failures[0]?.reason).toBe( + "Source unit did not produce company-level evidence.", + ); + }); + it("falls back to page-level extraction when no source units are found", async () => { let calls = 0; const result = await runLocalCampaign( @@ -337,7 +474,7 @@ describe("local campaign runner", () => { page: { url, content_hash: "hash", - text: "Acme Displays LLC is an exhibitor display company in Chicago.", + text: "Acme Displays, https://acme.example, is an exhibitor display company in Chicago.", fetched_at: new Date().toISOString(), }, }), From 994886495b0c47b2a5db3cc6a205cf3048cafb46 Mon Sep 17 00:00:00 2001 From: Danny Date: Wed, 16 Sep 2026 20:33:14 -0400 Subject: [PATCH 32/33] Rank local campaign source units before extraction --- apps/agent/agent/lib/local-campaign-runner.ts | 2 + .../agent/lib/local-campaign-source-units.ts | 136 +++++++++++++++++- .../agent/agent/lib/local-campaign-staging.ts | 2 + apps/agent/test/local-campaign.spec.ts | 131 +++++++++++++++++ 4 files changed, 270 insertions(+), 1 deletion(-) diff --git a/apps/agent/agent/lib/local-campaign-runner.ts b/apps/agent/agent/lib/local-campaign-runner.ts index 40deaa744..a43884ed5 100644 --- a/apps/agent/agent/lib/local-campaign-runner.ts +++ b/apps/agent/agent/lib/local-campaign-runner.ts @@ -464,6 +464,8 @@ export async function runLocalCampaign( source_unit_id: candidate.sourceUnit?.unit_id, parent_source_url: candidate.sourceUnit?.parent_source_url, extraction_method: candidate.sourceUnit?.extraction_method, + source_unit_rank_score: candidate.sourceUnit?.rank_score, + source_unit_ranking_reasons: candidate.sourceUnit?.ranking_reasons, status: missingRequired.length > 0 ? "needs_review" : "new", }; let result: Awaited>; diff --git a/apps/agent/agent/lib/local-campaign-source-units.ts b/apps/agent/agent/lib/local-campaign-source-units.ts index 4be5ccd6d..56876df75 100644 --- a/apps/agent/agent/lib/local-campaign-source-units.ts +++ b/apps/agent/agent/lib/local-campaign-source-units.ts @@ -12,6 +12,8 @@ export type LocalCampaignSourceUnit = { evidence_excerpt: string; extraction_method: "table_row" | "list_item" | "card" | "link_context"; content_hash: string; + rank_score: number; + ranking_reasons: string[]; }; export type LocalCampaignSourceUnitCaps = { @@ -28,6 +30,15 @@ const BOILERPLATE = /\b(cookie|privacy|terms|register now|buy tickets|subscribe|newsletter|sponsor|navigation|menu|login|sign in|venue|floor plan|organizer|copyright|all rights reserved)\b/i; const EVENT_LEVEL = /\b(exhibitor directory|exhibitor list|trade show|conference|expo|event|venue|registration|attendee|speaker)\b/i; +const EXHIBITOR_PROFILE = /\b(exhibitor|profile|company|listing|directory)\b/i; +const BOOTH_PATTERN = + /\b(?:booth|stand|stall|pavilion)\s*(?:no\.?|number|#)?\s*[A-Z]?\d[\w.-]*\b/i; +const LOCATION_PATTERN = + /\b(?:[A-Z][a-z]+(?:\s+[A-Z][a-z]+)*,\s*)?(?:[A-Z]{2}|USA|United States|Canada|Mexico|Germany|Taiwan|China|Thailand|Vietnam|India|Japan|Korea|United Kingdom|UK|France|Italy|Spain)\b/; +const PRODUCT_SERVICE = + /\b(display|fixture|fabrication|manufacturing|supplier|distributor|automation|machinery|equipment|hardware|server|logistics|laboratory|lab|monitoring|energy|electrical|prototype|parts|services?|products?)\b/i; +const PROMOTIONAL = + /\b(register|registration|schedule|agenda|attendee|buy tickets|book now|learn more|sponsorship opportunity|floor plan|hotel|travel)\b/i; const BLOCK_PATTERNS = [ { pattern: /]*>([\s\S]*?)<\/tr>/gi, method: "table_row" as const }, { pattern: /]*>([\s\S]*?)<\/li>/gi, method: "list_item" as const }, @@ -79,7 +90,9 @@ export function extractLocalCampaignSourceUnits( ); if (unit) units.push(unit); } - return dedupeSourceUnits(units).slice(0, caps.max_source_units_per_page); + return dedupeSourceUnits(units) + .sort((left, right) => right.rank_score - left.rank_score) + .slice(0, caps.max_source_units_per_page); } export function hasUsefulPageEvidence(text: string): boolean { @@ -87,6 +100,106 @@ export function hasUsefulPageEvidence(text: string): boolean { return looksCompanyLevel(cleaned) && !isOnlyEventLevel(cleaned); } +export type LocalCampaignSourceUnitRanking = { + rank_score: number; + ranking_reasons: string[]; +}; + +export function rankLocalCampaignSourceUnitText( + text: string, + context: { + parent_source_url?: string; + source_domain?: string; + candidate_company_name?: string | null; + candidate_website?: string | null; + candidate_location?: string | null; + extraction_method?: LocalCampaignSourceUnit["extraction_method"]; + } = {}, +): LocalCampaignSourceUnitRanking { + const cleaned = cleanText(text); + const parentDomain = context.source_domain ?? null; + const website = + context.candidate_website ?? + cleaned.match(WEBSITE_PATTERN)?.[0]?.replace(/[),.]+$/, "") ?? + null; + const company = + context.candidate_company_name ?? candidateCompanyName(cleaned); + const location = context.candidate_location ?? candidateLocation(cleaned); + const reasons: string[] = []; + let score = 0; + + if (company) { + score += 30; + reasons.push("likely company name"); + } + if (website) { + const websiteDomain = safeDomain(website); + if (websiteDomain && parentDomain && websiteDomain !== parentDomain) { + score += 30; + reasons.push("external company website"); + } else if (EXHIBITOR_PROFILE.test(website)) { + score += 18; + reasons.push("exhibitor profile URL"); + } else { + score += 12; + reasons.push("website or profile URL"); + } + } + if ( + context.extraction_method === "link_context" && + website && + EXHIBITOR_PROFILE.test(website) + ) { + score += 12; + reasons.push("exhibitor profile link"); + } + if (BOOTH_PATTERN.test(cleaned)) { + score += 20; + reasons.push("booth or stand number"); + } + if (location || LOCATION_PATTERN.test(cleaned)) { + score += 10; + reasons.push("location evidence"); + } + if (PRODUCT_SERVICE.test(cleaned)) { + score += 15; + reasons.push("product or service description"); + } + if (/\b(exhibitor|sponsor|booth|stand|pavilion)\b/i.test(cleaned)) { + score += 10; + reasons.push("exhibitor context"); + } + if ( + cleaned.length <= 80 && + EVENT_LEVEL.test(cleaned) && + !company && + !website + ) { + score -= 30; + reasons.push("page title or event-only text"); + } + if (isOnlyEventLevel(cleaned)) { + score -= 40; + reasons.push("event or venue level evidence"); + } + if (BOILERPLATE.test(cleaned)) { + score -= 35; + reasons.push("navigation/footer/cookie boilerplate"); + } + if (PROMOTIONAL.test(cleaned) && !company) { + score -= 25; + reasons.push("registration or promotional copy"); + } + + return { + rank_score: score, + ranking_reasons: + reasons.length > 0 + ? [...new Set(reasons)] + : ["weak source unit evidence"], + }; +} + export function dedupeSourceUnits( units: LocalCampaignSourceUnit[], ): LocalCampaignSourceUnit[] { @@ -129,6 +242,14 @@ function buildSourceUnit( candidateText.match(WEBSITE_PATTERN)?.[0]?.replace(/[),.]+$/, "") ?? null; const location = candidateLocation(candidateText); + const ranking = rankLocalCampaignSourceUnitText(candidateText, { + parent_source_url: parentSourceUrl, + source_domain: sourceDomain, + candidate_company_name: company, + candidate_website: website, + candidate_location: location, + extraction_method: extractionMethod, + }); return { unit_id: contentHash.slice(0, 24), parent_source_url: parentSourceUrl, @@ -140,6 +261,7 @@ function buildSourceUnit( evidence_excerpt: candidateText.slice(0, 500), extraction_method: extractionMethod, content_hash: contentHash, + ...ranking, }; } @@ -193,6 +315,18 @@ function normalizeWebsite( } } +function safeDomain(value: string): string | null { + try { + return normalizeDomain(new URL(value).hostname); + } catch { + try { + return normalizeDomain(new URL(`https://${value}`).hostname); + } catch { + return null; + } + } +} + function cleanText(value: string): string { return value .replace(//gi, " ") diff --git a/apps/agent/agent/lib/local-campaign-staging.ts b/apps/agent/agent/lib/local-campaign-staging.ts index 20607a4b6..c62b769d9 100644 --- a/apps/agent/agent/lib/local-campaign-staging.ts +++ b/apps/agent/agent/lib/local-campaign-staging.ts @@ -18,6 +18,8 @@ export type LocalStagedLead = LocalCampaignLeadRecord & source_unit_id?: string; parent_source_url?: string; extraction_method?: string; + source_unit_rank_score?: number; + source_unit_ranking_reasons?: string[]; status: LocalCampaignStatus; }; diff --git a/apps/agent/test/local-campaign.spec.ts b/apps/agent/test/local-campaign.spec.ts index 495a09ce2..636f6ca2a 100644 --- a/apps/agent/test/local-campaign.spec.ts +++ b/apps/agent/test/local-campaign.spec.ts @@ -21,6 +21,7 @@ import { scoreLocalCampaignLead } from "../agent/lib/local-campaign-scoring"; import { extractLocalCampaignSourceUnits, hasUsefulPageEvidence, + rankLocalCampaignSourceUnitText, } from "../agent/lib/local-campaign-source-units"; import { assertLocalCampaignPath, @@ -373,6 +374,136 @@ describe("local campaign runner", () => { expect(result).toHaveLength(1); }); + it("ranks company units with websites high", () => { + const ranking = rankLocalCampaignSourceUnitText( + "Acme Displays LLC https://acme.example custom trade show display fixtures", + { source_domain: "event.example" }, + ); + expect(ranking.rank_score).toBeGreaterThanOrEqual(70); + expect(ranking.ranking_reasons).toContain("likely company name"); + expect(ranking.ranking_reasons).toContain("external company website"); + }); + + it("ranks company units with booth evidence high", () => { + const ranking = rankLocalCampaignSourceUnitText( + "Beta Fixtures Inc booth 1240 industrial display products Austin, TX", + { source_domain: "event.example" }, + ); + expect(ranking.rank_score).toBeGreaterThanOrEqual(70); + expect(ranking.ranking_reasons).toContain("booth or stand number"); + }); + + it("ranks event titles low", () => { + const ranking = rankLocalCampaignSourceUnitText( + "PartWall 2027 exhibitor directory trade show registration", + { source_domain: "event.example" }, + ); + expect(ranking.rank_score).toBeLessThan(0); + expect(ranking.ranking_reasons).toContain("page title or event-only text"); + }); + + it("ranks venue-only evidence low", () => { + const ranking = rankLocalCampaignSourceUnitText( + "JI Expo venue event floor plan attendee registration", + { source_domain: "event.example" }, + ); + expect(ranking.rank_score).toBeLessThan(0); + expect(ranking.ranking_reasons).toContain("event or venue level evidence"); + }); + + it("ranks organizer boilerplate low", () => { + const ranking = rankLocalCampaignSourceUnitText( + "Organizer newsletter privacy terms copyright all rights reserved", + { source_domain: "event.example" }, + ); + expect(ranking.rank_score).toBeLessThan(0); + expect(ranking.ranking_reasons).toContain( + "navigation/footer/cookie boilerplate", + ); + }); + + it("sends top-ranked source units to Ollama first", async () => { + const calls: string[] = []; + const result = await runLocalCampaign( + { ...campaign, max_ollama_calls_per_seed: 1, max_companies: 5 }, + ["https://example.com/exhibitors"], + { + pageLoader: async (url) => ({ + cached: false, + page: { + url, + content_hash: "hash", + text: [ + "Beta Fixtures Incgeneral supplier listing", + "Acme Displays LLChttps://acme.exampleBooth 1240custom trade show display fixtures", + ].join(""), + fetched_at: new Date().toISOString(), + }, + }), + client: { + generate: async ({ prompt }) => { + calls.push(prompt); + return { + text: leadJson.replace( + "https://example.com/one", + "https://example.com/exhibitors", + ), + latencyMs: 4, + }; + }, + }, + stager: async (lead, path) => ({ + path: path ?? "local", + staged: lead, + duplicate: false, + }), + }, + ); + expect(result.ollama_calls).toBe(1); + expect(calls[0]).toContain("Acme Displays LLC"); + expect(calls[0]).not.toContain("Beta Fixtures Inc"); + }); + + it("records source unit ranking reasons on staged leads", async () => { + const sourceText = + "Acme Displays LLC https://acme.example Booth 1240 custom display fixtures Chicago, IL"; + const sourceLead = JSON.stringify({ + ...JSON.parse(leadJson), + source_url: "https://example.com/exhibitors", + evidence_excerpt: sourceText, + }); + const result = await runLocalCampaign( + { ...campaign, max_ollama_calls_per_seed: 1, max_companies: 5 }, + ["https://example.com/exhibitors"], + { + pageLoader: async (url) => ({ + cached: false, + page: { + url, + content_hash: "hash", + text: `${sourceText}`, + fetched_at: new Date().toISOString(), + }, + }), + client: { + generate: async () => ({ + text: sourceLead, + latencyMs: 4, + }), + }, + stager: async (lead, path) => ({ + path: path ?? "local", + staged: lead, + duplicate: false, + }), + }, + ); + expect(result.leads[0]?.source_unit_rank_score).toBeGreaterThan(0); + expect(result.leads[0]?.source_unit_ranking_reasons).toContain( + "external company website", + ); + }); + it("avoids whole-page model calls when source units exist", async () => { const calls: string[] = []; const result = await runLocalCampaign( From 81863920354a02a00ec3968d0dc80106ad9de70b Mon Sep 17 00:00:00 2001 From: Danny Date: Thu, 17 Sep 2026 00:03:49 -0400 Subject: [PATCH 33/33] Filter weak local campaign source units --- apps/agent/agent/lib/local-campaign-runner.ts | 87 ++++++++-- .../agent/lib/local-campaign-source-units.ts | 159 +++++++++++++++--- apps/agent/test/local-campaign.spec.ts | 147 ++++++++++++++++ 3 files changed, 362 insertions(+), 31 deletions(-) diff --git a/apps/agent/agent/lib/local-campaign-runner.ts b/apps/agent/agent/lib/local-campaign-runner.ts index a43884ed5..f61085ef8 100644 --- a/apps/agent/agent/lib/local-campaign-runner.ts +++ b/apps/agent/agent/lib/local-campaign-runner.ts @@ -12,9 +12,10 @@ import { import { scoreLocalCampaignLead } from "./local-campaign-scoring"; import { DEFAULT_LOCAL_CAMPAIGN_SOURCE_UNIT_CAPS, - extractLocalCampaignSourceUnits, + extractLocalCampaignSourceUnitCandidates, hasUsefulPageEvidence, type LocalCampaignSourceUnit, + type LocalCampaignSourceUnitRejectionReason, } from "./local-campaign-source-units"; import { assertLocalCampaignPath, @@ -75,6 +76,8 @@ export type LocalCampaignProgressEvent = { elapsed_ms: number; reason?: string; source_unit_count?: number; + rejected_source_unit_count?: number; + rejection_reasons?: Record; ollama_call_count?: number; }; @@ -84,6 +87,8 @@ export type LocalCampaignSeedStatus = { failure_reason: string | null; elapsed_ms: number; source_unit_count: number; + rejected_source_unit_count: number; + rejection_reasons: Record; ollama_call_count: number; staged_count: number; created_at: string; @@ -95,6 +100,11 @@ export type LocalCampaignRunResult = { fetched: number; cached: number; source_units: number; + rejected_source_units: number; + source_unit_rejection_reasons: Record< + LocalCampaignSourceUnitRejectionReason, + number + >; ollama_calls: number; rejected_by_url_safety: number; failed_fetch: number; @@ -154,6 +164,7 @@ async function flushSeedStatus( function extractionCandidates( units: LocalCampaignSourceUnit[], + rejectedSourceUnitCount: number, text: string, url: string, maxOllamaCallsPerSeed: number, @@ -170,6 +181,7 @@ function extractionCandidates( sourceUnit: unit, })); } + if (rejectedSourceUnitCount > 0) return []; if (!hasUsefulPageEvidence(text)) return []; return [ { @@ -184,9 +196,7 @@ function lacksCompanyLevelEvidence( lead: LocalCampaignLeadRecord, sourceUnit: LocalCampaignSourceUnit | null, ): boolean { - return ( - sourceUnit !== null && lead.company_name === null && lead.website === null - ); + return sourceUnit !== null && lead.company_name === null; } function cacheCounters(wasCached: boolean): { @@ -208,6 +218,32 @@ function seedCompletionFailureReason( return "No extraction candidates."; } +function emptyRejectionReasons(): Record< + LocalCampaignSourceUnitRejectionReason, + number +> { + return { + weak_website_only: 0, + event_vendor_link: 0, + navigation_or_footer: 0, + registration_or_ticketing: 0, + venue_or_organizer: 0, + boilerplate: 0, + insufficient_company_evidence: 0, + }; +} + +function addRejectionReasons( + target: Record, + source: Record, +): void { + for (const key of Object.keys( + source, + ) as LocalCampaignSourceUnitRejectionReason[]) { + target[key] += source[key]; + } +} + export async function loadCampaignFile(path: string): Promise { return localCampaignSchema.parse(JSON.parse(await readFile(path, "utf8"))); } @@ -258,6 +294,8 @@ export async function runLocalCampaign( let fetched = 0; let cached = 0; let sourceUnits = 0; + let rejectedSourceUnits = 0; + const sourceUnitRejectionReasons = emptyRejectionReasons(); let ollamaCalls = 0; let rejectedByUrlSafety = 0; let failedFetch = 0; @@ -280,6 +318,8 @@ export async function runLocalCampaign( for (const url of uniqueSeeds) { const seedStartedAt = performance.now(); let seedSourceUnitCount = 0; + let seedRejectedSourceUnitCount = 0; + let seedRejectionReasons = emptyRejectionReasons(); let seedOllamaCalls = 0; let seedStagedCount = 0; const emit = ( @@ -294,6 +334,8 @@ export async function runLocalCampaign( url, elapsed_ms: elapsedSince(seedStartedAt), source_unit_count: seedSourceUnitCount, + rejected_source_unit_count: seedRejectedSourceUnitCount, + rejection_reasons: seedRejectionReasons, ollama_call_count: seedOllamaCalls, ...details, }); @@ -307,6 +349,8 @@ export async function runLocalCampaign( failure_reason: failureReason, elapsed_ms: elapsedSince(seedStartedAt), source_unit_count: seedSourceUnitCount, + rejected_source_unit_count: seedRejectedSourceUnitCount, + rejection_reasons: seedRejectionReasons, ollama_call_count: seedOllamaCalls, staged_count: seedStagedCount, created_at: new Date().toISOString(), @@ -360,15 +404,32 @@ export async function runLocalCampaign( cached += cache.cached; fetched += cache.fetched; emit(cache.event); - const units = extractLocalCampaignSourceUnits(loaded.page.text, url, { - max_source_units_per_page: maxSourceUnitsPerPage, - max_source_unit_chars: maxSourceUnitChars, - }); + const sourceUnitCandidates = extractLocalCampaignSourceUnitCandidates( + loaded.page.text, + url, + { + max_source_units_per_page: maxSourceUnitsPerPage, + max_source_unit_chars: maxSourceUnitChars, + }, + ); + const units = sourceUnitCandidates.units; + seedRejectedSourceUnitCount = sourceUnitCandidates.rejected.length; + seedRejectionReasons = emptyRejectionReasons(); + for (const rejection of sourceUnitCandidates.rejected) { + seedRejectionReasons[rejection.rejection_reason] += 1; + } + rejectedSourceUnits += seedRejectedSourceUnitCount; + addRejectionReasons(sourceUnitRejectionReasons, seedRejectionReasons); seedSourceUnitCount = units.length; sourceUnits += units.length; - emit("source_units_found", { source_unit_count: units.length }); + emit("source_units_found", { + source_unit_count: units.length, + rejected_source_unit_count: seedRejectedSourceUnitCount, + rejection_reasons: seedRejectionReasons, + }); const candidates = extractionCandidates( units, + seedRejectedSourceUnitCount, loaded.page.text, url, maxOllamaCallsPerSeed, @@ -376,7 +437,11 @@ export async function runLocalCampaign( ); if (units.length === 0 && candidates.length === 0) { const reason = "No company-level source unit or useful page evidence."; - emit("source_units_skipped", { reason }); + emit("source_units_skipped", { + reason, + rejected_source_unit_count: seedRejectedSourceUnitCount, + rejection_reasons: seedRejectionReasons, + }); emit("seed_completed", { reason }); await flushSeedStatus( seedStatuses, @@ -518,6 +583,8 @@ export async function runLocalCampaign( fetched, cached, source_units: sourceUnits, + rejected_source_units: rejectedSourceUnits, + source_unit_rejection_reasons: sourceUnitRejectionReasons, ollama_calls: ollamaCalls, rejected_by_url_safety: rejectedByUrlSafety, failed_fetch: failedFetch, diff --git a/apps/agent/agent/lib/local-campaign-source-units.ts b/apps/agent/agent/lib/local-campaign-source-units.ts index 56876df75..569f0b38e 100644 --- a/apps/agent/agent/lib/local-campaign-source-units.ts +++ b/apps/agent/agent/lib/local-campaign-source-units.ts @@ -16,6 +16,24 @@ export type LocalCampaignSourceUnit = { ranking_reasons: string[]; }; +export type LocalCampaignSourceUnitRejectionReason = + | "weak_website_only" + | "event_vendor_link" + | "navigation_or_footer" + | "registration_or_ticketing" + | "venue_or_organizer" + | "boilerplate" + | "insufficient_company_evidence"; + +export type LocalCampaignRejectedSourceUnit = { + parent_source_url: string; + source_domain: string; + candidate_text: string; + candidate_website: string | null; + extraction_method: LocalCampaignSourceUnit["extraction_method"]; + rejection_reason: LocalCampaignSourceUnitRejectionReason; +}; + export type LocalCampaignSourceUnitCaps = { max_source_units_per_page: number; max_source_unit_chars: number; @@ -39,6 +57,16 @@ const PRODUCT_SERVICE = /\b(display|fixture|fabrication|manufacturing|supplier|distributor|automation|machinery|equipment|hardware|server|logistics|laboratory|lab|monitoring|energy|electrical|prototype|parts|services?|products?)\b/i; const PROMOTIONAL = /\b(register|registration|schedule|agenda|attendee|buy tickets|book now|learn more|sponsorship opportunity|floor plan|hotel|travel)\b/i; +const REGISTRATION_TICKETING = + /\b(register|registration|tickets?|buy tickets|book now|attendee registration|visitor registration|badge|passes)\b/i; +const NAVIGATION_FOOTER = + /\b(home|about us|contact us|contact-us|privacy|terms|cookie|newsletter|subscribe|login|sign in|menu|navigation|copyright|all rights reserved|media kit|advertise)\b/i; +const SOCIAL_OR_UTILITY_DOMAIN = + /\b(?:facebook|instagram|linkedin|twitter|x|youtube|tiktok|pinterest|google|apple|wa\.me|whatsapp)\.com\b/i; +const EVENT_VENDOR = + /\b(website by|powered by|designed by|marketing \+ guidance|amplify industrial marketing|aimg|map your show|expocad|cvent|eventbrite|swoogo|swapcard|bizzabo|event platform|marketing platform)\b/i; +const VENUE_OR_ORGANIZER = + /\b(venue|organizer|organiser|convention center|conference center|exhibition center|expo center|floor plan|show management|event management)\b/i; const BLOCK_PATTERNS = [ { pattern: /]*>([\s\S]*?)<\/tr>/gi, method: "table_row" as const }, { pattern: /]*>([\s\S]*?)<\/li>/gi, method: "list_item" as const }, @@ -60,18 +88,32 @@ export function extractLocalCampaignSourceUnits( parentSourceUrl: string, caps: LocalCampaignSourceUnitCaps = DEFAULT_LOCAL_CAMPAIGN_SOURCE_UNIT_CAPS, ): LocalCampaignSourceUnit[] { + return extractLocalCampaignSourceUnitCandidates(text, parentSourceUrl, caps) + .units; +} + +export function extractLocalCampaignSourceUnitCandidates( + text: string, + parentSourceUrl: string, + caps: LocalCampaignSourceUnitCaps = DEFAULT_LOCAL_CAMPAIGN_SOURCE_UNIT_CAPS, +): { + units: LocalCampaignSourceUnit[]; + rejected: LocalCampaignRejectedSourceUnit[]; +} { const sourceDomain = normalizeDomain(new URL(parentSourceUrl).hostname); const units: LocalCampaignSourceUnit[] = []; + const rejected: LocalCampaignRejectedSourceUnit[] = []; for (const entry of BLOCK_PATTERNS) { for (const match of text.matchAll(entry.pattern)) { - const unit = buildSourceUnit( + const result = buildSourceUnit( match[1] ?? "", parentSourceUrl, sourceDomain, entry.method, caps.max_source_unit_chars, ); - if (unit) units.push(unit); + if (result.unit) units.push(result.unit); + else if (result.rejected) rejected.push(result.rejected); } } for (const match of text.matchAll( @@ -80,7 +122,7 @@ export function extractLocalCampaignSourceUnits( const href = normalizeWebsite(match[1] ?? "", parentSourceUrl); const label = cleanText(match[2] ?? ""); const block = [label, href].filter(Boolean).join(" "); - const unit = buildSourceUnit( + const result = buildSourceUnit( block, parentSourceUrl, sourceDomain, @@ -88,11 +130,15 @@ export function extractLocalCampaignSourceUnits( caps.max_source_unit_chars, href, ); - if (unit) units.push(unit); + if (result.unit) units.push(result.unit); + else if (result.rejected) rejected.push(result.rejected); } - return dedupeSourceUnits(units) - .sort((left, right) => right.rank_score - left.rank_score) - .slice(0, caps.max_source_units_per_page); + return { + units: dedupeSourceUnits(units) + .sort((left, right) => right.rank_score - left.rank_score) + .slice(0, caps.max_source_units_per_page), + rejected, + }; } export function hasUsefulPageEvidence(text: string): boolean { @@ -229,42 +275,113 @@ function buildSourceUnit( extractionMethod: LocalCampaignSourceUnit["extraction_method"], maxChars: number, websiteHint: string | null = null, -): LocalCampaignSourceUnit | null { +): { + unit: LocalCampaignSourceUnit | null; + rejected: LocalCampaignRejectedSourceUnit | null; +} { const cleaned = cleanText(raw); - if (!looksCompanyLevel(cleaned) || isOnlyEventLevel(cleaned)) return null; const candidateText = cleaned.slice(0, maxChars); - const contentHash = createHash("sha256") - .update(`${parentSourceUrl}:${candidateText}`, "utf8") - .digest("hex"); const company = candidateCompanyName(candidateText); const website = websiteHint ?? candidateText.match(WEBSITE_PATTERN)?.[0]?.replace(/[),.]+$/, "") ?? null; const location = candidateLocation(candidateText); - const ranking = rankLocalCampaignSourceUnitText(candidateText, { - parent_source_url: parentSourceUrl, - source_domain: sourceDomain, + const rejectionReason = sourceUnitRejectionReason(candidateText, { candidate_company_name: company, candidate_website: website, candidate_location: location, extraction_method: extractionMethod, }); - return { - unit_id: contentHash.slice(0, 24), + if (rejectionReason) { + return { + unit: null, + rejected: { + parent_source_url: parentSourceUrl, + source_domain: sourceDomain, + candidate_text: candidateText, + candidate_website: website, + extraction_method: extractionMethod, + rejection_reason: rejectionReason, + }, + }; + } + const contentHash = createHash("sha256") + .update(`${parentSourceUrl}:${candidateText}`, "utf8") + .digest("hex"); + const ranking = rankLocalCampaignSourceUnitText(candidateText, { parent_source_url: parentSourceUrl, source_domain: sourceDomain, candidate_company_name: company, candidate_website: website, candidate_location: location, - candidate_text: candidateText, - evidence_excerpt: candidateText.slice(0, 500), extraction_method: extractionMethod, - content_hash: contentHash, - ...ranking, + }); + return { + unit: { + unit_id: contentHash.slice(0, 24), + parent_source_url: parentSourceUrl, + source_domain: sourceDomain, + candidate_company_name: company, + candidate_website: website, + candidate_location: location, + candidate_text: candidateText, + evidence_excerpt: candidateText.slice(0, 500), + extraction_method: extractionMethod, + content_hash: contentHash, + ...ranking, + }, + rejected: null, }; } +export function sourceUnitRejectionReason( + text: string, + context: { + candidate_company_name?: string | null; + candidate_website?: string | null; + candidate_location?: string | null; + extraction_method?: LocalCampaignSourceUnit["extraction_method"]; + } = {}, +): LocalCampaignSourceUnitRejectionReason | null { + const cleaned = cleanText(text); + const website = context.candidate_website ?? null; + const company = + context.candidate_company_name ?? candidateCompanyName(cleaned); + const location = context.candidate_location ?? candidateLocation(cleaned); + const hasWebsite = website !== null || WEBSITE_PATTERN.test(cleaned); + const hasBooth = BOOTH_PATTERN.test(cleaned); + const hasProduct = PRODUCT_SERVICE.test(cleaned); + const hasExhibitorContext = + /\b(exhibitor|booth|stand|pavilion)\b/i.test(cleaned) || + (context.extraction_method === "link_context" && + website !== null && + EXHIBITOR_PROFILE.test(website)); + + if (cleaned.length < 20) return "insufficient_company_evidence"; + if (EVENT_VENDOR.test(cleaned) || (website && EVENT_VENDOR.test(website))) { + return "event_vendor_link"; + } + if (website && SOCIAL_OR_UTILITY_DOMAIN.test(website)) { + return "navigation_or_footer"; + } + if (REGISTRATION_TICKETING.test(cleaned)) return "registration_or_ticketing"; + if (VENUE_OR_ORGANIZER.test(cleaned) && !company) return "venue_or_organizer"; + if (BOILERPLATE.test(cleaned) || NAVIGATION_FOOTER.test(cleaned)) { + return "boilerplate"; + } + if (isOnlyEventLevel(cleaned)) return "venue_or_organizer"; + if (!looksCompanyLevel(cleaned)) return "insufficient_company_evidence"; + if (!company) + return hasWebsite ? "weak_website_only" : "insufficient_company_evidence"; + if ( + !(hasWebsite || location || hasBooth || hasProduct || hasExhibitorContext) + ) { + return "insufficient_company_evidence"; + } + return null; +} + function looksCompanyLevel(text: string): boolean { if (text.length < 20 || BOILERPLATE.test(text)) return false; return ( diff --git a/apps/agent/test/local-campaign.spec.ts b/apps/agent/test/local-campaign.spec.ts index 636f6ca2a..648a69f9c 100644 --- a/apps/agent/test/local-campaign.spec.ts +++ b/apps/agent/test/local-campaign.spec.ts @@ -19,9 +19,11 @@ import { import { localCampaignSchema } from "../agent/lib/local-campaign-schema"; import { scoreLocalCampaignLead } from "../agent/lib/local-campaign-scoring"; import { + extractLocalCampaignSourceUnitCandidates, extractLocalCampaignSourceUnits, hasUsefulPageEvidence, rankLocalCampaignSourceUnitText, + sourceUnitRejectionReason, } from "../agent/lib/local-campaign-source-units"; import { assertLocalCampaignPath, @@ -503,7 +505,103 @@ describe("local campaign runner", () => { "external company website", ); }); +}); + +describe("local campaign source unit filtering", () => { + it("rejects AIMG-style website-only event vendor links", () => { + const candidates = extractLocalCampaignSourceUnitCandidates( + "Website by Amplify Industrial Marketing + Guidance.", + "https://www.automateshow.com/exhibitors", + ); + expect(candidates.units).toHaveLength(0); + expect(candidates.rejected[0]?.rejection_reason).toBe("event_vendor_link"); + }); + + it("rejects registration and ticketing source units", () => { + expect( + sourceUnitRejectionReason( + "Register now for attendee badges and buy tickets for the expo", + ), + ).toBe("registration_or_ticketing"); + }); + + it("rejects footer social privacy and contact links", () => { + const candidates = extractLocalCampaignSourceUnitCandidates( + [ + "Facebook", + "Privacy Policy", + "Contact Us", + ].join(""), + "https://example.com/exhibitors", + ); + expect(candidates.units).toHaveLength(0); + expect(candidates.rejected.map((unit) => unit.rejection_reason)).toContain( + "navigation_or_footer", + ); + expect(candidates.rejected.map((unit) => unit.rejection_reason)).toContain( + "boilerplate", + ); + }); + + it("rejects organizer or venue-only source units", () => { + expect( + sourceUnitRejectionReason( + "Show organizer convention center floor plan event management", + ), + ).toBe("venue_or_organizer"); + }); + + it("keeps company source units with website and product text", () => { + const candidates = extractLocalCampaignSourceUnitCandidates( + "Acme Displays LLChttps://acme.examplecustom display fixtures and booth walls", + "https://example.com/exhibitors", + ); + expect(candidates.units).toHaveLength(1); + expect(candidates.rejected).toHaveLength(0); + }); + it("keeps company source units with booth or profile evidence", () => { + const candidates = extractLocalCampaignSourceUnitCandidates( + "Beta Fixtures Inc booth 1240", + "https://example.com/exhibitors", + ); + expect(candidates.units).toHaveLength(1); + expect(candidates.units[0]?.ranking_reasons).toContain( + "booth or stand number", + ); + }); + + it("records rejected source unit reasons in run diagnostics", async () => { + const result = await runLocalCampaign( + { ...campaign, max_ollama_calls_per_seed: 1, max_companies: 5 }, + ["https://example.com/exhibitors"], + { + pageLoader: async (url) => ({ + cached: false, + page: { + url, + content_hash: "hash", + text: "Website by Amplify Industrial Marketing + Guidance.", + fetched_at: new Date().toISOString(), + }, + }), + client: { + generate: async () => { + throw new Error("Ollama should not be called."); + }, + }, + }, + ); + expect(result.ollama_calls).toBe(0); + expect(result.rejected_source_units).toBe(1); + expect(result.source_unit_rejection_reasons.event_vendor_link).toBe(1); + expect(result.seed_statuses[0]?.rejection_reasons.event_vendor_link).toBe( + 1, + ); + }); +}); + +describe("local campaign runner extraction flow", () => { it("avoids whole-page model calls when source units exist", async () => { const calls: string[] = []; const result = await runLocalCampaign( @@ -594,6 +692,55 @@ describe("local campaign runner", () => { ); }); + it("rejects source unit output that only has a website", async () => { + const websiteOnlyLead = JSON.stringify({ + ...JSON.parse(leadJson), + company_name: null, + website: "https://acme.example", + location: null, + source_url: "https://example.com/exhibitors", + evidence_excerpt: "Acme Displays LLC https://acme.example", + missing_fields: [ + "company_name", + "contact_name", + "role", + "email", + "phone", + "location", + ], + }); + const result = await runLocalCampaign( + { ...campaign, max_ollama_calls_per_seed: 1, max_companies: 5 }, + ["https://example.com/exhibitors"], + { + pageLoader: async (url) => ({ + cached: false, + page: { + url, + content_hash: "hash", + text: "
  • Acme Displays LLC https://acme.example custom display fixtures
    • ", + fetched_at: new Date().toISOString(), + }, + }), + client: { + generate: async () => ({ + text: websiteOnlyLead, + latencyMs: 4, + }), + }, + stager: async (lead, path) => ({ + path: path ?? "local", + staged: lead, + duplicate: false, + }), + }, + ); + expect(result.staged).toBe(0); + expect(result.failures[0]?.reason).toBe( + "Source unit did not produce company-level evidence.", + ); + }); + it("falls back to page-level extraction when no source units are found", async () => { let calls = 0; const result = await runLocalCampaign(