diff --git a/infrastructure/nixos/flake.nix b/infrastructure/nixos/flake.nix index d5cbc6b2e..2364e15d9 100644 --- a/infrastructure/nixos/flake.nix +++ b/infrastructure/nixos/flake.nix @@ -22,6 +22,7 @@ ./modules/homepage.nix ./modules/notify.nix ./modules/notify-failure.nix + ./modules/activity-digest.nix ./modules/secrets.nix ]; }; diff --git a/infrastructure/nixos/modules/activity-digest.nix b/infrastructure/nixos/modules/activity-digest.nix new file mode 100644 index 000000000..1e2103baa --- /dev/null +++ b/infrastructure/nixos/modules/activity-digest.nix @@ -0,0 +1,33 @@ +{ config, pkgs, lib, ... }: + +{ + systemd.services.forms-lab-activity-digest = { + description = "Forms Lab Daily Activity Digest"; + after = [ "forms-lab-notify.service" ]; + requires = [ "forms-lab-notify.service" ]; + + path = with pkgs; [ bun ]; + + serviceConfig = { + Type = "oneshot"; + User = "forms-lab"; + Group = "forms-lab"; + WorkingDirectory = "/srv/forms-lab/main"; + }; + + script = '' + export ACTIVITY_DB_PATH="/srv/forms-lab/data/activity.sqlite" + exec ${pkgs.bun}/bin/bun run src/entrypoints/cli/main.ts activity digest + ''; + }; + + systemd.timers.forms-lab-activity-digest = { + description = "Daily Forms Lab Activity Digest"; + wantedBy = [ "timers.target" ]; + + timerConfig = { + OnCalendar = "*-*-* 06:00:00"; + Persistent = true; + }; + }; +} diff --git a/src/entrypoints/app/routes/auth/index.tsx b/src/entrypoints/app/routes/auth/index.tsx index b587a4773..422b1cf2c 100644 --- a/src/entrypoints/app/routes/auth/index.tsx +++ b/src/entrypoints/app/routes/auth/index.tsx @@ -1,5 +1,6 @@ import { Hono } from 'hono' import { deleteCookie, setCookie } from 'hono/cookie' +import type { ActivityStore } from '../../../../services/activity' import { Layout } from '../../../../design-system/components/flex-layout' import type { AccessStore, UserStore } from '../../../../services/auth' import { @@ -21,6 +22,7 @@ import { export function createAuthRoutes( userStore: UserStore, accessStore: AccessStore, + options?: { activityStore?: ActivityStore }, ): Hono { const auth = new Hono() @@ -210,6 +212,11 @@ export function createAuthRoutes( path: '/', }) + options?.activityStore?.track({ + eventType: 'sign_in', + userId: ghUser.login, + }) + const returnTo = state.returnTo || resolveUrl('/') // Prevent open redirect — only allow internal paths const safeReturnTo = diff --git a/src/entrypoints/app/routes/forms/index.tsx b/src/entrypoints/app/routes/forms/index.tsx index 5ae6e7397..5edb58737 100644 --- a/src/entrypoints/app/routes/forms/index.tsx +++ b/src/entrypoints/app/routes/forms/index.tsx @@ -675,6 +675,8 @@ export function createFormRouter(deps: FormRouterDeps) { groups: page.groups, collectedFields: session.fields, messages: [], + userId: user.login, + projectId: session.specId, }, null, ) @@ -842,6 +844,8 @@ export function createFormRouter(deps: FormRouterDeps) { groups: page.groups, collectedFields: session.fields, messages, + userId: user.login, + projectId: session.specId, }, userMessage, ) diff --git a/src/entrypoints/app/routes/owner/edit/index.tsx b/src/entrypoints/app/routes/owner/edit/index.tsx index 2ffd7ab69..e518d95a1 100644 --- a/src/entrypoints/app/routes/owner/edit/index.tsx +++ b/src/entrypoints/app/routes/owner/edit/index.tsx @@ -233,6 +233,8 @@ export function createEditRoutes( intent: body.intent, state, previousAttempt: body.previousAttempt, + userId: user.login, + projectId: slug, }) return c.json({ diff --git a/src/entrypoints/app/server.tsx b/src/entrypoints/app/server.tsx index 6fbf2e605..ebc1d8cea 100644 --- a/src/entrypoints/app/server.tsx +++ b/src/entrypoints/app/server.tsx @@ -8,6 +8,7 @@ import { loadFixturePdf, } from '../../../fixtures/index' import { Layout } from '../../design-system/components/flex-layout' +import { createActivityStore } from '../../services/activity' import { createAccessStore, createUserStore } from '../../services/auth' import type { DataCollectionSpec } from '../../services/data-collection' import { createExtractorRegistry } from '../../services/extraction' @@ -71,6 +72,10 @@ mkdirSync(dirname(projectDbPath), { recursive: true }) mkdirSync(dirname(cacheDbPath), { recursive: true }) mkdirSync(reposPath, { recursive: true }) +const activityDbPath = process.env.ACTIVITY_DB_PATH ?? 'data/activity.sqlite' +mkdirSync(dirname(activityDbPath), { recursive: true }) +const activityStore = createActivityStore(activityDbPath) + const projectStore = createProjectStore(projectDbPath) const cacheStore = createCacheStore(cacheDbPath) const userStore = createUserStore(projectDbPath) @@ -80,8 +85,8 @@ const formProjectRepo = createFormProjectRepo(reposPath) // Variant registries: one per task. Each user's preferred variant is // resolved against these at call time so a settings change takes effect // on the next extraction without restarting the process. -const extractionRegistry = createExtractorRegistry() -const shapingRegistry = createShapingRegistry() +const extractionRegistry = createExtractorRegistry(activityStore) +const shapingRegistry = createShapingRegistry(activityStore) const fillingRegistry = createFillingRegistry() const mappingRegistry = createMappingRegistry() const authoringCriteriaRegistry = createAuthoringCriteriaRegistry() @@ -132,7 +137,7 @@ const conversationGateway = new SqliteConversationGateway(formsDbPath) const fillingAgent = process.env.USE_SCRIPTED_AGENT === 'true' ? new ScriptedFillingAgent() - : new BedrockFillingAgent() + : new BedrockFillingAgent({ activityStore }) /** * Adapter: resolve a DataCollectionSpec id to (owner, slug, spec, formSpec) @@ -288,7 +293,7 @@ app.use( ) // Mount auth routes -app.route('/auth', createAuthRoutes(userStore, accessStore)) +app.route('/auth', createAuthRoutes(userStore, accessStore, { activityStore })) // Mount admin routes (requireAdmin is applied inside createAdminRoutes) app.use('/admin/*', requireAuth(accessStore)) @@ -402,6 +407,16 @@ app.post('/new', async (c) => { const pdf = Buffer.from(await file.arrayBuffer()) const name = file.name.replace(/\.pdf$/i, '') const project = await projectService.createProject(name, pdf, user) + activityStore.track({ + eventType: 'pdf_uploaded', + userId: user.login, + projectId: project.slug, + }) + activityStore.track({ + eventType: 'project_created', + userId: user.login, + projectId: project.slug, + }) return c.redirect( resolveUrl(`/${user.login}/${project.slug}/edit/import`), ) @@ -433,6 +448,11 @@ app.post('/new', async (c) => { user, { corpusSlug: corpusMetadata.slug }, ) + activityStore.track({ + eventType: 'project_created', + userId: user.login, + projectId: project.slug, + }) return c.redirect( resolveUrl(`/${user.login}/${project.slug}/edit/import`), ) @@ -456,6 +476,11 @@ app.post('/new', async (c) => { const pdf = loadFixturePdf(fixture) const name = fixture.name const project = await projectService.createProject(name, pdf, user) + activityStore.track({ + eventType: 'project_created', + userId: user.login, + projectId: project.slug, + }) return c.redirect(resolveUrl(`/${user.login}/${project.slug}`)) } catch (err) { console.error('Error creating project:', err) diff --git a/src/entrypoints/cli/commands/activity.ts b/src/entrypoints/cli/commands/activity.ts new file mode 100644 index 000000000..e7ce98deb --- /dev/null +++ b/src/entrypoints/cli/commands/activity.ts @@ -0,0 +1,170 @@ +import { mkdirSync } from 'node:fs' +import { dirname } from 'node:path' +import type { ActivityRow, UsageSummary } from '../../../services/activity' +import { createActivityStore } from '../../../services/activity' +import { notifyEvent } from '../../../services/notifications' + +const DB_PATH = process.env.ACTIVITY_DB_PATH ?? 'data/activity.sqlite' + +function getStore() { + mkdirSync(dirname(DB_PATH), { recursive: true }) + return createActivityStore(DB_PATH) +} + +export function formatSummary(summary: UsageSummary): string { + const lines: string[] = [] + lines.push( + `Total: ${summary.totalEvents} events | ~$${summary.estimatedCost.toFixed(2)} estimated cost`, + ) + lines.push('') + + if (summary.byUser.length > 0) { + lines.push('By user:') + for (const entry of summary.byUser) { + const tokens = + entry.tokens > 1000 + ? `${(entry.tokens / 1000).toFixed(0)}K` + : `${entry.tokens}` + lines.push( + ` ${entry.userId}: ${entry.events} events, ${tokens} tokens, ~$${entry.cost.toFixed(2)}`, + ) + } + lines.push('') + } + + if (summary.byOperation.length > 0) { + lines.push('By operation:') + for (const entry of summary.byOperation) { + lines.push( + ` ${entry.operation}: ${entry.events} calls, ~$${entry.cost.toFixed(2)}`, + ) + } + lines.push('') + } + + if (summary.byProject.length > 0) { + lines.push('Top projects:') + for (const entry of summary.byProject) { + lines.push( + ` ${entry.projectId}: ${entry.events} events, ~$${entry.cost.toFixed(2)}`, + ) + } + } + + return lines.join('\n') +} + +export function formatEvents(events: ActivityRow[]): string { + if (events.length === 0) return 'No events found.' + + const lines: string[] = [] + for (const event of events) { + const time = new Date(event.timestamp * 1000).toISOString().slice(0, 19) + const parts = [ + time, + event.eventType, + event.userId ?? '-', + event.projectId ?? '-', + event.operation ?? '-', + ] + lines.push(parts.join('\t')) + } + return lines.join('\n') +} + +function parseNumericArg( + args: string[], + flag: string, + defaultValue: number, +): number { + const idx = args.indexOf(flag) + if (idx !== -1 && args[idx + 1]) { + return Number.parseInt(args[idx + 1], 10) + } + return defaultValue +} + +function parseStringArg(args: string[], flag: string): string | undefined { + const idx = args.indexOf(flag) + if (idx !== -1 && args[idx + 1]) { + return args[idx + 1] + } + return undefined +} + +export async function activity(args: string[]): Promise { + const subcommand = args[0] + + if (!subcommand || subcommand === '--help') { + console.log('Usage: bun run cli activity \n') + console.log('Subcommands:') + console.log(' summary Show usage summary (default: today)') + console.log(' events List activity events') + console.log(' digest Post daily digest to Slack') + console.log('\nOptions:') + console.log(' --days Number of days to look back (default: 1)') + console.log(' --user Filter by user') + console.log(' --project Filter by project') + console.log(' --operation Filter by operation') + console.log(' --limit Max events to show (default: 50)') + return 0 + } + + const store = getStore() + const now = Math.floor(Date.now() / 1000) + + if (subcommand === 'summary') { + const days = parseNumericArg(args, '--days', 1) + const from = now - days * 86400 + const summary = store.summarize({ from, to: now }) + console.log( + `\nActivity Summary (last ${days} day${days > 1 ? 's' : ''}):\n`, + ) + console.log(formatSummary(summary)) + return 0 + } + + if (subcommand === 'events') { + const days = parseNumericArg(args, '--days', 1) + const limit = parseNumericArg(args, '--limit', 50) + const events = store.query({ + from: now - days * 86400, + to: now, + userId: parseStringArg(args, '--user'), + projectId: parseStringArg(args, '--project'), + operation: parseStringArg(args, '--operation'), + limit, + }) + console.log(formatEvents(events)) + return 0 + } + + if (subcommand === 'digest') { + const todayMidnight = new Date() + todayMidnight.setHours(0, 0, 0, 0) + const to = Math.floor(todayMidnight.getTime() / 1000) + const from = to - 86400 + + const summary = store.summarize({ from, to }) + const dateStr = new Date(from * 1000).toISOString().slice(0, 10) + + if (summary.totalEvents === 0) { + console.log(`No activity on ${dateStr}. Skipping digest.`) + return 0 + } + + const details = formatSummary(summary) + await notifyEvent({ + type: 'activity-digest', + title: `Forms Lab Daily Usage — ${dateStr}`, + status: 'info', + details, + }) + + console.log(`Digest posted for ${dateStr}`) + return 0 + } + + console.error(`Unknown subcommand: ${subcommand}`) + return 1 +} diff --git a/src/entrypoints/cli/main.ts b/src/entrypoints/cli/main.ts index 4d82619e8..2b8e3ede4 100644 --- a/src/entrypoints/cli/main.ts +++ b/src/entrypoints/cli/main.ts @@ -1,3 +1,4 @@ +import { activity } from './commands/activity' import { deploy } from './commands/deploy' import { evaluate } from './commands/evaluate' import { extract } from './commands/extract' @@ -19,6 +20,11 @@ export interface Command { } const commands: Command[] = [ + { + name: 'activity', + description: 'View activity reports and post usage digests', + run: activity, + }, { name: 'sync-stories', description: 'Sync user stories from GitHub Issues', diff --git a/src/services/activity/index.ts b/src/services/activity/index.ts new file mode 100644 index 000000000..427f9db68 --- /dev/null +++ b/src/services/activity/index.ts @@ -0,0 +1,15 @@ +// Public interface for the activity service. +// External imports (other services, entrypoints, design-system) MUST come +// through this file. Enforced by test/architecture/dependency-rule.test.ts. + +export type { LlmCallMetrics } from './instrumentation' +export { trackLlmCall } from './instrumentation' +export { BEDROCK_PRICING, estimateCost } from './pricing' +export { createActivityStore } from './store' +export type { + ActivityEvent, + ActivityQuery, + ActivityRow, + ActivityStore, + UsageSummary, +} from './types' diff --git a/src/services/activity/instrumentation.ts b/src/services/activity/instrumentation.ts new file mode 100644 index 000000000..216849f32 --- /dev/null +++ b/src/services/activity/instrumentation.ts @@ -0,0 +1,32 @@ +import type { ActivityStore } from './types' + +export interface LlmCallMetrics { + userId?: string + projectId?: string + operation: string + model: string + usage: { + inputTokens?: number | undefined + outputTokens?: number | undefined + } + durationMs: number +} + +/** Fire-and-forget helper for tracking an LLM call. */ +export function trackLlmCall( + store: ActivityStore, + metrics: LlmCallMetrics, +): void { + store.track({ + eventType: 'llm_call', + userId: metrics.userId, + projectId: metrics.projectId, + operation: metrics.operation, + metadata: { + model: metrics.model, + inputTokens: metrics.usage.inputTokens ?? 0, + outputTokens: metrics.usage.outputTokens ?? 0, + durationMs: metrics.durationMs, + }, + }) +} diff --git a/src/services/activity/pricing.ts b/src/services/activity/pricing.ts new file mode 100644 index 000000000..c100b9a57 --- /dev/null +++ b/src/services/activity/pricing.ts @@ -0,0 +1,25 @@ +/** Dollars per 1M tokens. Update when AWS announces price changes. */ +export const BEDROCK_PRICING: Record< + string, + { input: number; output: number } +> = { + 'us.anthropic.claude-opus-4-6-v1': { input: 15.0, output: 75.0 }, + 'us.anthropic.claude-sonnet-4-20250514-v1:0': { input: 3.0, output: 15.0 }, + 'us.anthropic.claude-haiku-4-5-20251001-v1:0': { input: 0.8, output: 4.0 }, + 'us.amazon.nova-pro-v1:0': { input: 0.8, output: 3.2 }, + 'us.amazon.nova-lite-v1:0': { input: 0.06, output: 0.24 }, + 'us.meta.llama3-2-90b-instruct-v1:0': { input: 2.0, output: 2.0 }, +} + +/** Compute estimated cost in dollars for a single LLM call. */ +export function estimateCost( + model: string, + inputTokens: number, + outputTokens: number, +): number { + const pricing = BEDROCK_PRICING[model] + if (!pricing) return 0 + return ( + (inputTokens * pricing.input + outputTokens * pricing.output) / 1_000_000 + ) +} diff --git a/src/services/activity/store.ts b/src/services/activity/store.ts new file mode 100644 index 000000000..2bad146b8 --- /dev/null +++ b/src/services/activity/store.ts @@ -0,0 +1,176 @@ +import { Database } from 'bun:sqlite' +import { estimateCost } from './pricing' +import type { + ActivityEvent, + ActivityQuery, + ActivityRow, + ActivityStore, + UsageSummary, +} from './types' + +export function createActivityStore(dbPath: string): ActivityStore { + const db = new Database(dbPath) + db.run('PRAGMA journal_mode = WAL') + db.run(` + CREATE TABLE IF NOT EXISTS activity_events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + timestamp INTEGER NOT NULL, + event_type TEXT NOT NULL, + user_id TEXT, + project_id TEXT, + operation TEXT, + metadata TEXT + ) + `) + + return { + track(event: ActivityEvent): void { + try { + db.run( + `INSERT INTO activity_events (timestamp, event_type, user_id, project_id, operation, metadata) + VALUES (?, ?, ?, ?, ?, ?)`, + [ + Math.floor(Date.now() / 1000), + event.eventType, + event.userId ?? null, + event.projectId ?? null, + event.operation ?? null, + event.metadata ? JSON.stringify(event.metadata) : null, + ], + ) + } catch (err) { + console.error('[activity] Failed to track event:', err) + } + }, + + query(filters: ActivityQuery): ActivityRow[] { + const conditions: string[] = [] + const params: (string | number)[] = [] + + if (filters.from !== undefined) { + conditions.push('timestamp >= ?') + params.push(filters.from) + } + if (filters.to !== undefined) { + conditions.push('timestamp <= ?') + params.push(filters.to) + } + if (filters.userId) { + conditions.push('user_id = ?') + params.push(filters.userId) + } + if (filters.projectId) { + conditions.push('project_id = ?') + params.push(filters.projectId) + } + if (filters.eventType) { + conditions.push('event_type = ?') + params.push(filters.eventType) + } + if (filters.operation) { + conditions.push('operation = ?') + params.push(filters.operation) + } + + const where = + conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : '' + if (filters.limit) { + params.push(filters.limit) + } + const sql = `SELECT * FROM activity_events ${where} ORDER BY timestamp DESC${filters.limit ? ' LIMIT ?' : ''}` + + const rows = db.query(sql).all(...params) as Array< + Record + > + return rows.map((row) => ({ + id: row.id as number, + timestamp: row.timestamp as number, + eventType: row.event_type as string, + userId: row.user_id as string | null, + projectId: row.project_id as string | null, + operation: row.operation as string | null, + metadata: row.metadata ? JSON.parse(row.metadata as string) : {}, + })) + }, + + summarize(period: { from: number; to: number }): UsageSummary { + const events = this.query({ from: period.from, to: period.to }) + + const byUserMap = new Map< + string, + { events: number; tokens: number; cost: number } + >() + const byOpMap = new Map< + string, + { events: number; tokens: number; cost: number } + >() + const byProjectMap = new Map() + let totalCost = 0 + + for (const event of events) { + const userId = event.userId ?? 'system' + const userEntry = byUserMap.get(userId) ?? { + events: 0, + tokens: 0, + cost: 0, + } + userEntry.events++ + + const projEntry = event.projectId + ? (byProjectMap.get(event.projectId) ?? { events: 0, cost: 0 }) + : null + if (projEntry) { + projEntry.events++ + } + + if (event.eventType === 'llm_call' && event.metadata) { + const model = event.metadata.model as string | undefined + const inputTokens = (event.metadata.inputTokens as number) ?? 0 + const outputTokens = (event.metadata.outputTokens as number) ?? 0 + const tokens = inputTokens + outputTokens + const cost = model + ? estimateCost(model, inputTokens, outputTokens) + : 0 + + userEntry.tokens += tokens + userEntry.cost += cost + totalCost += cost + + const op = event.operation ?? 'unknown' + const opEntry = byOpMap.get(op) ?? { events: 0, tokens: 0, cost: 0 } + opEntry.events++ + opEntry.tokens += tokens + opEntry.cost += cost + byOpMap.set(op, opEntry) + + if (projEntry) { + projEntry.cost += cost + } + } + + if (event.projectId && projEntry) { + byProjectMap.set(event.projectId, projEntry) + } + + byUserMap.set(userId, userEntry) + } + + return { + totalEvents: events.length, + estimatedCost: totalCost, + byUser: [...byUserMap.entries()].map(([userId, data]) => ({ + userId, + ...data, + })), + byOperation: [...byOpMap.entries()].map(([operation, data]) => ({ + operation, + ...data, + })), + byProject: [...byProjectMap.entries()].map(([projectId, data]) => ({ + projectId, + ...data, + })), + } + }, + } +} diff --git a/src/services/activity/types.ts b/src/services/activity/types.ts new file mode 100644 index 000000000..d1e463a3b --- /dev/null +++ b/src/services/activity/types.ts @@ -0,0 +1,51 @@ +export interface ActivityEvent { + eventType: string + userId?: string + projectId?: string + operation?: string + metadata?: Record +} + +export interface ActivityQuery { + from?: number + to?: number + userId?: string + projectId?: string + eventType?: string + operation?: string + limit?: number +} + +export interface ActivityRow { + id: number + timestamp: number + eventType: string + userId: string | null + projectId: string | null + operation: string | null + metadata: Record +} + +export interface UsageSummary { + totalEvents: number + estimatedCost: number + byUser: Array<{ + userId: string + events: number + tokens: number + cost: number + }> + byOperation: Array<{ + operation: string + events: number + tokens: number + cost: number + }> + byProject: Array<{ projectId: string; events: number; cost: number }> +} + +export interface ActivityStore { + track(event: ActivityEvent): void + query(filters: ActivityQuery): ActivityRow[] + summarize(period: { from: number; to: number }): UsageSummary +} diff --git a/src/services/extraction/registry.ts b/src/services/extraction/registry.ts index 12109ccd0..624c165a9 100644 --- a/src/services/extraction/registry.ts +++ b/src/services/extraction/registry.ts @@ -1,4 +1,5 @@ import { StrategyRegistry } from '../../shared/strategy-registry' +import type { ActivityStore } from '../activity' import { createBedrockPdfExtractor, createToolUsePdfExtractor, @@ -15,7 +16,9 @@ import { } from './models' import { getRagRetriever } from './rag-corpus' -export function createExtractorRegistry(): StrategyRegistry { +export function createExtractorRegistry( + activityStore?: ActivityStore, +): StrategyRegistry { const registry = new StrategyRegistry() const [nestedGroupsExemplar] = exemplars @@ -31,7 +34,8 @@ export function createExtractorRegistry(): StrategyRegistry { modelId: OPUS_MODEL_ID, pricing: { inputPer1k: 0.015, outputPer1k: 0.075 }, }, - create: () => createBedrockPdfExtractor({ model: OPUS_MODEL_ID }), + create: () => + createBedrockPdfExtractor({ model: OPUS_MODEL_ID, activityStore }), }) registry.register({ @@ -46,7 +50,8 @@ export function createExtractorRegistry(): StrategyRegistry { modelId: SONNET_MODEL_ID, pricing: { inputPer1k: 0.003, outputPer1k: 0.015 }, }, - create: () => createBedrockPdfExtractor({ model: SONNET_MODEL_ID }), + create: () => + createBedrockPdfExtractor({ model: SONNET_MODEL_ID, activityStore }), }) registry.register({ @@ -61,7 +66,8 @@ export function createExtractorRegistry(): StrategyRegistry { modelId: HAIKU_MODEL_ID, pricing: { inputPer1k: 0.0008, outputPer1k: 0.004 }, }, - create: () => createBedrockPdfExtractor({ model: HAIKU_MODEL_ID }), + create: () => + createBedrockPdfExtractor({ model: HAIKU_MODEL_ID, activityStore }), }) registry.register({ @@ -78,7 +84,11 @@ export function createExtractorRegistry(): StrategyRegistry { pricing: { inputPer1k: 0.003, outputPer1k: 0.015 }, }, create: () => - createBedrockPdfExtractor({ model: SONNET_MODEL_ID, temperature: 0 }), + createBedrockPdfExtractor({ + model: SONNET_MODEL_ID, + temperature: 0, + activityStore, + }), }) registry.register({ @@ -104,6 +114,7 @@ export function createExtractorRegistry(): StrategyRegistry { temperature: 0, promptVariant: 'hybrid', hybridExemplar: nestedGroupsExemplar, + activityStore, }) }, }) @@ -121,7 +132,11 @@ export function createExtractorRegistry(): StrategyRegistry { pricing: { inputPer1k: 0.003, outputPer1k: 0.015 }, }, create: () => - createBedrockPdfExtractor({ model: SONNET_MODEL_ID, exemplars }), + createBedrockPdfExtractor({ + model: SONNET_MODEL_ID, + exemplars, + activityStore, + }), }) registry.register({ @@ -141,6 +156,7 @@ export function createExtractorRegistry(): StrategyRegistry { model: SONNET_MODEL_ID, retriever: getRagRetriever(), retrievalK: 2, + activityStore, }), }) @@ -156,7 +172,8 @@ export function createExtractorRegistry(): StrategyRegistry { modelId: SONNET_MODEL_ID, pricing: { inputPer1k: 0.003, outputPer1k: 0.015 }, }, - create: () => createToolUsePdfExtractor({ model: SONNET_MODEL_ID }), + create: () => + createToolUsePdfExtractor({ model: SONNET_MODEL_ID, activityStore }), }) registry.register({ @@ -175,6 +192,7 @@ export function createExtractorRegistry(): StrategyRegistry { createBedrockPdfExtractor({ model: NOVA_PRO_MODEL_ID, maxOutputTokens: 10000, + activityStore, }), }) @@ -199,6 +217,7 @@ export function createExtractorRegistry(): StrategyRegistry { createBedrockPdfExtractor({ model: NOVA_LITE_MODEL_ID, maxOutputTokens: 10000, + activityStore, }), }) @@ -223,6 +242,7 @@ export function createExtractorRegistry(): StrategyRegistry { createBedrockPdfExtractor({ model: LLAMA_3_2_VISION_MODEL_ID, maxOutputTokens: 8000, + activityStore, }), }) diff --git a/src/services/form-documents/extraction-steps.ts b/src/services/form-documents/extraction-steps.ts index 51b5f17f6..c95aa881e 100644 --- a/src/services/form-documents/extraction-steps.ts +++ b/src/services/form-documents/extraction-steps.ts @@ -7,6 +7,8 @@ import type { LanguageModel } from 'ai' import { generateText } from 'ai' +import type { ActivityStore } from '../activity' +import { trackLlmCall } from '../activity' import type { DataCollectionSpec } from '../data-collection' import type { FormSpec } from '../forms' import { enumerateFields } from './field-mapping' @@ -30,7 +32,12 @@ export function parseJsonResponse( export async function generateFormSpec( model: LanguageModel, spec: DataCollectionSpec, + activityStore?: ActivityStore, + userId?: string, + projectId?: string, + modelId?: string, ): Promise { + const startTime = Date.now() const result = await generateText({ model, maxOutputTokens: 8192, @@ -65,6 +72,16 @@ ${JSON.stringify(spec, null, 2)}`, }, ], }) + if (activityStore && modelId) { + trackLlmCall(activityStore, { + userId, + projectId, + operation: 'extraction-formspec', + model: modelId, + usage: result.usage, + durationMs: Date.now() - startTime, + }) + } return parseJsonResponse(result.text, formSpecSchema) } @@ -74,6 +91,10 @@ export async function mapAcroFormFields( model: LanguageModel, pdf: Buffer, spec: DataCollectionSpec, + activityStore?: ActivityStore, + userId?: string, + projectId?: string, + modelId?: string, ): Promise { const pdfFieldNames = await enumerateFields(pdf) let fieldMapping: FieldMapping = {} @@ -83,6 +104,7 @@ export async function mapAcroFormFields( .flatMap((g) => g.requirements) .map((r) => ({ fieldName: r.fieldName, label: r.label })) + const startTime = Date.now() const mappingResult = await generateText({ model, maxOutputTokens: 4096, @@ -104,6 +126,16 @@ Rules: }, ], }) + if (activityStore && modelId) { + trackLlmCall(activityStore, { + userId, + projectId, + operation: 'extraction-fieldmapping', + model: modelId, + usage: mappingResult.usage, + durationMs: Date.now() - startTime, + }) + } const mappingText = mappingResult.text.trim() const jsonStr = mappingText.startsWith('```') diff --git a/src/services/form-documents/extraction.ts b/src/services/form-documents/extraction.ts index d322b1c47..1124cad9f 100644 --- a/src/services/form-documents/extraction.ts +++ b/src/services/form-documents/extraction.ts @@ -1,6 +1,8 @@ import { createAmazonBedrock } from '@ai-sdk/amazon-bedrock' import { fromNodeProviderChain } from '@aws-sdk/credential-providers' import { generateText } from 'ai' +import type { ActivityStore } from '../activity' +import { trackLlmCall } from '../activity' import type { ExtractionExemplar } from '../extraction' import type { PolicyChunk, PolicyRetriever } from '../rag' import type { CacheStore } from '../storage' @@ -61,6 +63,8 @@ export interface BedrockExtractorOptions { model?: string exemplars?: ExtractionExemplar[] maxOutputTokens?: number + /** Optional activity store for tracking LLM usage. */ + activityStore?: ActivityStore /** * Sampling temperature for Step 1 (the extraction prompt). When * undefined, the underlying provider default is used. Setting `0` @@ -265,6 +269,7 @@ ${exemplarSection}Guidelines: // Step 1: Extract DataCollectionSpec + confidence from PDF // Use generateText + manual JSON parsing because generateObject's // tool-use mode returns empty objects on Bedrock. + const startTime = Date.now() const extraction = await generateText({ model: bedrock(model), maxOutputTokens: options?.maxOutputTokens ?? 32768, @@ -288,6 +293,16 @@ ${exemplarSection}Guidelines: }, ], }) + if (options?.activityStore) { + trackLlmCall(options.activityStore, { + userId: extractionOptions?.userId, + projectId: extractionOptions?.slug, + operation: 'extraction', + model, + usage: extraction.usage, + durationMs: Date.now() - startTime, + }) + } const { spec, confidence } = parseJsonResponse( extraction.text, @@ -296,10 +311,25 @@ ${exemplarSection}Guidelines: // Step 2: Generate default FormSpec from extracted spec const bedrockModel = bedrock(model) - const formSpec = await generateFormSpec(bedrockModel, spec) + const formSpec = await generateFormSpec( + bedrockModel, + spec, + options?.activityStore, + extractionOptions?.userId, + extractionOptions?.slug, + model, + ) // Step 3: Enumerate PDF AcroForm fields and map to spec fieldNames - const fieldMapping = await mapAcroFormFields(bedrockModel, pdf, spec) + const fieldMapping = await mapAcroFormFields( + bedrockModel, + pdf, + spec, + options?.activityStore, + extractionOptions?.userId, + extractionOptions?.slug, + model, + ) return { spec, diff --git a/src/services/form-documents/tool-use-extraction.ts b/src/services/form-documents/tool-use-extraction.ts index b441e82c9..b4bb579ec 100644 --- a/src/services/form-documents/tool-use-extraction.ts +++ b/src/services/form-documents/tool-use-extraction.ts @@ -11,6 +11,8 @@ import { createAmazonBedrock } from '@ai-sdk/amazon-bedrock' import { fromNodeProviderChain } from '@aws-sdk/credential-providers' import { generateText, stepCountIs } from 'ai' +import type { ActivityStore } from '../activity' +import { trackLlmCall } from '../activity' import type { PdfExtractor } from './extraction' import { generateFormSpec, mapAcroFormFields } from './extraction-steps' import { extractionTools, reconstructSpec } from './extraction-tools' @@ -20,6 +22,8 @@ const DEFAULT_MODEL = 'us.anthropic.claude-sonnet-4-20250514-v1:0' export interface ToolUseExtractorOptions { model?: string + /** Optional activity store for tracking LLM usage. */ + activityStore?: ActivityStore } export function createToolUsePdfExtractor( @@ -50,6 +54,7 @@ export function createToolUsePdfExtractor( // Step 1: Extract DataCollectionSpec via tool-use (constrained generation) // maxSteps allows the model to make multiple rounds of tool calls — // large forms need 50+ calls which exceed a single response. + const startTime = Date.now() const response = await generateText({ model: bedrock(model), maxOutputTokens: 32768, @@ -84,6 +89,16 @@ Guidelines: }, ], }) + if (options?.activityStore) { + trackLlmCall(options.activityStore, { + userId: extractionOptions?.userId, + projectId: extractionOptions?.slug, + operation: 'extraction', + model, + usage: response.usage, + durationMs: Date.now() - startTime, + }) + } // Collect tool calls from all steps (model may take multiple rounds) const toolCalls = response.steps @@ -105,10 +120,25 @@ Guidelines: // Step 2: Generate default FormSpec from extracted spec (shared helper) const bedrockModel = bedrock(model) - const formSpec = await generateFormSpec(bedrockModel, spec) + const formSpec = await generateFormSpec( + bedrockModel, + spec, + options?.activityStore, + extractionOptions?.userId, + extractionOptions?.slug, + model, + ) // Step 3: Enumerate PDF AcroForm fields and map to spec fieldNames (shared helper) - const fieldMapping = await mapAcroFormFields(bedrockModel, pdf, spec) + const fieldMapping = await mapAcroFormFields( + bedrockModel, + pdf, + spec, + options?.activityStore, + extractionOptions?.userId, + extractionOptions?.slug, + model, + ) return { spec, diff --git a/src/services/form-documents/types.ts b/src/services/form-documents/types.ts index a52452607..6bed29d4a 100644 --- a/src/services/form-documents/types.ts +++ b/src/services/form-documents/types.ts @@ -41,6 +41,8 @@ export interface ExtractionOptions { * extractors. Optional because non-RAG variants ignore it. */ slug?: string + /** User login for activity tracking. */ + userId?: string } /** diff --git a/src/services/forms/filling-agent/bedrock.ts b/src/services/forms/filling-agent/bedrock.ts index ad0fa003d..de58fcbc7 100644 --- a/src/services/forms/filling-agent/bedrock.ts +++ b/src/services/forms/filling-agent/bedrock.ts @@ -3,6 +3,8 @@ import { createAmazonBedrock } from '@ai-sdk/amazon-bedrock' import { fromNodeProviderChain } from '@aws-sdk/credential-providers' import { generateText, tool } from 'ai' import { z } from 'zod' +import type { ActivityStore } from '../../activity' +import { trackLlmCall } from '../../activity' import { evaluateCondition } from '../resolver' import { buildSystemPrompt } from './system-prompt-builder' import type { @@ -17,6 +19,8 @@ const DEFAULT_MODEL = 'us.anthropic.claude-sonnet-4-20250514-v1:0' export interface BedrockFillingAgentOptions { model?: string region?: string + /** Optional activity store for tracking LLM usage. */ + activityStore?: ActivityStore } /** @@ -31,9 +35,11 @@ export interface BedrockFillingAgentOptions { export class BedrockFillingAgent implements FillingAgent { private model: string private bedrock: ReturnType + private activityStore?: ActivityStore constructor(options?: BedrockFillingAgentOptions) { this.model = options?.model ?? DEFAULT_MODEL + this.activityStore = options?.activityStore this.bedrock = createAmazonBedrock({ credentialProvider: fromNodeProviderChain(), @@ -65,6 +71,7 @@ export class BedrockFillingAgent implements FillingAgent { ) // Call LLM with tools - use tool() helper for proper schema format + const startTime = Date.now() const result = await generateText({ model: this.bedrock(this.model), system: systemPrompt, @@ -98,6 +105,16 @@ export class BedrockFillingAgent implements FillingAgent { }), }, }) + if (this.activityStore) { + trackLlmCall(this.activityStore, { + userId: context.userId, + projectId: context.projectId, + operation: 'filling', + model: this.model, + usage: result.usage, + durationMs: Date.now() - startTime, + }) + } console.log( '[BedrockFillingAgent] Result:', diff --git a/src/services/forms/filling-agent/types.ts b/src/services/forms/filling-agent/types.ts index e63219c7b..e9f2c9ab7 100644 --- a/src/services/forms/filling-agent/types.ts +++ b/src/services/forms/filling-agent/types.ts @@ -20,6 +20,10 @@ export interface FillingContext { groups: RequirementGroup[] collectedFields: Record messages: ConversationMessage[] + /** User login for activity tracking. */ + userId?: string + /** Project identifier for activity tracking. */ + projectId?: string } export interface FillingTurn { diff --git a/src/services/forms/shaping/bedrock-shaper.ts b/src/services/forms/shaping/bedrock-shaper.ts index de8426f75..9f199b660 100644 --- a/src/services/forms/shaping/bedrock-shaper.ts +++ b/src/services/forms/shaping/bedrock-shaper.ts @@ -1,6 +1,8 @@ import { createAmazonBedrock } from '@ai-sdk/amazon-bedrock' import { fromIni, fromNodeProviderChain } from '@aws-sdk/credential-providers' import { generateText } from 'ai' +import type { ActivityStore } from '../../activity' +import { trackLlmCall } from '../../activity' import type { Command } from './commands' import { commandTools } from './tools' import type { FormShaper, ShapingRequest, ShapingResult } from './types' @@ -57,6 +59,8 @@ ${previous} export interface BedrockShaperOptions { model?: string + /** Optional activity store for tracking LLM usage. */ + activityStore?: ActivityStore } export function createBedrockFormShaper( @@ -74,12 +78,23 @@ export function createBedrockFormShaper( return { async shape(request: ShapingRequest): Promise { const model = options?.model ?? DEFAULT_MODEL + const startTime = Date.now() const response = await generateText({ model: bedrock(model), maxOutputTokens: 4096, tools: commandTools, messages: [{ role: 'user', content: buildPrompt(request) }], }) + if (options?.activityStore) { + trackLlmCall(options.activityStore, { + userId: request.userId, + projectId: request.projectId, + operation: 'shaping', + model, + usage: response.usage, + durationMs: Date.now() - startTime, + }) + } const commands: Command[] = [] for (const call of response.toolCalls ?? []) { diff --git a/src/services/forms/shaping/registry.ts b/src/services/forms/shaping/registry.ts index ba20a97d0..b7e109f4b 100644 --- a/src/services/forms/shaping/registry.ts +++ b/src/services/forms/shaping/registry.ts @@ -1,4 +1,5 @@ import { StrategyRegistry } from '../../../shared/strategy-registry' +import type { ActivityStore } from '../../activity' import { HAIKU_MODEL_ID, OPUS_MODEL_ID, @@ -8,11 +9,16 @@ import { createBedrockFormShaper } from './bedrock-shaper' import { withValidationRetry } from './retry' import type { FormShaper } from './types' -function registeredShaper(model: string): FormShaper { - return withValidationRetry(createBedrockFormShaper({ model })) +function registeredShaper( + model: string, + activityStore?: ActivityStore, +): FormShaper { + return withValidationRetry(createBedrockFormShaper({ model, activityStore })) } -export function createShapingRegistry(): StrategyRegistry { +export function createShapingRegistry( + activityStore?: ActivityStore, +): StrategyRegistry { const registry = new StrategyRegistry() registry.register({ @@ -26,7 +32,7 @@ export function createShapingRegistry(): StrategyRegistry { catalogPath: '/catalog/experiments/shaping-model-comparison/sonnet', modelId: SONNET_MODEL_ID, }, - create: () => registeredShaper(SONNET_MODEL_ID), + create: () => registeredShaper(SONNET_MODEL_ID, activityStore), }) registry.register({ @@ -40,7 +46,7 @@ export function createShapingRegistry(): StrategyRegistry { catalogPath: '/catalog/experiments/shaping-model-comparison/haiku', modelId: HAIKU_MODEL_ID, }, - create: () => registeredShaper(HAIKU_MODEL_ID), + create: () => registeredShaper(HAIKU_MODEL_ID, activityStore), }) registry.register({ @@ -54,7 +60,7 @@ export function createShapingRegistry(): StrategyRegistry { catalogPath: '/catalog/experiments/shaping-model-comparison/opus', modelId: OPUS_MODEL_ID, }, - create: () => registeredShaper(OPUS_MODEL_ID), + create: () => registeredShaper(OPUS_MODEL_ID, activityStore), }) registry.setDefault('bedrock-sonnet') diff --git a/src/services/forms/shaping/types.ts b/src/services/forms/shaping/types.ts index 5ba963860..c8f806830 100644 --- a/src/services/forms/shaping/types.ts +++ b/src/services/forms/shaping/types.ts @@ -8,6 +8,10 @@ export interface ShapingRequest { intent: string state: ProjectState previousAttempt?: { commands: Command[]; feedback: string } + /** User login for activity tracking. */ + userId?: string + /** Project identifier for activity tracking. */ + projectId?: string } export interface ShapingResult { diff --git a/src/services/projects/project-service.ts b/src/services/projects/project-service.ts index 73bacba16..9261b3629 100644 --- a/src/services/projects/project-service.ts +++ b/src/services/projects/project-service.ts @@ -276,7 +276,7 @@ export function createProjectService( const { variantId, modelId } = extraction.resolveVariant(author) const extractor = extraction.resolveExtractor(variantId) extractor - .extract(pdf) + .extract(pdf, { userId: author }) .then(async (result) => { // Initial extraction lands on an "import" branch so the owner can // iterate before publishing to main via the review workflow. diff --git a/test/cli/activity-cli.test.ts b/test/cli/activity-cli.test.ts new file mode 100644 index 000000000..cfb0a1990 --- /dev/null +++ b/test/cli/activity-cli.test.ts @@ -0,0 +1,57 @@ +import { afterEach, beforeEach, describe, expect, it } from 'bun:test' +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { + formatEvents, + formatSummary, +} from '../../src/entrypoints/cli/commands/activity' +import { createActivityStore } from '../../src/services/activity' + +describe('activity CLI formatting', () => { + let tmpDir: string + + beforeEach(() => { + tmpDir = mkdtempSync(join(tmpdir(), 'activity-cli-')) + }) + + afterEach(() => { + rmSync(tmpDir, { recursive: true }) + }) + + it('formatSummary produces readable output', () => { + const store = createActivityStore(join(tmpDir, 'activity.sqlite')) + store.track({ + eventType: 'llm_call', + userId: 'daniel', + projectId: 'snap-form', + operation: 'extraction', + metadata: { + model: 'us.anthropic.claude-sonnet-4-20250514-v1:0', + inputTokens: 1000, + outputTokens: 500, + durationMs: 1500, + }, + }) + + const now = Math.floor(Date.now() / 1000) + const summary = store.summarize({ from: now - 86400, to: now + 60 }) + const output = formatSummary(summary) + + expect(output).toContain('daniel') + expect(output).toContain('extraction') + expect(output).toContain('snap-form') + expect(output).toContain('$') + }) + + it('formatEvents produces readable output', () => { + const store = createActivityStore(join(tmpDir, 'activity.sqlite')) + store.track({ eventType: 'sign_in', userId: 'daniel' }) + + const events = store.query({}) + const output = formatEvents(events) + + expect(output).toContain('sign_in') + expect(output).toContain('daniel') + }) +}) diff --git a/test/services/activity/activity-store.test.ts b/test/services/activity/activity-store.test.ts new file mode 100644 index 000000000..f765d520d --- /dev/null +++ b/test/services/activity/activity-store.test.ts @@ -0,0 +1,128 @@ +import { afterEach, beforeEach, describe, expect, it } from 'bun:test' +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { createActivityStore } from '../../../src/services/activity' + +describe('ActivityStore', () => { + let tmpDir: string + let store: ReturnType + + beforeEach(() => { + tmpDir = mkdtempSync(join(tmpdir(), 'activity-test-')) + store = createActivityStore(join(tmpDir, 'activity.sqlite')) + }) + + afterEach(() => { + rmSync(tmpDir, { recursive: true }) + }) + + it('tracks an event and retrieves it via query', () => { + store.track({ + eventType: 'llm_call', + userId: 'daniel', + projectId: 'snap-form', + operation: 'extraction', + metadata: { + model: 'us.anthropic.claude-sonnet-4-20250514-v1:0', + inputTokens: 1500, + outputTokens: 800, + durationMs: 2000, + }, + }) + + const events = store.query({}) + expect(events).toHaveLength(1) + expect(events[0].eventType).toBe('llm_call') + expect(events[0].userId).toBe('daniel') + expect(events[0].projectId).toBe('snap-form') + expect(events[0].operation).toBe('extraction') + expect(events[0].metadata.inputTokens).toBe(1500) + }) + + it('filters by userId', () => { + store.track({ eventType: 'sign_in', userId: 'daniel' }) + store.track({ eventType: 'sign_in', userId: 'maya' }) + + const events = store.query({ userId: 'daniel' }) + expect(events).toHaveLength(1) + expect(events[0].userId).toBe('daniel') + }) + + it('filters by time range', () => { + store.track({ eventType: 'sign_in', userId: 'daniel' }) + + const future = Math.floor(Date.now() / 1000) + 3600 + const events = store.query({ from: future }) + expect(events).toHaveLength(0) + }) + + it('filters by eventType and operation', () => { + store.track({ + eventType: 'llm_call', + userId: 'daniel', + operation: 'extraction', + }) + store.track({ + eventType: 'llm_call', + userId: 'daniel', + operation: 'shaping', + }) + store.track({ eventType: 'project_created', userId: 'daniel' }) + + const extractions = store.query({ + eventType: 'llm_call', + operation: 'extraction', + }) + expect(extractions).toHaveLength(1) + }) + + it('respects limit', () => { + store.track({ eventType: 'sign_in', userId: 'a' }) + store.track({ eventType: 'sign_in', userId: 'b' }) + store.track({ eventType: 'sign_in', userId: 'c' }) + + const events = store.query({ limit: 2 }) + expect(events).toHaveLength(2) + }) + + it('summarizes events for a time period', () => { + const now = Math.floor(Date.now() / 1000) + store.track({ + eventType: 'llm_call', + userId: 'daniel', + projectId: 'snap-form', + operation: 'extraction', + metadata: { + model: 'us.anthropic.claude-sonnet-4-20250514-v1:0', + inputTokens: 1000, + outputTokens: 500, + durationMs: 1500, + }, + }) + store.track({ + eventType: 'llm_call', + userId: 'maya', + projectId: 'i-9', + operation: 'shaping', + metadata: { + model: 'us.anthropic.claude-sonnet-4-20250514-v1:0', + inputTokens: 2000, + outputTokens: 1000, + durationMs: 3000, + }, + }) + store.track({ + eventType: 'project_created', + userId: 'daniel', + projectId: 'snap-form', + }) + + const summary = store.summarize({ from: now - 60, to: now + 60 }) + expect(summary.totalEvents).toBe(3) + expect(summary.estimatedCost).toBeGreaterThan(0) + expect(summary.byUser).toHaveLength(2) + expect(summary.byOperation).toHaveLength(2) + expect(summary.byProject).toHaveLength(2) + }) +}) diff --git a/test/services/activity/instrumentation.test.ts b/test/services/activity/instrumentation.test.ts new file mode 100644 index 000000000..b221a883d --- /dev/null +++ b/test/services/activity/instrumentation.test.ts @@ -0,0 +1,59 @@ +import { afterEach, beforeEach, describe, expect, it } from 'bun:test' +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import type { ActivityStore } from '../../../src/services/activity' +import { + createActivityStore, + trackLlmCall, +} from '../../../src/services/activity' + +describe('LLM instrumentation helper', () => { + let tmpDir: string + let store: ActivityStore + + beforeEach(() => { + tmpDir = mkdtempSync(join(tmpdir(), 'activity-instr-')) + store = createActivityStore(join(tmpDir, 'activity.sqlite')) + }) + + afterEach(() => { + rmSync(tmpDir, { recursive: true }) + }) + + it('trackLlmCall records model, tokens, and duration', () => { + trackLlmCall(store, { + userId: 'daniel', + projectId: 'snap-form', + operation: 'extraction', + model: 'us.anthropic.claude-sonnet-4-20250514-v1:0', + usage: { inputTokens: 1500, outputTokens: 800 }, + durationMs: 2340, + }) + + const events = store.query({}) + expect(events).toHaveLength(1) + expect(events[0].eventType).toBe('llm_call') + expect(events[0].metadata.model).toBe( + 'us.anthropic.claude-sonnet-4-20250514-v1:0', + ) + expect(events[0].metadata.inputTokens).toBe(1500) + expect(events[0].metadata.outputTokens).toBe(800) + expect(events[0].metadata.durationMs).toBe(2340) + }) + + it('trackLlmCall works without optional userId and projectId', () => { + trackLlmCall(store, { + operation: 'shaping', + model: 'us.anthropic.claude-sonnet-4-20250514-v1:0', + usage: { inputTokens: 500, outputTokens: 200 }, + durationMs: 1100, + }) + + const events = store.query({}) + expect(events).toHaveLength(1) + expect(events[0].userId).toBeNull() + expect(events[0].projectId).toBeNull() + expect(events[0].operation).toBe('shaping') + }) +}) diff --git a/test/services/activity/non-llm-events.test.ts b/test/services/activity/non-llm-events.test.ts new file mode 100644 index 000000000..e0979395f --- /dev/null +++ b/test/services/activity/non-llm-events.test.ts @@ -0,0 +1,52 @@ +import { afterEach, beforeEach, describe, expect, it } from 'bun:test' +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import type { ActivityStore } from '../../../src/services/activity' +import { createActivityStore } from '../../../src/services/activity' + +describe('Non-LLM event tracking', () => { + let tmpDir: string + let store: ActivityStore + + beforeEach(() => { + tmpDir = mkdtempSync(join(tmpdir(), 'activity-nonllm-')) + store = createActivityStore(join(tmpDir, 'activity.sqlite')) + }) + + afterEach(() => { + rmSync(tmpDir, { recursive: true }) + }) + + it('tracks project_created event', () => { + store.track({ + eventType: 'project_created', + userId: 'daniel', + projectId: 'snap-form', + }) + + const events = store.query({ eventType: 'project_created' }) + expect(events).toHaveLength(1) + expect(events[0].userId).toBe('daniel') + expect(events[0].projectId).toBe('snap-form') + }) + + it('tracks sign_in event', () => { + store.track({ eventType: 'sign_in', userId: 'daniel' }) + + const events = store.query({ eventType: 'sign_in' }) + expect(events).toHaveLength(1) + expect(events[0].userId).toBe('daniel') + }) + + it('tracks pdf_uploaded event', () => { + store.track({ + eventType: 'pdf_uploaded', + userId: 'daniel', + projectId: 'i-9', + }) + + const events = store.query({ eventType: 'pdf_uploaded' }) + expect(events).toHaveLength(1) + }) +}) diff --git a/test/services/activity/pricing.test.ts b/test/services/activity/pricing.test.ts new file mode 100644 index 000000000..cd0929ef9 --- /dev/null +++ b/test/services/activity/pricing.test.ts @@ -0,0 +1,38 @@ +import { describe, expect, it } from 'bun:test' +import { BEDROCK_PRICING, estimateCost } from '../../../src/services/activity' + +describe('estimateCost', () => { + it('computes cost for known model', () => { + const cost = estimateCost( + 'us.anthropic.claude-sonnet-4-20250514-v1:0', + 1_000_000, + 1_000_000, + ) + // $3 input + $15 output = $18 + expect(cost).toBe(18.0) + }) + + it('returns 0 for unknown model', () => { + const cost = estimateCost('unknown-model', 1000, 1000) + expect(cost).toBe(0) + }) + + it('handles small token counts correctly', () => { + const cost = estimateCost( + 'us.anthropic.claude-sonnet-4-20250514-v1:0', + 1000, + 500, + ) + // (1000 * 3 + 500 * 15) / 1_000_000 = 10500 / 1_000_000 = 0.0105 + expect(cost).toBeCloseTo(0.0105, 6) + }) + + it('pricing table has expected model entry', () => { + expect( + BEDROCK_PRICING['us.anthropic.claude-sonnet-4-20250514-v1:0'], + ).toEqual({ + input: 3.0, + output: 15.0, + }) + }) +})