feat(activity): add activity tracking and LLM cost visibility

infrastructure/nixos/flake.nix

@@ -22,6 +22,7 @@
         ./modules/homepage.nix
         ./modules/notify.nix
         ./modules/notify-failure.nix
+        ./modules/activity-digest.nix
         ./modules/secrets.nix
       ];
     };

infrastructure/nixos/modules/activity-digest.nix

@@ -0,0 +1,33 @@
+{ config, pkgs, lib, ... }:
+
+{
+  systemd.services.forms-lab-activity-digest = {
+    description = "Forms Lab Daily Activity Digest";
+    after = [ "forms-lab-notify.service" ];
+    requires = [ "forms-lab-notify.service" ];
+
+    path = with pkgs; [ bun ];
+
+    serviceConfig = {
+      Type = "oneshot";
+      User = "forms-lab";
+      Group = "forms-lab";
+      WorkingDirectory = "/srv/forms-lab/main";
+    };
+
+    script = ''
+      export ACTIVITY_DB_PATH="/srv/forms-lab/data/activity.sqlite"
+      exec ${pkgs.bun}/bin/bun run src/entrypoints/cli/main.ts activity digest
+    '';
+  };
+
+  systemd.timers.forms-lab-activity-digest = {
+    description = "Daily Forms Lab Activity Digest";
+    wantedBy = [ "timers.target" ];
+
+    timerConfig = {
+      OnCalendar = "*-*-* 06:00:00";
+      Persistent = true;
+    };
+  };
+}

src/entrypoints/app/routes/auth/index.tsx

@@ -1,5 +1,6 @@
 import { Hono } from 'hono'
 import { deleteCookie, setCookie } from 'hono/cookie'
+import type { ActivityStore } from '../../../../services/activity'
 import { Layout } from '../../../../design-system/components/flex-layout'
 import type { AccessStore, UserStore } from '../../../../services/auth'
 import {
@@ -21,6 +22,7 @@ import {
 export function createAuthRoutes(
   userStore: UserStore,
   accessStore: AccessStore,
+  options?: { activityStore?: ActivityStore },
 ): Hono {
   const auth = new Hono()
 
@@ -210,6 +212,11 @@ export function createAuthRoutes(
         path: '/',
       })
 
+      options?.activityStore?.track({
+        eventType: 'sign_in',
+        userId: ghUser.login,
+      })
+
       const returnTo = state.returnTo || resolveUrl('/')
       // Prevent open redirect — only allow internal paths
       const safeReturnTo =

src/entrypoints/app/routes/forms/index.tsx

@@ -675,6 +675,8 @@ export function createFormRouter(deps: FormRouterDeps) {
           groups: page.groups,
           collectedFields: session.fields,
           messages: [],
+          userId: user.login,
+          projectId: session.specId,
         },
         null,
       )
@@ -842,6 +844,8 @@ export function createFormRouter(deps: FormRouterDeps) {
           groups: page.groups,
           collectedFields: session.fields,
           messages,
+          userId: user.login,
+          projectId: session.specId,
         },
         userMessage,
       )

src/entrypoints/app/routes/owner/edit/index.tsx

@@ -233,6 +233,8 @@ export function createEditRoutes(
         intent: body.intent,
         state,
         previousAttempt: body.previousAttempt,
+        userId: user.login,
+        projectId: slug,
       })
 
       return c.json({

src/entrypoints/app/server.tsx

@@ -8,6 +8,7 @@ import {
   loadFixturePdf,
 } from '../../../fixtures/index'
 import { Layout } from '../../design-system/components/flex-layout'
+import { createActivityStore } from '../../services/activity'
 import { createAccessStore, createUserStore } from '../../services/auth'
 import type { DataCollectionSpec } from '../../services/data-collection'
 import { createExtractorRegistry } from '../../services/extraction'
@@ -71,6 +72,10 @@ mkdirSync(dirname(projectDbPath), { recursive: true })
 mkdirSync(dirname(cacheDbPath), { recursive: true })
 mkdirSync(reposPath, { recursive: true })
 
+const activityDbPath = process.env.ACTIVITY_DB_PATH ?? 'data/activity.sqlite'
+mkdirSync(dirname(activityDbPath), { recursive: true })
+const activityStore = createActivityStore(activityDbPath)
+
 const projectStore = createProjectStore(projectDbPath)
 const cacheStore = createCacheStore(cacheDbPath)
 const userStore = createUserStore(projectDbPath)
@@ -80,8 +85,8 @@ const formProjectRepo = createFormProjectRepo(reposPath)
 // Variant registries: one per task. Each user's preferred variant is
 // resolved against these at call time so a settings change takes effect
 // on the next extraction without restarting the process.
-const extractionRegistry = createExtractorRegistry()
-const shapingRegistry = createShapingRegistry()
+const extractionRegistry = createExtractorRegistry(activityStore)
+const shapingRegistry = createShapingRegistry(activityStore)
 const fillingRegistry = createFillingRegistry()
 const mappingRegistry = createMappingRegistry()
 const authoringCriteriaRegistry = createAuthoringCriteriaRegistry()
@@ -132,7 +137,7 @@ const conversationGateway = new SqliteConversationGateway(formsDbPath)
 const fillingAgent =
   process.env.USE_SCRIPTED_AGENT === 'true'
     ? new ScriptedFillingAgent()
-    : new BedrockFillingAgent()
+    : new BedrockFillingAgent({ activityStore })
 
 /**
  * Adapter: resolve a DataCollectionSpec id to (owner, slug, spec, formSpec)
@@ -288,7 +293,7 @@ app.use(
 )
 
 // Mount auth routes
-app.route('/auth', createAuthRoutes(userStore, accessStore))
+app.route('/auth', createAuthRoutes(userStore, accessStore, { activityStore }))
 
 // Mount admin routes (requireAdmin is applied inside createAdminRoutes)
 app.use('/admin/*', requireAuth(accessStore))
@@ -402,6 +407,16 @@ app.post('/new', async (c) => {
       const pdf = Buffer.from(await file.arrayBuffer())
       const name = file.name.replace(/\.pdf$/i, '')
       const project = await projectService.createProject(name, pdf, user)
+      activityStore.track({
+        eventType: 'pdf_uploaded',
+        userId: user.login,
+        projectId: project.slug,
+      })
+      activityStore.track({
+        eventType: 'project_created',
+        userId: user.login,
+        projectId: project.slug,
+      })
       return c.redirect(
         resolveUrl(`/${user.login}/${project.slug}/edit/import`),
       )
@@ -433,6 +448,11 @@ app.post('/new', async (c) => {
         user,
         { corpusSlug: corpusMetadata.slug },
       )
+      activityStore.track({
+        eventType: 'project_created',
+        userId: user.login,
+        projectId: project.slug,
+      })
       return c.redirect(
         resolveUrl(`/${user.login}/${project.slug}/edit/import`),
       )
@@ -456,6 +476,11 @@ app.post('/new', async (c) => {
     const pdf = loadFixturePdf(fixture)
     const name = fixture.name
     const project = await projectService.createProject(name, pdf, user)
+    activityStore.track({
+      eventType: 'project_created',
+      userId: user.login,
+      projectId: project.slug,
+    })
     return c.redirect(resolveUrl(`/${user.login}/${project.slug}`))
   } catch (err) {
     console.error('Error creating project:', err)

src/entrypoints/cli/commands/activity.ts

@@ -0,0 +1,170 @@
+import { mkdirSync } from 'node:fs'
+import { dirname } from 'node:path'
+import type { ActivityRow, UsageSummary } from '../../../services/activity'
+import { createActivityStore } from '../../../services/activity'
+import { notifyEvent } from '../../../services/notifications'
+
+const DB_PATH = process.env.ACTIVITY_DB_PATH ?? 'data/activity.sqlite'
+
+function getStore() {
+  mkdirSync(dirname(DB_PATH), { recursive: true })
+  return createActivityStore(DB_PATH)
+}
+
+export function formatSummary(summary: UsageSummary): string {
+  const lines: string[] = []
+  lines.push(
+    `Total: ${summary.totalEvents} events | ~$${summary.estimatedCost.toFixed(2)} estimated cost`,
+  )
+  lines.push('')
+
+  if (summary.byUser.length > 0) {
+    lines.push('By user:')
+    for (const entry of summary.byUser) {
+      const tokens =
+        entry.tokens > 1000
+          ? `${(entry.tokens / 1000).toFixed(0)}K`
+          : `${entry.tokens}`
+      lines.push(
+        `  ${entry.userId}: ${entry.events} events, ${tokens} tokens, ~$${entry.cost.toFixed(2)}`,
+      )
+    }
+    lines.push('')
+  }
+
+  if (summary.byOperation.length > 0) {
+    lines.push('By operation:')
+    for (const entry of summary.byOperation) {
+      lines.push(
+        `  ${entry.operation}: ${entry.events} calls, ~$${entry.cost.toFixed(2)}`,
+      )
+    }
+    lines.push('')
+  }
+
+  if (summary.byProject.length > 0) {
+    lines.push('Top projects:')
+    for (const entry of summary.byProject) {
+      lines.push(
+        `  ${entry.projectId}: ${entry.events} events, ~$${entry.cost.toFixed(2)}`,
+      )
+    }
+  }
+
+  return lines.join('\n')
+}
+
+export function formatEvents(events: ActivityRow[]): string {
+  if (events.length === 0) return 'No events found.'
+
+  const lines: string[] = []
+  for (const event of events) {
+    const time = new Date(event.timestamp * 1000).toISOString().slice(0, 19)
+    const parts = [
+      time,
+      event.eventType,
+      event.userId ?? '-',
+      event.projectId ?? '-',
+      event.operation ?? '-',
+    ]
+    lines.push(parts.join('\t'))
+  }
+  return lines.join('\n')
+}
+
+function parseNumericArg(
+  args: string[],
+  flag: string,
+  defaultValue: number,
+): number {
+  const idx = args.indexOf(flag)
+  if (idx !== -1 && args[idx + 1]) {
+    return Number.parseInt(args[idx + 1], 10)
+  }
+  return defaultValue
+}
+
+function parseStringArg(args: string[], flag: string): string | undefined {
+  const idx = args.indexOf(flag)
+  if (idx !== -1 && args[idx + 1]) {
+    return args[idx + 1]
+  }
+  return undefined
+}
+
+export async function activity(args: string[]): Promise<number> {
+  const subcommand = args[0]
+
+  if (!subcommand || subcommand === '--help') {
+    console.log('Usage: bun run cli activity <subcommand>\n')
+    console.log('Subcommands:')
+    console.log('  summary     Show usage summary (default: today)')
+    console.log('  events      List activity events')
+    console.log('  digest      Post daily digest to Slack')
+    console.log('\nOptions:')
+    console.log('  --days <n>      Number of days to look back (default: 1)')
+    console.log('  --user <login>  Filter by user')
+    console.log('  --project <slug> Filter by project')
+    console.log('  --operation <op> Filter by operation')
+    console.log('  --limit <n>     Max events to show (default: 50)')
+    return 0
+  }
+
+  const store = getStore()
+  const now = Math.floor(Date.now() / 1000)
+
+  if (subcommand === 'summary') {
+    const days = parseNumericArg(args, '--days', 1)
+    const from = now - days * 86400
+    const summary = store.summarize({ from, to: now })
+    console.log(
+      `\nActivity Summary (last ${days} day${days > 1 ? 's' : ''}):\n`,
+    )
+    console.log(formatSummary(summary))
+    return 0
+  }
+
+  if (subcommand === 'events') {
+    const days = parseNumericArg(args, '--days', 1)
+    const limit = parseNumericArg(args, '--limit', 50)
+    const events = store.query({
+      from: now - days * 86400,
+      to: now,
+      userId: parseStringArg(args, '--user'),
+      projectId: parseStringArg(args, '--project'),
+      operation: parseStringArg(args, '--operation'),
+      limit,
+    })
+    console.log(formatEvents(events))
+    return 0
+  }
+
+  if (subcommand === 'digest') {
+    const todayMidnight = new Date()
+    todayMidnight.setHours(0, 0, 0, 0)
+    const to = Math.floor(todayMidnight.getTime() / 1000)
+    const from = to - 86400
+
+    const summary = store.summarize({ from, to })
+    const dateStr = new Date(from * 1000).toISOString().slice(0, 10)
+
+    if (summary.totalEvents === 0) {
+      console.log(`No activity on ${dateStr}. Skipping digest.`)
+      return 0
+    }
+
+    const details = formatSummary(summary)
+    await notifyEvent({
+      type: 'activity-digest',
+      title: `Forms Lab Daily Usage — ${dateStr}`,
+      status: 'info',
+      details,
+    })
+
+    console.log(`Digest posted for ${dateStr}`)
+    return 0
+  }
+
+  console.error(`Unknown subcommand: ${subcommand}`)
+  return 1
+}

src/entrypoints/cli/main.ts

@@ -1,3 +1,4 @@
+import { activity } from './commands/activity'
 import { deploy } from './commands/deploy'
 import { evaluate } from './commands/evaluate'
 import { extract } from './commands/extract'
@@ -19,6 +20,11 @@ export interface Command {
 }
 
 const commands: Command[] = [
+  {
+    name: 'activity',
+    description: 'View activity reports and post usage digests',
+    run: activity,
+  },
   {
     name: 'sync-stories',
     description: 'Sync user stories from GitHub Issues',

src/services/activity/index.ts

@@ -0,0 +1,15 @@
+// Public interface for the activity service.
+// External imports (other services, entrypoints, design-system) MUST come
+// through this file. Enforced by test/architecture/dependency-rule.test.ts.
+
+export type { LlmCallMetrics } from './instrumentation'
+export { trackLlmCall } from './instrumentation'
+export { BEDROCK_PRICING, estimateCost } from './pricing'
+export { createActivityStore } from './store'
+export type {
+  ActivityEvent,
+  ActivityQuery,
+  ActivityRow,
+  ActivityStore,
+  UsageSummary,
+} from './types'