fix(webapp): bound trace API ClickHouse reads by run completion

Bound the time window the run trace API reads for finished runs, so
retrieving
the trace of an old run no longer scans every day of stored events since
the
run was created.

For a finished run, events stored more than 7 days after the run
completed are
no longer read. In rare cases, a span that finishes more than 7 days
after the
run, or a cancellation the run would inherit from a parent recorded that
late,
is not reflected: the span is returned in its last recorded state. Runs
that
are still executing are not affected.

Mono-RevId: 549b041b446c38ab61e8ce68c809a8d376df3dd6

Author: Mariefay

apps/webapp/app/routes/api.v1.runs.$runId.trace.ts

@@ -4,6 +4,7 @@ import { z } from "zod";
 import { $replica } from "~/db.server";
 import { anyResource, createLoaderApiRoute } from "~/services/routeBuilders/apiBuilder.server";
 import { getEventRepositoryForStore } from "~/v3/eventRepository/index.server";
+import { getTraceInsertedAtEnd } from "~/v3/eventRepository/traceInsertedAtBound";
 import { getTaskEventStoreTableForRun } from "~/v3/taskEventStore.server";
 import { findRunByIdWithMollifierFallback } from "~/v3/mollifier/readFallback.server";
 import { buildSyntheticTraceBody } from "~/v3/mollifier/syntheticApiResponses.server";
@@ -99,7 +100,8 @@ export const loader = createLoaderApiRoute(
       run.traceId,
       run.spanId,
       run.createdAt,
-      run.completedAt ?? undefined
+      run.completedAt ?? undefined,
+      { insertedAtEnd: getTraceInsertedAtEnd(run) }
     );
 
     if (!traceSummary) {

apps/webapp/app/v3/eventRepository/clickhouseEventRepository.server.ts

@@ -1765,7 +1765,7 @@ export class ClickhouseEventRepository implements IEventRepository {
     anchorSpanId: string;
     startCreatedAt: Date;
     endCreatedAt?: Date;
-    options?: { includeDebugLogs?: boolean };
+    options?: { includeDebugLogs?: boolean; insertedAtEnd?: Date };
     limit?: number;
   }): Promise<{
     records: TaskEventDetailedSummaryV1Result[];
@@ -2546,7 +2546,7 @@ export class ClickhouseEventRepository implements IEventRepository {
     traceId: string;
     startCreatedAt?: Date;
     endCreatedAt?: Date;
-    options?: { includeDebugLogs?: boolean };
+    options?: { includeDebugLogs?: boolean; insertedAtEnd?: Date };
     spanIds?: string[];
     parentSpanIds?: string[];
     limit?: number;
@@ -2578,6 +2578,12 @@ export class ClickhouseEventRepository implements IEventRepository {
         queryBuilder.where("inserted_at >= {insertedAtStart: DateTime64(3)}", {
           insertedAtStart: convertDateToClickhouseDateTime(startCreatedAtWithBuffer),
         });
+
+        if (options?.insertedAtEnd) {
+          queryBuilder.where("inserted_at <= {insertedAtEnd: DateTime64(3)}", {
+            insertedAtEnd: convertDateToClickhouseDateTime(options.insertedAtEnd),
+          });
+        }
       }
     }
 
@@ -2724,7 +2730,7 @@ export class ClickhouseEventRepository implements IEventRepository {
     anchorSpanId: string,
     startCreatedAt: Date,
     endCreatedAt?: Date,
-    options?: { includeDebugLogs?: boolean }
+    options?: { includeDebugLogs?: boolean; insertedAtEnd?: Date }
   ): Promise<TraceDetailedSummary | undefined> {
     const limit = this._config.maximumTraceDetailedSummaryViewCount;
 

apps/webapp/app/v3/eventRepository/eventRepository.server.ts

@@ -500,7 +500,7 @@ export class EventRepository implements IEventRepository {
     anchorSpanId: string,
     startCreatedAt: Date,
     endCreatedAt?: Date,
-    options?: { includeDebugLogs?: boolean }
+    options?: { includeDebugLogs?: boolean; insertedAtEnd?: Date }
   ): Promise<TraceDetailedSummary | undefined> {
     const events = await this.taskEventStore.findDetailedTraceEvents(
       storeTable,

apps/webapp/app/v3/eventRepository/eventRepository.types.ts

@@ -430,7 +430,7 @@ export interface IEventRepository {
     anchorSpanId: string,
     startCreatedAt: Date,
     endCreatedAt?: Date,
-    options?: { includeDebugLogs?: boolean }
+    options?: { includeDebugLogs?: boolean; insertedAtEnd?: Date }
   ): Promise<TraceDetailedSummary | undefined>;
 
   // Streams a trace's events in start_time order, one at a time, without ever

apps/webapp/app/v3/eventRepository/traceInsertedAtBound.ts

@@ -0,0 +1,18 @@
+import type { TaskRunStatus } from "@trigger.dev/database";
+import { isFinalRunStatus } from "~/v3/taskStatus";
+
+// Spans in a run's subtree can finish after the run, so their final rows land later.
+const TRACE_INSERTED_AT_END_BUFFER_MS = 7 * 24 * 60 * 60 * 1000;
+
+export function getTraceInsertedAtEnd(run: {
+  status: TaskRunStatus;
+  completedAt: Date | null;
+  updatedAt: Date;
+}): Date | undefined {
+  if (!isFinalRunStatus(run.status)) {
+    return undefined;
+  }
+
+  const finishedAt = run.completedAt ?? run.updatedAt;
+  return new Date(finishedAt.getTime() + TRACE_INSERTED_AT_END_BUFFER_MS);
+}

apps/webapp/test/getTraceDetailedSubtreeSummary.integration.test.ts

@@ -426,3 +426,170 @@ describe("getTraceDetailedSubtreeSummary", () => {
     INTEGRATION_TIMEOUT_MS
   );
 });
+
+describe("getTraceDetailedSubtreeSummary insertedAtEnd bound", () => {
+  const DAY_MS = 24 * 60 * 60 * 1000;
+
+  clickhouseTest(
+    "bounds full and descendant fetches by insertedAtEnd but not ancestor fetches",
+    async ({ clickhouseContainer }) => {
+      const clickhouse = new ClickHouse({
+        url: clickhouseContainer.getConnectionUrl(),
+        logLevel: "warn",
+      });
+      const repository = new ClickhouseEventRepository({
+        clickhouse,
+        version: "v2",
+        maximumTraceDetailedSummaryViewCount: TRACE_ROW_LIMIT,
+      });
+
+      const environmentId = "env_trace_bound";
+      const baseMs = Date.now() - 30 * DAY_MS;
+      const expiresAt = convertDateToClickhouseDateTime(new Date(baseMs + 400 * DAY_MS));
+
+      function row(
+        traceId: string,
+        spanId: string,
+        parentSpanId: string,
+        startOffsetMs: number,
+        insertedOffsetMs: number,
+        status: string
+      ): TaskEventV2Input {
+        return {
+          environment_id: environmentId,
+          organization_id: "org_trace_bound",
+          project_id: "proj_trace_bound",
+          task_identifier: "trace-bound-test-task",
+          run_id: `run_${spanId}`,
+          start_time: formatClickhouseStartTime(baseMs, startOffsetMs),
+          inserted_at: convertDateToClickhouseDateTime(new Date(baseMs + insertedOffsetMs)),
+          duration: "1000000000",
+          trace_id: traceId,
+          span_id: spanId,
+          parent_span_id: parentSpanId,
+          message: spanId,
+          kind: "SPAN",
+          status,
+          attributes: {},
+          metadata: "{}",
+          expires_at: expiresAt,
+        };
+      }
+
+      // Full fetch: the requested run is the trace root and completes at +10s.
+      const fullTrace = "c".repeat(32);
+      const fullCompletedMs = 10_000;
+      // Subtree walk: the requested run starts after its parent, so the full fetch can't re-root.
+      const walkTrace = "d".repeat(32);
+      const walkCreatedMs = 100_000;
+      const walkCompletedMs = 110_000;
+      // Ancestor: the parent's cancellation is stored long after the bound.
+      const ancestorTrace = "e".repeat(32);
+
+      const [insertError] = await clickhouse.taskEventsV2.insert(
+        [
+          row(fullTrace, "fullroot", "", 0, 0, "PARTIAL"),
+          row(fullTrace, "fullroot", "", 0, fullCompletedMs, "OK"),
+          row(fullTrace, "fullinside", "fullroot", 1_000, 1_000, "PARTIAL"),
+          row(fullTrace, "fullinside", "fullroot", 1_000, fullCompletedMs + 6 * DAY_MS, "OK"),
+          row(fullTrace, "fulllate", "fullroot", 2_000, 2_000, "PARTIAL"),
+          row(fullTrace, "fulllate", "fullroot", 2_000, fullCompletedMs + 8 * DAY_MS, "OK"),
+
+          row(walkTrace, "walkroot", "", 0, 0, "PARTIAL"),
+          row(walkTrace, "walkanchor", "walkroot", walkCreatedMs, walkCreatedMs, "PARTIAL"),
+          row(walkTrace, "walkanchor", "walkroot", walkCreatedMs, walkCompletedMs, "OK"),
+          row(
+            walkTrace,
+            "walkinside",
+            "walkanchor",
+            walkCreatedMs + 1_000,
+            walkCreatedMs + 1_000,
+            "PARTIAL"
+          ),
+          row(
+            walkTrace,
+            "walkinside",
+            "walkanchor",
+            walkCreatedMs + 1_000,
+            walkCompletedMs + DAY_MS,
+            "OK"
+          ),
+          row(
+            walkTrace,
+            "walklate",
+            "walkanchor",
+            walkCreatedMs + 2_000,
+            walkCreatedMs + 2_000,
+            "PARTIAL"
+          ),
+          row(
+            walkTrace,
+            "walklate",
+            "walkanchor",
+            walkCreatedMs + 2_000,
+            walkCompletedMs + 9 * DAY_MS,
+            "OK"
+          ),
+
+          row(ancestorTrace, "ancroot", "", 0, 0, "PARTIAL"),
+          row(ancestorTrace, "ancroot", "", 0, 20 * DAY_MS, "CANCELLED"),
+          row(ancestorTrace, "ancanchor", "ancroot", walkCreatedMs, walkCreatedMs, "PARTIAL"),
+        ],
+        { clickhouse_settings: { async_insert: 0 } }
+      );
+      expect(insertError).toBeNull();
+
+      const fullCompletedAt = new Date(baseMs + fullCompletedMs);
+      const fullBounded = await repository.getTraceDetailedSubtreeSummary(
+        "taskEvent",
+        environmentId,
+        fullTrace,
+        "fullroot",
+        new Date(baseMs),
+        fullCompletedAt,
+        { insertedAtEnd: new Date(fullCompletedAt.getTime() + 7 * DAY_MS) }
+      );
+      expect(findSpan(fullBounded?.rootSpan, "fullinside")?.data.isPartial).toBe(false);
+      expect(findSpan(fullBounded?.rootSpan, "fulllate")?.data.isPartial).toBe(true);
+
+      const fullUnbounded = await repository.getTraceDetailedSubtreeSummary(
+        "taskEvent",
+        environmentId,
+        fullTrace,
+        "fullroot",
+        new Date(baseMs),
+        fullCompletedAt
+      );
+      expect(findSpan(fullUnbounded?.rootSpan, "fullinside")?.data.isPartial).toBe(false);
+      expect(findSpan(fullUnbounded?.rootSpan, "fulllate")?.data.isPartial).toBe(false);
+
+      const walkCompletedAt = new Date(baseMs + walkCompletedMs);
+      const walk = await repository.getTraceDetailedSubtreeSummary(
+        "taskEvent",
+        environmentId,
+        walkTrace,
+        "walkanchor",
+        new Date(baseMs + walkCreatedMs),
+        walkCompletedAt,
+        { insertedAtEnd: new Date(walkCompletedAt.getTime() + 7 * DAY_MS) }
+      );
+      expect(walk?.rootSpan.id).toBe("walkanchor");
+      expect(walk?.rootSpan.parentId).toBe("walkroot");
+      expect(findSpan(walk?.rootSpan, "walkinside")?.data.isPartial).toBe(false);
+      expect(findSpan(walk?.rootSpan, "walklate")?.data.isPartial).toBe(true);
+
+      const ancestor = await repository.getTraceDetailedSubtreeSummary(
+        "taskEvent",
+        environmentId,
+        ancestorTrace,
+        "ancanchor",
+        new Date(baseMs + walkCreatedMs),
+        undefined,
+        { insertedAtEnd: new Date(baseMs + 7 * DAY_MS) }
+      );
+      expect(ancestor?.rootSpan.id).toBe("ancanchor");
+      expect(ancestor?.rootSpan.data.isCancelled).toBe(true);
+    },
+    INTEGRATION_TIMEOUT_MS
+  );
+});

apps/webapp/test/traceInsertedAtBound.test.ts

@@ -0,0 +1,48 @@
+import { describe, expect, it } from "vitest";
+import { getTraceInsertedAtEnd } from "~/v3/eventRepository/traceInsertedAtBound";
+
+const DAY_MS = 24 * 60 * 60 * 1000;
+const COMPLETED_AT = new Date("2026-05-13T20:13:30.310Z");
+const UPDATED_AT = new Date("2026-05-14T09:00:00.000Z");
+
+describe("getTraceInsertedAtEnd", () => {
+  it("bounds a completed run at completedAt + 7 days", () => {
+    const end = getTraceInsertedAtEnd({
+      status: "COMPLETED_WITH_ERRORS",
+      completedAt: COMPLETED_AT,
+      updatedAt: UPDATED_AT,
+    });
+
+    expect(end).toEqual(new Date(COMPLETED_AT.getTime() + 7 * DAY_MS));
+  });
+
+  it("falls back to updatedAt + 7 days for a final run with no completedAt", () => {
+    const end = getTraceInsertedAtEnd({
+      status: "CANCELED",
+      completedAt: null,
+      updatedAt: UPDATED_AT,
+    });
+
+    expect(end).toEqual(new Date(UPDATED_AT.getTime() + 7 * DAY_MS));
+  });
+
+  it("does not bound a run that is not in a final status", () => {
+    const end = getTraceInsertedAtEnd({
+      status: "EXECUTING",
+      completedAt: null,
+      updatedAt: UPDATED_AT,
+    });
+
+    expect(end).toBeUndefined();
+  });
+
+  it("does not bound a non-final run even if completedAt is set", () => {
+    const end = getTraceInsertedAtEnd({
+      status: "WAITING_TO_RESUME",
+      completedAt: COMPLETED_AT,
+      updatedAt: UPDATED_AT,
+    });
+
+    expect(end).toBeUndefined();
+  });
+});

docs/management/runs/retrieve-trace.mdx

@@ -6,3 +6,5 @@ openapi: "v3-openapi GET /api/v1/runs/{runId}/trace"
 Returns the OpenTelemetry trace subtree for the run you request. The response `trace.rootSpan` is that run's span — not necessarily the trace-wide root — with its descendant spans nested under `children`.
 
 For a child or nested run inside a large trace, this endpoint scopes the tree to that run so you still get a useful subtree even when the full trace has more spans than the platform can return in one response.
+
+For a finished run, spans that finish more than 7 days after the run are returned in their last recorded state.