Skip to content

Commit e5b4e3f

Browse files
authored
fix(logs): keep checked trace content when the display projection budget runs out (#9682)
* fix(logs): keep checked trace content when the display projection budget runs out - Project each log display value on its own, so one oversized or ref-backed value withholds only itself instead of the error, message and every other envelope field - When the whole-trace content walk exhausts its node budget, keep the spans that already passed it and withhold only the rest; every other projection failure still withholds the whole trace - Put the bulky block dumps after the trace spans so budget pressure drops them first - Attribute projection warnings to their execution, workflow, workspace, mode, span and field * fix(logs): bound the withheld-span fallback by the trace's structure budget and make envelope span ids unguessable * fix(logs): resume the withheld-span fallback from the retained spans' structure budget * improvement(logs): name the projection warning's read/write phase explicitly
1 parent f0cfdfa commit e5b4e3f

4 files changed

Lines changed: 424 additions & 63 deletions

File tree

‎apps/sim/lib/logs/execution/trace-secret-projection.test.ts‎

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1019,6 +1019,13 @@ describe('projectTraceSpansForSecrets', () => {
10191019
}
10201020

10211021
expect(warnMock).toHaveBeenCalledWith('Omitting trace content that could not be sanitized', {
1022+
executionId: 'execution-1',
1023+
workflowId: 'workflow-1',
1024+
workspaceId: 'workspace-1',
1025+
phase: 'write',
1026+
spanId: 'span-1',
1027+
spanType: 'function',
1028+
field: 'output',
10221029
failure: { name: 'SyntaxError' },
10231030
})
10241031
})
@@ -1048,3 +1055,145 @@ describe('projectTraceSpansForSecrets', () => {
10481055
expect(cursor?.children).toEqual([])
10491056
})
10501057
})
1058+
1059+
describe('projectTraceSpansForSecrets whole-trace content budget', () => {
1060+
const SECRET_REGISTRY = createRegistry([
1061+
{ plaintext: 'top-secret', replacement: '{{API_SECRET}}' },
1062+
])
1063+
1064+
/** A span whose output alone fits the content budget but two-and-a-half of which do not. */
1065+
function heavySpan(id: string, marker: string, overrides: Partial<TraceSpan> = {}): TraceSpan {
1066+
return createSpan({
1067+
id,
1068+
output: { marker, items: Array.from({ length: 40_000 }, (_, index) => `row-${index}`) },
1069+
...overrides,
1070+
})
1071+
}
1072+
1073+
it('keeps the spans checked before the budget ran out and withholds the rest', async () => {
1074+
const projected = await projectTraceSpansForSecrets(
1075+
[
1076+
heavySpan('span-a', 'first top-secret'),
1077+
heavySpan('span-b', 'second'),
1078+
heavySpan('span-c', 'third top-secret'),
1079+
createSpan({ id: 'span-d', output: { marker: 'fourth top-secret' } }),
1080+
],
1081+
{ registry: SECRET_REGISTRY, store: STORE, allowLargeValueWrites: false }
1082+
)
1083+
1084+
expect(projected.map((span) => span.id)).toEqual(['span-a', 'span-b', 'span-c', 'span-d'])
1085+
expect(projected[0].output?.marker).toBe('first {{API_SECRET}}')
1086+
expect(projected[0]).not.toHaveProperty('contentWithheld')
1087+
expect(projected[1].output?.marker).toBe('second')
1088+
for (const withheld of projected.slice(2)) {
1089+
expect(withheld).not.toHaveProperty('output')
1090+
expect(withheld.contentWithheld).toBe(true)
1091+
}
1092+
expect(JSON.stringify(projected)).not.toContain('top-secret')
1093+
expect(warnMock).toHaveBeenCalledWith(
1094+
'Trace content exceeded the projection budget; withholding the remaining spans',
1095+
expect.objectContaining({
1096+
executionId: 'execution-1',
1097+
phase: 'read',
1098+
retainedSpanCount: 2,
1099+
withheldSpanCount: 2,
1100+
})
1101+
)
1102+
})
1103+
1104+
it('still withholds every span when a later span fails for any reason but the budget', async () => {
1105+
const projected = await projectTraceSpansForSecrets(
1106+
[
1107+
createSpan({ id: 'span-a', output: { marker: 'first' } }),
1108+
createSpan({
1109+
id: 'span-b',
1110+
toolCalls: Array.from({ length: 100_001 }, (_, index) => ({
1111+
name: `tool-${index}`,
1112+
duration: 0,
1113+
startTime: '2026-07-01T00:00:00.000Z',
1114+
endTime: '2026-07-01T00:00:00.000Z',
1115+
status: 'success',
1116+
})),
1117+
}),
1118+
],
1119+
{ registry: SECRET_REGISTRY, store: STORE, allowLargeValueWrites: false }
1120+
)
1121+
1122+
expect(projected[0]).not.toHaveProperty('output')
1123+
expect(projected[0].contentWithheld).toBe(true)
1124+
expect(warnMock).toHaveBeenCalledWith(
1125+
'Trace secret projection failed; retaining structural spans only',
1126+
expect.objectContaining({ executionId: 'execution-1', phase: 'read' })
1127+
)
1128+
})
1129+
1130+
it('keeps the retained spans and the withheld fallback within one structure budget', async () => {
1131+
const emptyChildren = (prefix: string) =>
1132+
Array.from({ length: 60_000 }, (_, index) => createSpan({ id: `${prefix}-${index}` }))
1133+
const projected = await projectTraceSpansForSecrets(
1134+
[
1135+
heavySpan('span-a', 'first', { children: emptyChildren('a') }),
1136+
createSpan({
1137+
id: 'span-b',
1138+
output: { items: Array.from({ length: 70_000 }, (_, index) => `row-${index}`) },
1139+
children: emptyChildren('b'),
1140+
}),
1141+
],
1142+
{ registry: SECRET_REGISTRY, store: STORE, allowLargeValueWrites: false }
1143+
)
1144+
const countSpans = (spans: TraceSpan[]): number =>
1145+
spans.reduce((total, span) => total + 1 + countSpans(span.children ?? []), 0)
1146+
1147+
expect(projected[0].output?.marker).toBe('first')
1148+
expect(projected[1]).not.toHaveProperty('output')
1149+
expect(countSpans(projected)).toBeLessThanOrEqual(100_000)
1150+
})
1151+
1152+
it('keeps the structure a whole-trace fallback would for the span that crossed the budget', async () => {
1153+
const toolCall = (index: number) => ({
1154+
name: `tool-${index}`,
1155+
duration: 0,
1156+
startTime: '2026-07-01T00:00:00.000Z',
1157+
endTime: '2026-07-01T00:00:00.000Z',
1158+
status: 'success' as const,
1159+
})
1160+
const projected = await projectTraceSpansForSecrets(
1161+
[
1162+
heavySpan('span-a', 'first', {
1163+
children: Array.from({ length: 30_000 }, (_, index) => createSpan({ id: `a-${index}` })),
1164+
}),
1165+
createSpan({
1166+
id: 'span-b',
1167+
output: { items: Array.from({ length: 70_000 }, (_, index) => `row-${index}`) },
1168+
toolCalls: Array.from({ length: 50_000 }, (_, index) => toolCall(index)),
1169+
}),
1170+
],
1171+
{ registry: SECRET_REGISTRY, store: STORE, allowLargeValueWrites: false }
1172+
)
1173+
1174+
expect(projected[0].output?.marker).toBe('first')
1175+
expect(projected[1]).not.toHaveProperty('output')
1176+
expect(projected[1].toolCalls).toHaveLength(50_000)
1177+
})
1178+
1179+
it('withholds the whole top-level span whose descendant crossed the budget', async () => {
1180+
const projected = await projectTraceSpansForSecrets(
1181+
[
1182+
heavySpan('span-a', 'first'),
1183+
createSpan({
1184+
id: 'span-b',
1185+
output: { marker: 'parent top-secret' },
1186+
children: [heavySpan('child-1', 'child one'), heavySpan('child-2', 'child two')],
1187+
}),
1188+
],
1189+
{ registry: SECRET_REGISTRY, store: STORE, allowLargeValueWrites: false }
1190+
)
1191+
1192+
expect(projected[0].output?.marker).toBe('first')
1193+
expect(projected[1]).not.toHaveProperty('output')
1194+
expect(projected[1].contentWithheld).toBe(true)
1195+
expect(projected[1].children?.map((child) => child.id)).toEqual(['child-1', 'child-2'])
1196+
expect(projected[1].children?.every((child) => !Object.hasOwn(child, 'output'))).toBe(true)
1197+
expect(JSON.stringify(projected)).not.toContain('top-secret')
1198+
})
1199+
})

0 commit comments

Comments
 (0)