From 2d08f728f72d7b834b38754673079ca0aef5df4e Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Mon, 5 Oct 2026 12:28:46 +0200 Subject: [PATCH 01/12] feat(server-utils): Map Mastra classifier evaluations to gen_ai.evaluate spans Co-Authored-By: Claude Opus 5.5 --- .../tracing/mastra/scenario-classifier.mjs | 49 +++++++++++++++++++ .../suites/tracing/mastra/test.ts | 38 ++++++++++++++ .../server-utils/src/ai/mastra/constants.ts | 2 + packages/server-utils/src/ai/mastra/types.ts | 3 ++ packages/server-utils/src/ai/mastra/utils.ts | 21 ++++++-- .../ai/lib/tracing/mastra-exporter.test.ts | 22 +++++++++ 6 files changed, 132 insertions(+), 3 deletions(-) create mode 100644 dev-packages/node-integration-tests/suites/tracing/mastra/scenario-classifier.mjs diff --git a/dev-packages/node-integration-tests/suites/tracing/mastra/scenario-classifier.mjs b/dev-packages/node-integration-tests/suites/tracing/mastra/scenario-classifier.mjs new file mode 100644 index 000000000000..ca0015249cde --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/mastra/scenario-classifier.mjs @@ -0,0 +1,49 @@ +import * as Sentry from '@sentry/node'; +import { Mastra } from '@mastra/core'; +import { Classifier } from '@mastra/core/classifier'; +import { Observability } from '@mastra/observability'; +import { SentryMastraExporter } from '@sentry/node'; + +// Inlined `EvaluationModelV4` mock: the ESM/CJS runner copies only this file into its temp dir. +const evaluationModel = { + specificationVersion: 'v4', + provider: 'typesafe-ai.evaluation', + modelId: 'jev-latest', + supportedQuestionTypes: ['choice', 'score', 'boolean'], + async doEvaluate() { + return { + answers: { urgent: { type: 'boolean', probability: 0.9 } }, + usage: { inputTokens: 30, outputTokens: 2 }, + warnings: [], + }; + }, +}; + +async function run() { + const classifier = new Classifier({ + id: 'request-classifier', + model: evaluationModel, + questions: { urgent: { type: 'boolean', instructions: 'Does this request need an immediate response?' } }, + }); + + const mastra = new Mastra({ + classifiers: { classifier }, + logger: false, + observability: new Observability({ + configs: { + default: { + serviceName: 'mastra-test', + exporters: [new SentryMastraExporter()], + }, + }, + }), + }); + + await Sentry.startSpan({ op: 'function', name: 'mastra-test' }, async () => { + await mastra.getClassifier('classifier').evaluate({ state: { message: 'I was charged twice.' } }); + }); + + await mastra.observability.shutdown(); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/mastra/test.ts b/dev-packages/node-integration-tests/suites/tracing/mastra/test.ts index 281362c074f4..75e0ecb48e08 100644 --- a/dev-packages/node-integration-tests/suites/tracing/mastra/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/mastra/test.ts @@ -43,6 +43,14 @@ const MASTRA_NESTING_DEPENDENCIES = { }, }; +// `Classifier` (the `@mastra/core/classifier` entry) ships in newer `@mastra/core` releases only. +const MASTRA_CLASSIFIER_DEPENDENCIES = { + additionalDependencies: { + '@mastra/core': '1.74.0', + '@mastra/observability': '1.18.3', + }, +}; + conditionalTest({ min: 22 })('Mastra integration', () => { afterAll(() => { cleanupChildProcesses(); @@ -207,6 +215,36 @@ conditionalTest({ min: 22 })('Mastra integration', () => { MASTRA_DEPENDENCIES, ); + createEsmAndCjsTests( + __dirname, + 'scenario-classifier.mjs', + 'instrument.mjs', + (createRunner, test) => { + test('maps a classifier evaluation to a gen_ai.evaluate span', async () => { + await createRunner() + .expect({ transaction: { transaction: 'mastra-test' } }) + .expect({ + span: container => { + expect(container.items.map(span => span.name)).toEqual(['evaluate jev-latest']); + + const evaluateSpan = container.items[0]!; + expect(evaluateSpan.attributes['sentry.op'].value).toBe('gen_ai.evaluate'); + expect(evaluateSpan.attributes['sentry.origin'].value).toBe('auto.ai.mastra'); + expect(evaluateSpan.attributes[GEN_AI_OPERATION_NAME].value).toBe('evaluate'); + expect(evaluateSpan.attributes[GEN_AI_REQUEST_MODEL].value).toBe('jev-latest'); + expect(evaluateSpan.attributes[GEN_AI_PROVIDER_NAME].value).toBe('typesafe-ai.evaluation'); + expect(evaluateSpan.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(30); + expect(evaluateSpan.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(2); + expect(evaluateSpan.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(32); + }, + }) + .start() + .completed(); + }); + }, + MASTRA_CLASSIFIER_DEPENDENCIES, + ); + createEsmAndCjsTests( __dirname, 'scenario-auto.mjs', diff --git a/packages/server-utils/src/ai/mastra/constants.ts b/packages/server-utils/src/ai/mastra/constants.ts index 26a05910c45d..66168f00088c 100644 --- a/packages/server-utils/src/ai/mastra/constants.ts +++ b/packages/server-utils/src/ai/mastra/constants.ts @@ -1,4 +1,5 @@ import { GEN_AI_CHAT, GEN_AI_EMBEDDINGS, GEN_AI_EXECUTE_TOOL, GEN_AI_INVOKE_AGENT } from '@sentry/conventions/op'; +import { getGenAiSpanOp } from '../core/utils'; import type { MastraSpanType } from './types'; export const MASTRA_INTEGRATION_NAME = 'Mastra' as const; @@ -36,6 +37,7 @@ export const SPAN_TYPE_OPS: Readonly = new Set([ diff --git a/packages/server-utils/src/ai/mastra/types.ts b/packages/server-utils/src/ai/mastra/types.ts index 35d125644346..86d3d518028a 100644 --- a/packages/server-utils/src/ai/mastra/types.ts +++ b/packages/server-utils/src/ai/mastra/types.ts @@ -15,6 +15,7 @@ export type MastraSpanType = | 'provider_tool_call' | 'client_tool_call' | 'rag_embedding' + | 'classifier_evaluation' | (string & {}); export type MastraTracingEventType = 'span_started' | 'span_updated' | 'span_ended'; @@ -58,6 +59,8 @@ export interface MastraSpanAttributes { stopSequences?: string[]; seed?: number; }; + // classifier_evaluation + modelId?: string; // tool_call / mcp_tool_call / provider_tool_call / client_tool_call toolDescription?: string; // agent_run / workflow_run diff --git a/packages/server-utils/src/ai/mastra/utils.ts b/packages/server-utils/src/ai/mastra/utils.ts index 0cdb20e1b80f..bf1840051755 100644 --- a/packages/server-utils/src/ai/mastra/utils.ts +++ b/packages/server-utils/src/ai/mastra/utils.ts @@ -62,9 +62,11 @@ export function getOperation(spanType: MastraSpanType): { op: string; operationN export function getSpanName(span: MastraExportedSpan): string { const operationName = getOperation(span.type)?.operationName ?? span.type; const identifier = - MODEL_SPAN_TYPES.has(span.type) || span.type === 'rag_embedding' - ? span.attributes?.model - : (span.entityName ?? span.entityId); + span.type === 'classifier_evaluation' + ? span.attributes?.modelId + : MODEL_SPAN_TYPES.has(span.type) || span.type === 'rag_embedding' + ? span.attributes?.model + : (span.entityName ?? span.entityId); return identifier ? `${operationName} ${identifier}` : operationName; } @@ -157,6 +159,10 @@ export function getSpanAttributes(span: MastraExportedSpan, options: AttributeRe addToolAttributes(attributes, span, options); } + if (span.type === 'classifier_evaluation') { + addEvaluationAttributes(attributes, span); + } + if (span.errorInfo) { attributes[ERROR_TYPE] = span.errorInfo.name ?? span.errorInfo.id; } @@ -243,6 +249,15 @@ function addToolAttributes( } } +/** Mastra's `Classifier` sets no input or output on this span, so only the model and usage are known. */ +function addEvaluationAttributes(attributes: SpanAttributes, span: MastraExportedSpan): void { + const attrs = span.attributes ?? {}; + + attributes[GEN_AI_REQUEST_MODEL] = attrs.modelId; + attributes[GEN_AI_PROVIDER_NAME] = attrs.provider; + Object.assign(attributes, getUsageAttributes(attrs.usage)); +} + function responseText(output: unknown): string | undefined { if (typeof output === 'string') { return output; diff --git a/packages/server-utils/test/ai/lib/tracing/mastra-exporter.test.ts b/packages/server-utils/test/ai/lib/tracing/mastra-exporter.test.ts index 5265272ffa0f..3095d7aec0f0 100644 --- a/packages/server-utils/test/ai/lib/tracing/mastra-exporter.test.ts +++ b/packages/server-utils/test/ai/lib/tracing/mastra-exporter.test.ts @@ -121,6 +121,28 @@ describe('SentryMastraExporter', () => { expect(json.data['gen_ai.response.tool_calls']).toBeUndefined(); }); + it('maps classifier evaluations to `evaluate {model}` spans', async () => { + const attributes = { classifierId: 'sentiment', modelId: 'jev-1', provider: 'typesafe.evaluation' }; + const span = makeSpan({ id: 'eval-1', type: 'classifier_evaluation', name: "classifier evaluate: 'sentiment'" }); + await run( + started({ ...span, attributes }), + ended({ ...span, attributes: { ...attributes, usage: { inputTokens: 30, outputTokens: 2 } } }), + ); + + const json = spanToStaticSpanJSON(endedSpans[0]!); + expect(json.description).toBe('evaluate jev-1'); + expect(json.data).toMatchObject({ + 'sentry.op': 'gen_ai.evaluate', + 'sentry.origin': 'auto.ai.mastra', + 'gen_ai.operation.name': 'evaluate', + 'gen_ai.request.model': 'jev-1', + 'gen_ai.provider.name': 'typesafe.evaluation', + 'gen_ai.usage.input_tokens': 30, + 'gen_ai.usage.output_tokens': 2, + 'gen_ai.usage.total_tokens': 32, + }); + }); + it('records the agent-level prompt and response as gen_ai messages', async () => { const span = makeSpan({ entityName: 'agent', From 6dc33ca5bcd3db376839588a2ea65f3bf2af83f3 Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Tue, 6 Oct 2026 11:13:27 +0200 Subject: [PATCH 02/12] Extract Mastra span identifier helper --- packages/server-utils/src/ai/mastra/utils.ts | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/packages/server-utils/src/ai/mastra/utils.ts b/packages/server-utils/src/ai/mastra/utils.ts index bf1840051755..5fa9f1b62316 100644 --- a/packages/server-utils/src/ai/mastra/utils.ts +++ b/packages/server-utils/src/ai/mastra/utils.ts @@ -61,16 +61,21 @@ export function getOperation(spanType: MastraSpanType): { op: string; operationN */ export function getSpanName(span: MastraExportedSpan): string { const operationName = getOperation(span.type)?.operationName ?? span.type; - const identifier = - span.type === 'classifier_evaluation' - ? span.attributes?.modelId - : MODEL_SPAN_TYPES.has(span.type) || span.type === 'rag_embedding' - ? span.attributes?.model - : (span.entityName ?? span.entityId); + const identifier = getSpanIdentifier(span); return identifier ? `${operationName} ${identifier}` : operationName; } +function getSpanIdentifier(span: MastraExportedSpan): string | undefined { + if (span.type === 'classifier_evaluation') { + return span.attributes?.modelId; + } + if (MODEL_SPAN_TYPES.has(span.type) || span.type === 'rag_embedding') { + return span.attributes?.model; + } + return span.entityName ?? span.entityId; +} + function serialize(value: unknown): string | undefined { if (value === undefined || value === null) { return undefined; From c110bf2de53b69e9104545113161ebefaa136956 Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Wed, 7 Oct 2026 09:43:17 +0200 Subject: [PATCH 03/12] Use GEN_AI_EVALUATE op from conventions --- packages/server-utils/src/ai/mastra/constants.ts | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/packages/server-utils/src/ai/mastra/constants.ts b/packages/server-utils/src/ai/mastra/constants.ts index 66168f00088c..ab55142a2bf5 100644 --- a/packages/server-utils/src/ai/mastra/constants.ts +++ b/packages/server-utils/src/ai/mastra/constants.ts @@ -1,5 +1,10 @@ -import { GEN_AI_CHAT, GEN_AI_EMBEDDINGS, GEN_AI_EXECUTE_TOOL, GEN_AI_INVOKE_AGENT } from '@sentry/conventions/op'; -import { getGenAiSpanOp } from '../core/utils'; +import { + GEN_AI_CHAT, + GEN_AI_EMBEDDINGS, + GEN_AI_EVALUATE, + GEN_AI_EXECUTE_TOOL, + GEN_AI_INVOKE_AGENT, +} from '@sentry/conventions/op'; import type { MastraSpanType } from './types'; export const MASTRA_INTEGRATION_NAME = 'Mastra' as const; @@ -37,7 +42,7 @@ export const SPAN_TYPE_OPS: Readonly = new Set([ From 8824cd64b3b7b82dd419c1b6c3ab3db41580839b Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Wed, 7 Oct 2026 09:46:19 +0200 Subject: [PATCH 04/12] Use conventions constants in Mastra exporter tests --- .../ai/lib/tracing/mastra-exporter.test.ts | 137 +++++++++++------- 1 file changed, 85 insertions(+), 52 deletions(-) diff --git a/packages/server-utils/test/ai/lib/tracing/mastra-exporter.test.ts b/packages/server-utils/test/ai/lib/tracing/mastra-exporter.test.ts index 3095d7aec0f0..b7f3ee8d1635 100644 --- a/packages/server-utils/test/ai/lib/tracing/mastra-exporter.test.ts +++ b/packages/server-utils/test/ai/lib/tracing/mastra-exporter.test.ts @@ -1,4 +1,37 @@ import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { + ERROR_TYPE, + GEN_AI_AGENT_NAME, + GEN_AI_CONVERSATION_ID, + GEN_AI_INPUT_MESSAGES, + GEN_AI_OPERATION_NAME, + GEN_AI_OUTPUT_MESSAGES, + GEN_AI_PIPELINE_NAME, + GEN_AI_PROVIDER_NAME, + GEN_AI_REQUEST_MODEL, + GEN_AI_REQUEST_STOP_SEQUENCES, + GEN_AI_RESPONSE_MODEL, + GEN_AI_RESPONSE_TEXT, + GEN_AI_RESPONSE_TOOL_CALLS, + GEN_AI_SYSTEM_INSTRUCTIONS, + GEN_AI_TOOL_CALL_ARGUMENTS, + GEN_AI_TOOL_NAME, + GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS, + GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS, + GEN_AI_USAGE_INPUT_TOKENS, + GEN_AI_USAGE_OUTPUT_TOKENS, + GEN_AI_USAGE_REASONING_OUTPUT_TOKENS, + GEN_AI_USAGE_TOTAL_TOKENS, + SENTRY_OP, + SENTRY_ORIGIN, +} from '@sentry/conventions/attributes'; +import { + GEN_AI_CHAT, + GEN_AI_EMBEDDINGS, + GEN_AI_EVALUATE, + GEN_AI_EXECUTE_TOOL, + GEN_AI_INVOKE_AGENT, +} from '@sentry/conventions/op'; import type { Span } from '@sentry/core'; import { _INTERNAL_clearAiProviderSkips, @@ -81,11 +114,11 @@ describe('SentryMastraExporter', () => { const json = spanToStaticSpanJSON(endedSpans[0]!); expect(json.description).toBe('invoke_agent weather_agent'); - expect(json.data['sentry.op']).toBe('gen_ai.invoke_agent'); - expect(json.data['sentry.origin']).toBe('auto.ai.mastra'); - expect(json.data['gen_ai.operation.name']).toBe('invoke_agent'); - expect(json.data['gen_ai.agent.name']).toBe('weather_agent'); - expect(json.data['gen_ai.pipeline.name']).toBe('weather_agent'); + expect(json.data[SENTRY_OP]).toBe(GEN_AI_INVOKE_AGENT); + expect(json.data[SENTRY_ORIGIN]).toBe('auto.ai.mastra'); + expect(json.data[GEN_AI_OPERATION_NAME]).toBe('invoke_agent'); + expect(json.data[GEN_AI_AGENT_NAME]).toBe('weather_agent'); + expect(json.data[GEN_AI_PIPELINE_NAME]).toBe('weather_agent'); }); it('names generation spans `chat {model}` and maps usage to the current conventions', async () => { @@ -108,17 +141,17 @@ describe('SentryMastraExporter', () => { const json = spanToStaticSpanJSON(endedSpans[0]!); expect(json.description).toBe('chat gpt-5'); - expect(json.data['sentry.op']).toBe('gen_ai.chat'); - expect(json.data['gen_ai.provider.name']).toBe('openai'); - expect(json.data['gen_ai.usage.input_tokens']).toBe(10); - expect(json.data['gen_ai.usage.output_tokens']).toBe(4); - expect(json.data['gen_ai.usage.total_tokens']).toBe(14); - expect(json.data['gen_ai.usage.reasoning.output_tokens']).toBe(3); - expect(json.data['gen_ai.usage.cache_read.input_tokens']).toBe(2); - expect(json.data['gen_ai.usage.cache_creation.input_tokens']).toBe(5); + expect(json.data[SENTRY_OP]).toBe(GEN_AI_CHAT); + expect(json.data[GEN_AI_PROVIDER_NAME]).toBe('openai'); + expect(json.data[GEN_AI_USAGE_INPUT_TOKENS]).toBe(10); + expect(json.data[GEN_AI_USAGE_OUTPUT_TOKENS]).toBe(4); + expect(json.data[GEN_AI_USAGE_TOTAL_TOKENS]).toBe(14); + expect(json.data[GEN_AI_USAGE_REASONING_OUTPUT_TOKENS]).toBe(3); + expect(json.data[GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS]).toBe(2); + expect(json.data[GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS]).toBe(5); expect(json.data['gen_ai.usage.reasoning_tokens']).toBeUndefined(); - expect(json.data['gen_ai.response.text']).toBeUndefined(); - expect(json.data['gen_ai.response.tool_calls']).toBeUndefined(); + expect(json.data[GEN_AI_RESPONSE_TEXT]).toBeUndefined(); + expect(json.data[GEN_AI_RESPONSE_TOOL_CALLS]).toBeUndefined(); }); it('maps classifier evaluations to `evaluate {model}` spans', async () => { @@ -132,14 +165,14 @@ describe('SentryMastraExporter', () => { const json = spanToStaticSpanJSON(endedSpans[0]!); expect(json.description).toBe('evaluate jev-1'); expect(json.data).toMatchObject({ - 'sentry.op': 'gen_ai.evaluate', - 'sentry.origin': 'auto.ai.mastra', - 'gen_ai.operation.name': 'evaluate', - 'gen_ai.request.model': 'jev-1', - 'gen_ai.provider.name': 'typesafe.evaluation', - 'gen_ai.usage.input_tokens': 30, - 'gen_ai.usage.output_tokens': 2, - 'gen_ai.usage.total_tokens': 32, + [SENTRY_OP]: GEN_AI_EVALUATE, + [SENTRY_ORIGIN]: 'auto.ai.mastra', + [GEN_AI_OPERATION_NAME]: 'evaluate', + [GEN_AI_REQUEST_MODEL]: 'jev-1', + [GEN_AI_PROVIDER_NAME]: 'typesafe.evaluation', + [GEN_AI_USAGE_INPUT_TOKENS]: 30, + [GEN_AI_USAGE_OUTPUT_TOKENS]: 2, + [GEN_AI_USAGE_TOTAL_TOKENS]: 32, }); }); @@ -153,10 +186,10 @@ describe('SentryMastraExporter', () => { await run(started(span), ended(span)); const json = spanToStaticSpanJSON(endedSpans[0]!); - expect(json.data['gen_ai.input.messages']).toBe('[{"role":"user","content":"hi"}]'); - expect(json.data['gen_ai.output.messages']).toBe('{"text":"hello"}'); - expect(json.data['gen_ai.response.text']).toBe('hello'); - expect(json.data['gen_ai.system_instructions']).toBe('be brief'); + expect(json.data[GEN_AI_INPUT_MESSAGES]).toBe('[{"role":"user","content":"hi"}]'); + expect(json.data[GEN_AI_OUTPUT_MESSAGES]).toBe('{"text":"hello"}'); + expect(json.data[GEN_AI_RESPONSE_TEXT]).toBe('hello'); + expect(json.data[GEN_AI_SYSTEM_INSTRUCTIONS]).toBe('be brief'); }); it.each([ @@ -172,7 +205,7 @@ describe('SentryMastraExporter', () => { }); await run(started(span), ended(span)); - expect(spanToStaticSpanJSON(endedSpans[0]!).data['gen_ai.response.text']).toBe(expected); + expect(spanToStaticSpanJSON(endedSpans[0]!).data[GEN_AI_RESPONSE_TEXT]).toBe(expected); }); it.each([ @@ -180,28 +213,28 @@ describe('SentryMastraExporter', () => { 'workflow_run', { type: 'workflow_run' as const, entityName: 'math_workflow' }, 'invoke_agent math_workflow', - 'gen_ai.invoke_agent', + GEN_AI_INVOKE_AGENT, ], [ 'rag_embedding', { type: 'rag_embedding' as const, attributes: { model: 'text-embedding-3' } }, 'embeddings text-embedding-3', - 'gen_ai.embeddings', + GEN_AI_EMBEDDINGS, ], [ 'mcp_tool_call', { type: 'mcp_tool_call' as const, entityName: 'search' }, 'execute_tool search', - 'gen_ai.execute_tool', + GEN_AI_EXECUTE_TOOL, ], - ['entityId when entityName is missing', { entityId: 'agent-42' }, 'invoke_agent agent-42', 'gen_ai.invoke_agent'], + ['entityId when entityName is missing', { entityId: 'agent-42' }, 'invoke_agent agent-42', GEN_AI_INVOKE_AGENT], ])('names a %s span', async (_label, overrides, description, op) => { const span = makeSpan(overrides); await run(started(span), ended(span)); const json = spanToStaticSpanJSON(endedSpans[0]!); expect(json.description).toBe(description); - expect(json.data['sentry.op']).toBe(op); + expect(json.data[SENTRY_OP]).toBe(op); }); it('omits inputs and outputs when recording is disabled', async () => { @@ -210,8 +243,8 @@ describe('SentryMastraExporter', () => { await run(started(span), ended(span)); const json = spanToStaticSpanJSON(endedSpans[0]!); - expect(json.data['gen_ai.input.messages']).toBeUndefined(); - expect(json.data['gen_ai.output.messages']).toBeUndefined(); + expect(json.data[GEN_AI_INPUT_MESSAGES]).toBeUndefined(); + expect(json.data[GEN_AI_OUTPUT_MESSAGES]).toBeUndefined(); expect(JSON.stringify(json.data)).not.toContain('pii'); }); @@ -248,9 +281,9 @@ describe('SentryMastraExporter', () => { const chatJson = json.find(span => span.description === 'chat gpt-5'); expect(toolJson).toBeDefined(); expect(chatJson).toBeDefined(); - expect(toolJson!.data['sentry.op']).toBe('gen_ai.execute_tool'); - expect(toolJson!.data['gen_ai.tool.name']).toBe('get_weather'); - expect(toolJson!.data['gen_ai.tool.call.arguments']).toBe('{"city":"Berlin"}'); + expect(toolJson!.data[SENTRY_OP]).toBe(GEN_AI_EXECUTE_TOOL); + expect(toolJson!.data[GEN_AI_TOOL_NAME]).toBe('get_weather'); + expect(toolJson!.data[GEN_AI_TOOL_CALL_ARGUMENTS]).toBe('{"city":"Berlin"}'); expect(toolJson!.parent_span_id).toBe(chatJson!.span_id); }); @@ -273,10 +306,10 @@ describe('SentryMastraExporter', () => { .map(span => spanToStaticSpanJSON(span)) .find(span => span.description === 'invoke_agent agent'); expect(agentJson).toBeDefined(); - expect(agentJson!.data['gen_ai.usage.input_tokens']).toBe(10); - expect(agentJson!.data['gen_ai.usage.output_tokens']).toBe(4); - expect(agentJson!.data['gen_ai.usage.total_tokens']).toBe(14); - expect(agentJson!.data['gen_ai.response.model']).toBe('gpt-5-2026'); + expect(agentJson!.data[GEN_AI_USAGE_INPUT_TOKENS]).toBe(10); + expect(agentJson!.data[GEN_AI_USAGE_OUTPUT_TOKENS]).toBe(4); + expect(agentJson!.data[GEN_AI_USAGE_TOTAL_TOKENS]).toBe(14); + expect(agentJson!.data[GEN_AI_RESPONSE_MODEL]).toBe('gpt-5-2026'); }); it('sums usage across multiple generations onto the parent agent', async () => { @@ -307,11 +340,11 @@ describe('SentryMastraExporter', () => { .map(span => spanToStaticSpanJSON(span)) .find(span => span.description === 'invoke_agent agent'); expect(agentJson).toBeDefined(); - expect(agentJson!.data['gen_ai.usage.input_tokens']).toBe(40); - expect(agentJson!.data['gen_ai.usage.output_tokens']).toBe(12); - expect(agentJson!.data['gen_ai.usage.total_tokens']).toBe(52); - expect(agentJson!.data['gen_ai.usage.cache_read.input_tokens']).toBe(3); - expect(agentJson!.data['gen_ai.response.model']).toBe('gpt-5-2026'); + expect(agentJson!.data[GEN_AI_USAGE_INPUT_TOKENS]).toBe(40); + expect(agentJson!.data[GEN_AI_USAGE_OUTPUT_TOKENS]).toBe(12); + expect(agentJson!.data[GEN_AI_USAGE_TOTAL_TOKENS]).toBe(52); + expect(agentJson!.data[GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS]).toBe(3); + expect(agentJson!.data[GEN_AI_RESPONSE_MODEL]).toBe('gpt-5-2026'); }); it('marks the span errored without capturing a reconstructed exception', async () => { @@ -320,7 +353,7 @@ describe('SentryMastraExporter', () => { const json = spanToStaticSpanJSON(endedSpans[0]!); expect(json.status).toBe('internal_error'); - expect(json.data['error.type']).toBe('ToolError'); + expect(json.data[ERROR_TYPE]).toBe('ToolError'); }); it.each([ @@ -356,7 +389,7 @@ describe('SentryMastraExporter', () => { const json = spanToStaticSpanJSON(endedSpans[0]!); expect(Object.keys(json.data).filter(key => key.startsWith('mastra.'))).toEqual([]); - expect(json.data['gen_ai.conversation.id']).toBe('thread-9'); + expect(json.data[GEN_AI_CONVERSATION_ID]).toBe('thread-9'); }); it.each([ @@ -370,14 +403,14 @@ describe('SentryMastraExporter', () => { const span = makeSpan({ entityName: 'agent', ...overrides }); await run(started(span), ended(span)); - expect(spanToStaticSpanJSON(endedSpans[0]!).data['gen_ai.conversation.id']).toBe(expected); + expect(spanToStaticSpanJSON(endedSpans[0]!).data[GEN_AI_CONVERSATION_ID]).toBe(expected); }); it('does not use runId as gen_ai.conversation.id', async () => { const span = makeSpan({ entityName: 'agent', metadata: { runId: 'run-3' } }); await run(started(span), ended(span)); - expect(spanToStaticSpanJSON(endedSpans[0]!).data['gen_ai.conversation.id']).toBeUndefined(); + expect(spanToStaticSpanJSON(endedSpans[0]!).data[GEN_AI_CONVERSATION_ID]).toBeUndefined(); }); it('falls back to the bare operation name when there is no identifier', async () => { @@ -433,7 +466,7 @@ describe('SentryMastraExporter', () => { }); await run(started(span), ended(span)); - expect(spanToStaticSpanJSON(endedSpans[0]!).data['gen_ai.request.stop_sequences']).toEqual(['\n\n', 'END']); + expect(spanToStaticSpanJSON(endedSpans[0]!).data[GEN_AI_REQUEST_STOP_SEQUENCES]).toEqual(['\n\n', 'END']); }); // Parentless dropped spans used to be stored under a falsy sentinel that `LRUMap.remove` never From 4495ee804d1beca24d000aa56c1b84b4a9c824c2 Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Wed, 7 Oct 2026 10:02:21 +0200 Subject: [PATCH 05/12] Record Mastra classifier inputs and answers --- .../suites/tracing/mastra/test.ts | 34 +++++++++ .../src/ai/mastra/classifier-evaluation.ts | 28 +++++++ packages/server-utils/src/ai/mastra/index.ts | 19 ++++- .../server-utils/src/ai/typesafe/index.ts | 12 ++- .../src/integrations/mastra-classifier.ts | 75 +++++++++++++++++++ .../server-utils/src/integrations/mastra.ts | 2 + .../src/orchestrion/config/mastra.ts | 21 +++++- .../ai/lib/tracing/mastra-exporter.test.ts | 14 ++++ 8 files changed, 200 insertions(+), 5 deletions(-) create mode 100644 packages/server-utils/src/ai/mastra/classifier-evaluation.ts create mode 100644 packages/server-utils/src/integrations/mastra-classifier.ts diff --git a/dev-packages/node-integration-tests/suites/tracing/mastra/test.ts b/dev-packages/node-integration-tests/suites/tracing/mastra/test.ts index 75e0ecb48e08..55a5a915605d 100644 --- a/dev-packages/node-integration-tests/suites/tracing/mastra/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/mastra/test.ts @@ -236,6 +236,40 @@ conditionalTest({ min: 22 })('Mastra integration', () => { expect(evaluateSpan.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(30); expect(evaluateSpan.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(2); expect(evaluateSpan.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(32); + expect(evaluateSpan.attributes[GEN_AI_INPUT_MESSAGES]).toBeUndefined(); + expect(evaluateSpan.attributes[GEN_AI_OUTPUT_MESSAGES]).toBeUndefined(); + }, + }) + .start() + .completed(); + }); + }, + MASTRA_CLASSIFIER_DEPENDENCIES, + ); + + createEsmAndCjsTests( + __dirname, + 'scenario-classifier.mjs', + 'instrument-with-pii.mjs', + (createRunner, test) => { + test('records the evaluated state, questions and answers when genAI recording is on', async () => { + await createRunner() + .expect({ transaction: { transaction: 'mastra-test' } }) + .expect({ + span: container => { + const evaluateSpan = container.items.find(span => span.name === 'evaluate jev-latest')!; + expect(JSON.parse(evaluateSpan.attributes[GEN_AI_INPUT_MESSAGES].value)).toEqual([ + { + type: 'evaluation', + state: { message: 'I was charged twice.' }, + questions: { + urgent: { type: 'boolean', instructions: 'Does this request need an immediate response?' }, + }, + }, + ]); + expect(JSON.parse(evaluateSpan.attributes[GEN_AI_OUTPUT_MESSAGES].value)).toEqual([ + { type: 'evaluation', answers: { urgent: { type: 'boolean', probability: 0.9 } } }, + ]); }, }) .start() diff --git a/packages/server-utils/src/ai/mastra/classifier-evaluation.ts b/packages/server-utils/src/ai/mastra/classifier-evaluation.ts new file mode 100644 index 000000000000..5886d0bb28f7 --- /dev/null +++ b/packages/server-utils/src/ai/mastra/classifier-evaluation.ts @@ -0,0 +1,28 @@ +import type { Span, SpanTimeInput } from '@sentry/core'; + +/** + * Links a `Classifier.evaluate()` call to the Sentry span the exporter opens for its + * `classifier_evaluation` span. Mastra's span carries no input or output, so the integration adds + * them from the call. Mastra ends its span before `evaluate()` returns the answers, so the exporter + * leaves the Sentry span open and the integration ends it once the call settles. + */ +export interface ClassifierEvaluationCall { + span?: Span; + /** Mastra's end time, set when Mastra ended its span before the call settled. */ + endTime?: SpanTimeInput; + settled?: boolean; +} + +// Mastra emits `span_started` synchronously inside `evaluate()`, before its first `await`, so at +// most one call is ever in this window. +let startingCall: ClassifierEvaluationCall | undefined; + +export function setStartingClassifierEvaluation(call: ClassifierEvaluationCall | undefined): void { + startingCall = call; +} + +export function takeStartingClassifierEvaluation(): ClassifierEvaluationCall | undefined { + const call = startingCall; + startingCall = undefined; + return call; +} diff --git a/packages/server-utils/src/ai/mastra/index.ts b/packages/server-utils/src/ai/mastra/index.ts index 0bfcdd712777..ea315122abd7 100644 --- a/packages/server-utils/src/ai/mastra/index.ts +++ b/packages/server-utils/src/ai/mastra/index.ts @@ -7,6 +7,7 @@ import { LRUMap, SPAN_STATUS_ERROR, startInactiveSpan, + timestampInSeconds, } from '@sentry/core'; import { GEN_AI_RESPONSE_MODEL, SENTRY_OP, SENTRY_ORIGIN } from '@sentry/conventions/attributes'; import { DEBUG_BUILD } from '../../debug-build'; @@ -29,6 +30,8 @@ import { MAX_TRACKED_MASTRA_SPANS, MODEL_SPAN_TYPES, } from './constants'; +import type { ClassifierEvaluationCall } from './classifier-evaluation'; +import { takeStartingClassifierEvaluation } from './classifier-evaluation'; import { registerMastraSpan, unregisterMastraSpan } from './span-registry'; import type { MastraExportedSpan, MastraObservabilityExporter, MastraSpanType, MastraTracingEvent } from './types'; @@ -38,6 +41,7 @@ interface TrackedSpan { span: Span; spanType: MastraSpanType; usage: SpanAttributes; + evaluation?: ClassifierEvaluationCall; } const FLUSH_TIMEOUT_MS = 2000; @@ -147,7 +151,12 @@ export class SentryMastraExporter implements MastraObservabilityExporter { }, }); - this._trackSpan(span.id, { span: sentrySpan, spanType: span.type, usage: {} }); + const evaluation = span.type === 'classifier_evaluation' ? takeStartingClassifierEvaluation() : undefined; + if (evaluation) { + evaluation.span = sentrySpan; + } + + this._trackSpan(span.id, { span: sentrySpan, spanType: span.type, usage: {}, evaluation }); } /** Track a started span, ending any Sentry span that would otherwise be dropped without `end()`. */ @@ -201,7 +210,13 @@ export class SentryMastraExporter implements MastraObservabilityExporter { sentrySpan.setStatus({ code: SPAN_STATUS_ERROR, message: span.errorInfo.message }); } - sentrySpan.end(span.endTime); + const { evaluation } = tracked; + if (evaluation && !evaluation.settled) { + // The integration adds the call's input and answers once `evaluate()` settles, then ends the span. + evaluation.endTime = span.endTime ?? timestampInSeconds(); + } else { + sentrySpan.end(span.endTime); + } this._removeTracked(span.id); } diff --git a/packages/server-utils/src/ai/typesafe/index.ts b/packages/server-utils/src/ai/typesafe/index.ts index 9552e5e96a4d..702d26dc2ead 100644 --- a/packages/server-utils/src/ai/typesafe/index.ts +++ b/packages/server-utils/src/ai/typesafe/index.ts @@ -54,7 +54,10 @@ function getRequestAttributes( }; } -/** Serialize the `state` and `questions` of an evaluation request. Also used for TypeSafe models on Workers AI. */ +/** + * Serialize the `state` and `questions` of an evaluation request. Also used for TypeSafe models on Workers AI + * and for Mastra classifier evaluations. + */ export function getEvaluationInputMessages(request: Record): string | undefined { return stringify([{ type: 'evaluation', state: request.state, questions: request.questions }]); } @@ -86,10 +89,15 @@ export function addResponseAttributes(span: Span, result: unknown, recordOutputs } if (recordOutputs && result.answers !== undefined) { - span.setAttribute(GEN_AI_OUTPUT_MESSAGES, stringify([{ type: 'evaluation', answers: result.answers }])); + span.setAttribute(GEN_AI_OUTPUT_MESSAGES, getEvaluationOutputMessages(result.answers)); } } +/** Serialize the `answers` of an evaluation result. Also used for Mastra classifier evaluations. */ +export function getEvaluationOutputMessages(answers: unknown): string | undefined { + return stringify([{ type: 'evaluation', answers }]); +} + /** * `systemOne` returns a lazy `APIPromise` before the request settles, and parses the body on the first * `.then()`. Wait on `asResponse()` (the raw fetch, which does not parse the body) and read a clone of diff --git a/packages/server-utils/src/integrations/mastra-classifier.ts b/packages/server-utils/src/integrations/mastra-classifier.ts new file mode 100644 index 000000000000..d7f8c433f582 --- /dev/null +++ b/packages/server-utils/src/integrations/mastra-classifier.ts @@ -0,0 +1,75 @@ +import { GEN_AI_INPUT_MESSAGES, GEN_AI_OUTPUT_MESSAGES } from '@sentry/conventions/attributes'; +import { isObjectLike } from '@sentry/core'; +import { resolveAIRecordingOptions } from '../ai/core/utils'; +import type { GenAiOptions } from '../ai/core/utils'; +import type { ClassifierEvaluationCall } from '../ai/mastra/classifier-evaluation'; +import { setStartingClassifierEvaluation } from '../ai/mastra/classifier-evaluation'; +import { getEvaluationInputMessages, getEvaluationOutputMessages } from '../ai/typesafe'; +import { CHANNELS } from '../orchestrion/channels'; +import { safeChannelCallback } from '../tracing-channel'; +import * as diagnosticsChannel from '../utils/diagnosticsChannel'; + +interface ClassifierEvaluateChannelContext { + // `evaluate({ state, questions? })`; `self` is the `Classifier`. + arguments: unknown[]; + self?: unknown; + result?: unknown; +} + +/** + * Add the evaluated state, questions and answers of a `Classifier.evaluate()` call to the exporter's + * `classifier_evaluation` span, which Mastra leaves without input or output. + */ +export function recordClassifierEvaluations(options: GenAiOptions): void { + const channel = diagnosticsChannel.tracingChannel( + CHANNELS.MASTRA_CLASSIFIER_EVALUATE, + ); + const calls = new WeakMap(); + + channel.start.subscribe(message => { + safeChannelCallback(() => { + const call: ClassifierEvaluationCall = {}; + calls.set(message as object, call); + setStartingClassifierEvaluation(call); + }); + }); + // `evaluate()` has reached its first `await`, so Mastra has already started its span. + channel.end.subscribe(() => { + safeChannelCallback(() => setStartingClassifierEvaluation(undefined)); + }); + channel.asyncEnd.subscribe(message => { + safeChannelCallback(() => { + const call = calls.get(message as object); + if (call) { + finishClassifierEvaluation(call, message as ClassifierEvaluateChannelContext, options); + } + }); + }); +} + +function finishClassifierEvaluation( + call: ClassifierEvaluationCall, + message: ClassifierEvaluateChannelContext, + options: GenAiOptions, +): void { + call.settled = true; + const { span } = call; + if (!span) { + return; + } + + const { recordInputs, recordOutputs } = resolveAIRecordingOptions(options); + if (recordInputs) { + const params = isObjectLike(message.arguments[0]) ? message.arguments[0] : {}; + // Questions given to the constructor take precedence over the ones passed to `evaluate()`, as in Mastra. + const questions = (isObjectLike(message.self) ? message.self.questions : undefined) ?? params.questions; + span.setAttribute(GEN_AI_INPUT_MESSAGES, getEvaluationInputMessages({ state: params.state, questions })); + } + if (recordOutputs && isObjectLike(message.result)) { + span.setAttribute(GEN_AI_OUTPUT_MESSAGES, getEvaluationOutputMessages(message.result.answers)); + } + + if (call.endTime) { + span.end(call.endTime); + } +} diff --git a/packages/server-utils/src/integrations/mastra.ts b/packages/server-utils/src/integrations/mastra.ts index 9856ef51651d..332f9906455d 100644 --- a/packages/server-utils/src/integrations/mastra.ts +++ b/packages/server-utils/src/integrations/mastra.ts @@ -24,6 +24,7 @@ import type { MastraObservabilityExporter } from '../ai/mastra/types'; import { DEBUG_BUILD } from '../debug-build'; import { CHANNELS } from '../orchestrion/channels'; import { mastraModuleNames } from '../orchestrion/config/mastra'; +import { recordClassifierEvaluations } from './mastra-classifier'; import { invokeOrchestrionInstrumentation } from '../orchestrion/instrumentation'; import { bindSpanToChannelStore, safeChannelCallback } from '../tracing-channel'; @@ -96,6 +97,7 @@ function instrumentExporter(options: MastraOptions): void { }); captureExecuteWithContextErrors(); + recordClassifierEvaluations(options); } /** diff --git a/packages/server-utils/src/orchestrion/config/mastra.ts b/packages/server-utils/src/orchestrion/config/mastra.ts index dfa6ee151cbf..3e5c0d53c658 100644 --- a/packages/server-utils/src/orchestrion/config/mastra.ts +++ b/packages/server-utils/src/orchestrion/config/mastra.ts @@ -41,11 +41,30 @@ const mastraContextConfig: InstrumentationConfig[] = ['executeWithContext', 'exe }), ); -export const mastraConfig = [...mastraConstructorConfig, ...mastraContextConfig] satisfies InstrumentationConfig[]; +// Mastra's `classifier_evaluation` span carries no input or output, so we read them from the +// `Classifier.evaluate()` call. `Classifier` lives in the stable, unhashed `classifier` entry. +const mastraClassifierConfig: InstrumentationConfig[] = [ + { + channelName: 'mastraClassifierEvaluate', + module: { + name: '@mastra/core', + versionRange: '>=1.63.2 <2.0.0', + filePath: /^dist\/classifier\/index\.(?:cjs|mjs|js)$/, + }, + functionQuery: { className: 'Classifier', methodName: 'evaluate', kind: 'Async' }, + }, +]; + +export const mastraConfig = [ + ...mastraConstructorConfig, + ...mastraContextConfig, + ...mastraClassifierConfig, +] satisfies InstrumentationConfig[]; export const mastraModuleNames = getModuleNames(mastraConfig); export const mastraChannels = { MASTRA_CONSTRUCTOR: 'orchestrion:@mastra/core:mastraConstructor', MASTRA_EXECUTE_WITH_CONTEXT: 'orchestrion:@mastra/core:mastraExecuteWithContext', + MASTRA_CLASSIFIER_EVALUATE: 'orchestrion:@mastra/core:mastraClassifierEvaluate', } as const; diff --git a/packages/server-utils/test/ai/lib/tracing/mastra-exporter.test.ts b/packages/server-utils/test/ai/lib/tracing/mastra-exporter.test.ts index b7f3ee8d1635..95e98971312b 100644 --- a/packages/server-utils/test/ai/lib/tracing/mastra-exporter.test.ts +++ b/packages/server-utils/test/ai/lib/tracing/mastra-exporter.test.ts @@ -41,6 +41,8 @@ import { spanToStaticSpanJSON, } from '@sentry/core'; import { SentryMastraExporter } from '../../../../src/ai/mastra'; +import type { ClassifierEvaluationCall } from '../../../../src/ai/mastra/classifier-evaluation'; +import { setStartingClassifierEvaluation } from '../../../../src/ai/mastra/classifier-evaluation'; import type { MastraExportedSpan, MastraSpanType, MastraTracingEvent } from '../../../../src/ai/mastra/types'; import { OPENAI_INTEGRATION_NAME } from '../../../../src/ai/openai/constants'; import { getDefaultTestClientOptions, TestClient } from '../../../mocks/client'; @@ -176,6 +178,18 @@ describe('SentryMastraExporter', () => { }); }); + it('leaves a classifier evaluation span open for the call to end once it settles', async () => { + const call: ClassifierEvaluationCall = {}; + const span = makeSpan({ id: 'eval-1', type: 'classifier_evaluation', attributes: { modelId: 'jev-1' } }); + + setStartingClassifierEvaluation(call); + await run(started(span), ended(span)); + + expect(endedSpans).toHaveLength(0); + expect(call.endTime).toEqual(span.endTime); + expect(spanToStaticSpanJSON(call.span!).description).toBe('evaluate jev-1'); + }); + it('records the agent-level prompt and response as gen_ai messages', async () => { const span = makeSpan({ entityName: 'agent', From 35dd1637f6bd141afb2f3f0961d782586c66173e Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Wed, 7 Oct 2026 10:13:24 +0200 Subject: [PATCH 06/12] Use conventions constants in Mastra integration tests --- .../suites/tracing/mastra/test.ts | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/dev-packages/node-integration-tests/suites/tracing/mastra/test.ts b/dev-packages/node-integration-tests/suites/tracing/mastra/test.ts index 55a5a915605d..7108f1c30aa5 100644 --- a/dev-packages/node-integration-tests/suites/tracing/mastra/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/mastra/test.ts @@ -20,6 +20,7 @@ import { SENTRY_ORIGIN, URL_FULL, } from '@sentry/conventions/attributes'; +import { GEN_AI_CHAT, GEN_AI_EVALUATE, GEN_AI_EXECUTE_TOOL, GEN_AI_INVOKE_AGENT } from '@sentry/conventions/op'; import { afterAll, expect } from 'vitest'; import { conditionalTest } from '../../../utils'; import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../utils/runner'; @@ -71,7 +72,7 @@ conditionalTest({ min: 22 })('Mastra integration', () => { const agentSpan = spans.find(span => span.name === 'invoke_agent weather_agent')!; expect(agentSpan.status).toBe('ok'); - expect(agentSpan.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); + expect(agentSpan.attributes[SENTRY_OP].value).toBe(GEN_AI_INVOKE_AGENT); expect(agentSpan.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.mastra'); expect(agentSpan.attributes[GEN_AI_OPERATION_NAME].value).toBe('invoke_agent'); expect(agentSpan.attributes[GEN_AI_AGENT_NAME].value).toBe('weather_agent'); @@ -83,7 +84,7 @@ conditionalTest({ min: 22 })('Mastra integration', () => { const chatSpan = spans.find(span => span.name === 'chat gpt-4o-mini')!; expect(chatSpan.status).toBe('ok'); - expect(chatSpan.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); + expect(chatSpan.attributes[SENTRY_OP].value).toBe(GEN_AI_CHAT); expect(chatSpan.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.mastra'); expect(chatSpan.attributes[GEN_AI_OPERATION_NAME].value).toBe('chat'); expect(chatSpan.attributes[GEN_AI_REQUEST_MODEL].value).toBe('gpt-4o-mini'); @@ -165,7 +166,7 @@ conditionalTest({ min: 22 })('Mastra integration', () => { const toolSpan = spans.find(span => span.name === 'execute_tool get_weather')!; expect(toolSpan.status).toBe('ok'); - expect(toolSpan.attributes[SENTRY_OP].value).toBe('gen_ai.execute_tool'); + expect(toolSpan.attributes[SENTRY_OP].value).toBe(GEN_AI_EXECUTE_TOOL); expect(toolSpan.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.mastra'); expect(toolSpan.attributes[GEN_AI_OPERATION_NAME].value).toBe('execute_tool'); expect(toolSpan.attributes[GEN_AI_TOOL_NAME].value).toBe('get_weather'); @@ -204,7 +205,7 @@ conditionalTest({ min: 22 })('Mastra integration', () => { expect(spans.map(span => span.name)).toEqual(['invoke_agent math_workflow']); const workflowSpan = spans[0]!; - expect(workflowSpan.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); + expect(workflowSpan.attributes[SENTRY_OP].value).toBe(GEN_AI_INVOKE_AGENT); expect(workflowSpan.attributes[GEN_AI_PIPELINE_NAME].value).toBe('math_workflow'); }, }) @@ -228,8 +229,8 @@ conditionalTest({ min: 22 })('Mastra integration', () => { expect(container.items.map(span => span.name)).toEqual(['evaluate jev-latest']); const evaluateSpan = container.items[0]!; - expect(evaluateSpan.attributes['sentry.op'].value).toBe('gen_ai.evaluate'); - expect(evaluateSpan.attributes['sentry.origin'].value).toBe('auto.ai.mastra'); + expect(evaluateSpan.attributes[SENTRY_OP].value).toBe(GEN_AI_EVALUATE); + expect(evaluateSpan.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.mastra'); expect(evaluateSpan.attributes[GEN_AI_OPERATION_NAME].value).toBe('evaluate'); expect(evaluateSpan.attributes[GEN_AI_REQUEST_MODEL].value).toBe('jev-latest'); expect(evaluateSpan.attributes[GEN_AI_PROVIDER_NAME].value).toBe('typesafe-ai.evaluation'); @@ -374,14 +375,14 @@ conditionalTest({ min: 22 })('Mastra integration', () => { ); expect(loadSpans.length).toBeGreaterThan(0); const cacheGetParentIds = loadSpans.map(span => span.parent_span_id); - const chat = container.items.find(span => span.attributes[SENTRY_OP]?.value === 'gen_ai.chat')!; + const chat = container.items.find(span => span.attributes[SENTRY_OP]?.value === GEN_AI_CHAT)!; expect(chat).toBeDefined(); const chatSpanId = chat.span_id; const executeTool = container.items.find( span => span.attributes[GEN_AI_TOOL_NAME]?.value === 'count_items', )!; - expect(executeTool.attributes[SENTRY_OP]?.value).toBe('gen_ai.execute_tool'); + expect(executeTool.attributes[SENTRY_OP]?.value).toBe(GEN_AI_EXECUTE_TOOL); const executeToolSpanId = executeTool.span_id; // The `executeWithContext` bridge makes the exporter spans active during the real work, so the @@ -427,14 +428,14 @@ conditionalTest({ min: 22 })('Mastra integration', () => { ); expect(loadSpans.length).toBeGreaterThan(0); const cacheGetParentIds = loadSpans.map(span => span.parent_span_id); - const chat = container.items.find(span => span.attributes[SENTRY_OP]?.value === 'gen_ai.chat')!; + const chat = container.items.find(span => span.attributes[SENTRY_OP]?.value === GEN_AI_CHAT)!; expect(chat).toBeDefined(); const chatSpanId = chat.span_id; const executeTool = container.items.find( span => span.attributes[GEN_AI_TOOL_NAME]?.value === 'count_items', )!; - expect(executeTool.attributes[SENTRY_OP]?.value).toBe('gen_ai.execute_tool'); + expect(executeTool.attributes[SENTRY_OP]?.value).toBe(GEN_AI_EXECUTE_TOOL); const executeToolSpanId = executeTool.span_id; expect(chatSpanId).toBeDefined(); From 2b9662c8ea2a4dce7dfe73f49478316ed1bce097 Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Wed, 7 Oct 2026 11:37:13 +0200 Subject: [PATCH 07/12] Test failed Mastra classifier evaluations --- .../mastra/scenario-classifier-error.mjs | 48 +++++++++++++++++++ .../suites/tracing/mastra/test.ts | 26 ++++++++++ 2 files changed, 74 insertions(+) create mode 100644 dev-packages/node-integration-tests/suites/tracing/mastra/scenario-classifier-error.mjs diff --git a/dev-packages/node-integration-tests/suites/tracing/mastra/scenario-classifier-error.mjs b/dev-packages/node-integration-tests/suites/tracing/mastra/scenario-classifier-error.mjs new file mode 100644 index 000000000000..7a70b22c7527 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/mastra/scenario-classifier-error.mjs @@ -0,0 +1,48 @@ +import * as Sentry from '@sentry/node'; +import { Mastra } from '@mastra/core'; +import { Classifier } from '@mastra/core/classifier'; +import { Observability } from '@mastra/observability'; +import { SentryMastraExporter } from '@sentry/node'; + +// Inlined `EvaluationModelV4` mock: the ESM/CJS runner copies only this file into its temp dir. +const evaluationModel = { + specificationVersion: 'v4', + provider: 'typesafe-ai.evaluation', + modelId: 'jev-latest', + supportedQuestionTypes: ['choice', 'score', 'boolean'], + async doEvaluate() { + throw new Error('evaluation failed'); + }, +}; + +async function run() { + const classifier = new Classifier({ + id: 'request-classifier', + model: evaluationModel, + questions: { urgent: { type: 'boolean', instructions: 'Does this request need an immediate response?' } }, + }); + + const mastra = new Mastra({ + classifiers: { classifier }, + logger: false, + observability: new Observability({ + configs: { + default: { + serviceName: 'mastra-test', + exporters: [new SentryMastraExporter()], + }, + }, + }), + }); + + await Sentry.startSpan({ op: 'function', name: 'mastra-test' }, async () => { + await mastra + .getClassifier('classifier') + .evaluate({ state: { message: 'I was charged twice.' }, maxRetries: 0 }) + .catch(() => undefined); + }); + + await mastra.observability.shutdown(); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/mastra/test.ts b/dev-packages/node-integration-tests/suites/tracing/mastra/test.ts index 7108f1c30aa5..c192d95ab150 100644 --- a/dev-packages/node-integration-tests/suites/tracing/mastra/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/mastra/test.ts @@ -280,6 +280,32 @@ conditionalTest({ min: 22 })('Mastra integration', () => { MASTRA_CLASSIFIER_DEPENDENCIES, ); + createEsmAndCjsTests( + __dirname, + 'scenario-classifier-error.mjs', + 'instrument-with-pii.mjs', + (createRunner, test) => { + test('ends a failed classifier evaluation span with an error status', async () => { + await createRunner() + .ignore('event') + .expect({ transaction: { transaction: 'mastra-test' } }) + .expect({ + span: container => { + expect(container.items.map(span => span.name)).toEqual(['evaluate jev-latest']); + + const evaluateSpan = container.items[0]!; + expect(evaluateSpan.status).toBe('error'); + expect(evaluateSpan.attributes[GEN_AI_INPUT_MESSAGES].value).toContain('I was charged twice.'); + expect(evaluateSpan.attributes[GEN_AI_OUTPUT_MESSAGES]).toBeUndefined(); + }, + }) + .start() + .completed(); + }); + }, + MASTRA_CLASSIFIER_DEPENDENCIES, + ); + createEsmAndCjsTests( __dirname, 'scenario-auto.mjs', From 01cf05012e849a92fe5ad5281b9b24549bcecdbd Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Wed, 7 Oct 2026 11:37:13 +0200 Subject: [PATCH 08/12] Follow exporter recording options for classifier data --- .../src/ai/mastra/classifier-evaluation.ts | 3 +++ packages/server-utils/src/ai/mastra/index.ts | 3 ++- .../src/integrations/mastra-classifier.ts | 15 ++++----------- packages/server-utils/src/integrations/mastra.ts | 2 +- .../test/ai/lib/tracing/mastra-exporter.test.ts | 10 ++++++++++ 5 files changed, 20 insertions(+), 13 deletions(-) diff --git a/packages/server-utils/src/ai/mastra/classifier-evaluation.ts b/packages/server-utils/src/ai/mastra/classifier-evaluation.ts index 5886d0bb28f7..df4bd4eb564c 100644 --- a/packages/server-utils/src/ai/mastra/classifier-evaluation.ts +++ b/packages/server-utils/src/ai/mastra/classifier-evaluation.ts @@ -8,6 +8,9 @@ import type { Span, SpanTimeInput } from '@sentry/core'; */ export interface ClassifierEvaluationCall { span?: Span; + /** The recording options of the exporter that opened the span, so the call's data follows them. */ + recordInputs?: boolean; + recordOutputs?: boolean; /** Mastra's end time, set when Mastra ended its span before the call settled. */ endTime?: SpanTimeInput; settled?: boolean; diff --git a/packages/server-utils/src/ai/mastra/index.ts b/packages/server-utils/src/ai/mastra/index.ts index ea315122abd7..431ce3e69df1 100644 --- a/packages/server-utils/src/ai/mastra/index.ts +++ b/packages/server-utils/src/ai/mastra/index.ts @@ -153,7 +153,8 @@ export class SentryMastraExporter implements MastraObservabilityExporter { const evaluation = span.type === 'classifier_evaluation' ? takeStartingClassifierEvaluation() : undefined; if (evaluation) { - evaluation.span = sentrySpan; + const { recordInputs, recordOutputs } = resolveAIRecordingOptions(this._options); + Object.assign(evaluation, { span: sentrySpan, recordInputs, recordOutputs }); } this._trackSpan(span.id, { span: sentrySpan, spanType: span.type, usage: {}, evaluation }); diff --git a/packages/server-utils/src/integrations/mastra-classifier.ts b/packages/server-utils/src/integrations/mastra-classifier.ts index d7f8c433f582..8b3917607f3d 100644 --- a/packages/server-utils/src/integrations/mastra-classifier.ts +++ b/packages/server-utils/src/integrations/mastra-classifier.ts @@ -1,7 +1,5 @@ import { GEN_AI_INPUT_MESSAGES, GEN_AI_OUTPUT_MESSAGES } from '@sentry/conventions/attributes'; import { isObjectLike } from '@sentry/core'; -import { resolveAIRecordingOptions } from '../ai/core/utils'; -import type { GenAiOptions } from '../ai/core/utils'; import type { ClassifierEvaluationCall } from '../ai/mastra/classifier-evaluation'; import { setStartingClassifierEvaluation } from '../ai/mastra/classifier-evaluation'; import { getEvaluationInputMessages, getEvaluationOutputMessages } from '../ai/typesafe'; @@ -20,7 +18,7 @@ interface ClassifierEvaluateChannelContext { * Add the evaluated state, questions and answers of a `Classifier.evaluate()` call to the exporter's * `classifier_evaluation` span, which Mastra leaves without input or output. */ -export function recordClassifierEvaluations(options: GenAiOptions): void { +export function recordClassifierEvaluations(): void { const channel = diagnosticsChannel.tracingChannel( CHANNELS.MASTRA_CLASSIFIER_EVALUATE, ); @@ -41,24 +39,19 @@ export function recordClassifierEvaluations(options: GenAiOptions): void { safeChannelCallback(() => { const call = calls.get(message as object); if (call) { - finishClassifierEvaluation(call, message as ClassifierEvaluateChannelContext, options); + finishClassifierEvaluation(call, message as ClassifierEvaluateChannelContext); } }); }); } -function finishClassifierEvaluation( - call: ClassifierEvaluationCall, - message: ClassifierEvaluateChannelContext, - options: GenAiOptions, -): void { +function finishClassifierEvaluation(call: ClassifierEvaluationCall, message: ClassifierEvaluateChannelContext): void { call.settled = true; - const { span } = call; + const { span, recordInputs, recordOutputs } = call; if (!span) { return; } - const { recordInputs, recordOutputs } = resolveAIRecordingOptions(options); if (recordInputs) { const params = isObjectLike(message.arguments[0]) ? message.arguments[0] : {}; // Questions given to the constructor take precedence over the ones passed to `evaluate()`, as in Mastra. diff --git a/packages/server-utils/src/integrations/mastra.ts b/packages/server-utils/src/integrations/mastra.ts index 332f9906455d..1f541fe83303 100644 --- a/packages/server-utils/src/integrations/mastra.ts +++ b/packages/server-utils/src/integrations/mastra.ts @@ -97,7 +97,7 @@ function instrumentExporter(options: MastraOptions): void { }); captureExecuteWithContextErrors(); - recordClassifierEvaluations(options); + recordClassifierEvaluations(); } /** diff --git a/packages/server-utils/test/ai/lib/tracing/mastra-exporter.test.ts b/packages/server-utils/test/ai/lib/tracing/mastra-exporter.test.ts index 95e98971312b..844cc9e82514 100644 --- a/packages/server-utils/test/ai/lib/tracing/mastra-exporter.test.ts +++ b/packages/server-utils/test/ai/lib/tracing/mastra-exporter.test.ts @@ -190,6 +190,16 @@ describe('SentryMastraExporter', () => { expect(spanToStaticSpanJSON(call.span!).description).toBe('evaluate jev-1'); }); + it('passes its own recording options to the classifier evaluation call', async () => { + exporter = new SentryMastraExporter({ recordInputs: false, recordOutputs: false }); + const call: ClassifierEvaluationCall = {}; + + setStartingClassifierEvaluation(call); + await run(started(makeSpan({ id: 'eval-1', type: 'classifier_evaluation' }))); + + expect(call).toMatchObject({ recordInputs: false, recordOutputs: false }); + }); + it('records the agent-level prompt and response as gen_ai messages', async () => { const span = makeSpan({ entityName: 'agent', From 9e56d1569a7720a32a8d9278cf44cc7b0c0ba714 Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Thu, 8 Oct 2026 11:11:07 +0200 Subject: [PATCH 09/12] Run classifier data through Mastra span output processors --- .../tracing/mastra/scenario-classifier.mjs | 4 ++- .../suites/tracing/mastra/test.ts | 2 +- .../src/ai/mastra/classifier-evaluation.ts | 27 +++++++++++++++++++ packages/server-utils/src/ai/mastra/index.ts | 23 ++++++++++++++-- packages/server-utils/src/ai/mastra/types.ts | 12 +++++++++ .../src/integrations/mastra-classifier.ts | 21 +++++++++------ 6 files changed, 77 insertions(+), 12 deletions(-) diff --git a/dev-packages/node-integration-tests/suites/tracing/mastra/scenario-classifier.mjs b/dev-packages/node-integration-tests/suites/tracing/mastra/scenario-classifier.mjs index ca0015249cde..e4e5efb15074 100644 --- a/dev-packages/node-integration-tests/suites/tracing/mastra/scenario-classifier.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/mastra/scenario-classifier.mjs @@ -40,7 +40,9 @@ async function run() { }); await Sentry.startSpan({ op: 'function', name: 'mastra-test' }, async () => { - await mastra.getClassifier('classifier').evaluate({ state: { message: 'I was charged twice.' } }); + await mastra + .getClassifier('classifier') + .evaluate({ state: { message: 'I was charged twice.', apiKey: 'sk-test-123' } }); }); await mastra.observability.shutdown(); diff --git a/dev-packages/node-integration-tests/suites/tracing/mastra/test.ts b/dev-packages/node-integration-tests/suites/tracing/mastra/test.ts index c192d95ab150..cbbaf3b128f2 100644 --- a/dev-packages/node-integration-tests/suites/tracing/mastra/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/mastra/test.ts @@ -262,7 +262,7 @@ conditionalTest({ min: 22 })('Mastra integration', () => { expect(JSON.parse(evaluateSpan.attributes[GEN_AI_INPUT_MESSAGES].value)).toEqual([ { type: 'evaluation', - state: { message: 'I was charged twice.' }, + state: { message: 'I was charged twice.', apiKey: '[REDACTED]' }, questions: { urgent: { type: 'boolean', instructions: 'Does this request need an immediate response?' }, }, diff --git a/packages/server-utils/src/ai/mastra/classifier-evaluation.ts b/packages/server-utils/src/ai/mastra/classifier-evaluation.ts index df4bd4eb564c..952748b89512 100644 --- a/packages/server-utils/src/ai/mastra/classifier-evaluation.ts +++ b/packages/server-utils/src/ai/mastra/classifier-evaluation.ts @@ -1,4 +1,5 @@ import type { Span, SpanTimeInput } from '@sentry/core'; +import type { MastraExportedSpan, MastraSpanOutputProcessor } from './types'; /** * Links a `Classifier.evaluate()` call to the Sentry span the exporter opens for its @@ -8,6 +9,9 @@ import type { Span, SpanTimeInput } from '@sentry/core'; */ export interface ClassifierEvaluationCall { span?: Span; + /** Mastra's span and the processors of its observability instance, to filter the call's data. */ + mastraSpan?: MastraExportedSpan; + spanOutputProcessors?: MastraSpanOutputProcessor[]; /** The recording options of the exporter that opened the span, so the call's data follows them. */ recordInputs?: boolean; recordOutputs?: boolean; @@ -29,3 +33,26 @@ export function takeStartingClassifierEvaluation(): ClassifierEvaluationCall | u startingCall = undefined; return call; } + +/** + * Run the call's data through Mastra's span output processors (for example its default + * `SensitiveDataFilter`), as Mastra does for the input and output of its own spans. Returns `undefined` + * when a processor drops the span or throws, so unfiltered data is never recorded. + */ +export function processClassifierEvaluationData( + call: ClassifierEvaluationCall, + data: { input?: unknown; output?: unknown }, +): { input?: unknown; output?: unknown } | undefined { + let span: MastraExportedSpan | undefined = { ...call.mastraSpan, ...data } as MastraExportedSpan; + for (const processor of call.spanOutputProcessors ?? []) { + try { + span = processor.process(span); + } catch { + return undefined; + } + if (!span) { + return undefined; + } + } + return span; +} diff --git a/packages/server-utils/src/ai/mastra/index.ts b/packages/server-utils/src/ai/mastra/index.ts index 431ce3e69df1..a3b5bf2699d9 100644 --- a/packages/server-utils/src/ai/mastra/index.ts +++ b/packages/server-utils/src/ai/mastra/index.ts @@ -33,7 +33,14 @@ import { import type { ClassifierEvaluationCall } from './classifier-evaluation'; import { takeStartingClassifierEvaluation } from './classifier-evaluation'; import { registerMastraSpan, unregisterMastraSpan } from './span-registry'; -import type { MastraExportedSpan, MastraObservabilityExporter, MastraSpanType, MastraTracingEvent } from './types'; +import type { + MastraExportedSpan, + MastraExporterInitOptions, + MastraObservabilityExporter, + MastraSpanOutputProcessor, + MastraSpanType, + MastraTracingEvent, +} from './types'; export type MastraExporterOptions = GenAiOptions; @@ -68,11 +75,17 @@ export class SentryMastraExporter implements MastraObservabilityExporter { */ private readonly _skipped = new LRUMap(MAX_TRACKED_MASTRA_SPANS); private readonly _options: MastraExporterOptions; + private _spanOutputProcessors: MastraSpanOutputProcessor[] = []; public constructor(options: MastraExporterOptions = {}) { this._options = options; } + /** Called by Mastra with the config of the observability instance this exporter belongs to. */ + public init(options: MastraExporterInitOptions): void { + this._spanOutputProcessors = options.config?.spanOutputProcessors ?? []; + } + /** Mastra's interface is async; the work is synchronous. */ public async exportTracingEvent(event: MastraTracingEvent): Promise { if (!getClient()) { @@ -154,7 +167,13 @@ export class SentryMastraExporter implements MastraObservabilityExporter { const evaluation = span.type === 'classifier_evaluation' ? takeStartingClassifierEvaluation() : undefined; if (evaluation) { const { recordInputs, recordOutputs } = resolveAIRecordingOptions(this._options); - Object.assign(evaluation, { span: sentrySpan, recordInputs, recordOutputs }); + Object.assign(evaluation, { + span: sentrySpan, + mastraSpan: span, + spanOutputProcessors: this._spanOutputProcessors, + recordInputs, + recordOutputs, + }); } this._trackSpan(span.id, { span: sentrySpan, spanType: span.type, usage: {}, evaluation }); diff --git a/packages/server-utils/src/ai/mastra/types.ts b/packages/server-utils/src/ai/mastra/types.ts index 86d3d518028a..5de6ad91efcc 100644 --- a/packages/server-utils/src/ai/mastra/types.ts +++ b/packages/server-utils/src/ai/mastra/types.ts @@ -72,6 +72,7 @@ export interface MastraSpanAttributes { export interface MastraExportedSpan { id: string; + traceId?: string; parentSpanId?: string; /** Mastra's own name; unused for Sentry span names. */ name: string; @@ -94,6 +95,17 @@ export interface MastraTracingEvent { exportedSpan: MastraExportedSpan; } +/** Mastra's `SpanOutputProcessor`, such as its default `SensitiveDataFilter`. */ +export interface MastraSpanOutputProcessor { + name: string; + process(span: MastraExportedSpan): MastraExportedSpan | undefined; +} + +/** Subset of what Mastra passes to `ObservabilityExporter.init()`. */ +export interface MastraExporterInitOptions { + config?: { spanOutputProcessors?: MastraSpanOutputProcessor[] }; +} + /** Subset of Mastra's `ObservabilityExporter`. The rest of that interface is optional. */ export interface MastraObservabilityExporter { name: string; diff --git a/packages/server-utils/src/integrations/mastra-classifier.ts b/packages/server-utils/src/integrations/mastra-classifier.ts index 8b3917607f3d..310651bff301 100644 --- a/packages/server-utils/src/integrations/mastra-classifier.ts +++ b/packages/server-utils/src/integrations/mastra-classifier.ts @@ -1,7 +1,7 @@ import { GEN_AI_INPUT_MESSAGES, GEN_AI_OUTPUT_MESSAGES } from '@sentry/conventions/attributes'; import { isObjectLike } from '@sentry/core'; import type { ClassifierEvaluationCall } from '../ai/mastra/classifier-evaluation'; -import { setStartingClassifierEvaluation } from '../ai/mastra/classifier-evaluation'; +import { processClassifierEvaluationData, setStartingClassifierEvaluation } from '../ai/mastra/classifier-evaluation'; import { getEvaluationInputMessages, getEvaluationOutputMessages } from '../ai/typesafe'; import { CHANNELS } from '../orchestrion/channels'; import { safeChannelCallback } from '../tracing-channel'; @@ -52,14 +52,19 @@ function finishClassifierEvaluation(call: ClassifierEvaluationCall, message: Cla return; } - if (recordInputs) { - const params = isObjectLike(message.arguments[0]) ? message.arguments[0] : {}; - // Questions given to the constructor take precedence over the ones passed to `evaluate()`, as in Mastra. - const questions = (isObjectLike(message.self) ? message.self.questions : undefined) ?? params.questions; - span.setAttribute(GEN_AI_INPUT_MESSAGES, getEvaluationInputMessages({ state: params.state, questions })); + const params = isObjectLike(message.arguments[0]) ? message.arguments[0] : {}; + // Questions given to the constructor take precedence over the ones passed to `evaluate()`, as in Mastra. + const questions = (isObjectLike(message.self) ? message.self.questions : undefined) ?? params.questions; + const data = processClassifierEvaluationData(call, { + input: recordInputs ? { state: params.state, questions } : undefined, + output: recordOutputs && isObjectLike(message.result) ? message.result.answers : undefined, + }); + + if (isObjectLike(data?.input)) { + span.setAttribute(GEN_AI_INPUT_MESSAGES, getEvaluationInputMessages(data.input)); } - if (recordOutputs && isObjectLike(message.result)) { - span.setAttribute(GEN_AI_OUTPUT_MESSAGES, getEvaluationOutputMessages(message.result.answers)); + if (data?.output !== undefined) { + span.setAttribute(GEN_AI_OUTPUT_MESSAGES, getEvaluationOutputMessages(data.output)); } if (call.endTime) { From 3dd55f211ca4ae13d164d6e0954115eede3a0967 Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Thu, 8 Oct 2026 13:09:34 +0200 Subject: [PATCH 10/12] Resolve Mastra recording options once per event --- packages/server-utils/src/ai/mastra/index.ts | 31 +++++++++----------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/packages/server-utils/src/ai/mastra/index.ts b/packages/server-utils/src/ai/mastra/index.ts index a3b5bf2699d9..7943275fe6bd 100644 --- a/packages/server-utils/src/ai/mastra/index.ts +++ b/packages/server-utils/src/ai/mastra/index.ts @@ -20,6 +20,7 @@ import { getUsageAttributes, isExportedSpanType, mergeUsageAttributes, + type AttributeRecordingOptions, type SpanAttributes, } from './utils'; import { @@ -134,20 +135,22 @@ export class SentryMastraExporter implements MastraObservabilityExporter { return; } + // Per event: the exporter can be constructed before `Sentry.init()`, when `dataCollection.genAI` does not exist yet. + const recordingOptions = resolveAIRecordingOptions(this._options); switch (event.type) { case 'span_started': - this._onSpanStarted(span); + this._onSpanStarted(span, recordingOptions); break; case 'span_updated': - this._onSpanUpdated(span); + this._onSpanUpdated(span, recordingOptions); break; case 'span_ended': - this._onSpanEnded(span); + this._onSpanEnded(span, recordingOptions); break; } } - private _onSpanStarted(span: MastraExportedSpan): void { + private _onSpanStarted(span: MastraExportedSpan, recordingOptions: AttributeRecordingOptions): void { const parentId = this._resolveParentId(span.parentSpanId); const parentSpan = parentId ? this._spans.get(parentId)?.span : undefined; const activeSpan = getActiveSpan(); @@ -158,7 +161,7 @@ export class SentryMastraExporter implements MastraObservabilityExporter { // Prefer the Mastra parent so the tree stays together; else the active request span. parentSpan: parentSpan ?? activeSpan, attributes: { - ...this._attributesFor(span), + ...getSpanAttributes(span, recordingOptions), [SENTRY_OP]: getOperation(span.type)?.op, [SENTRY_ORIGIN]: MASTRA_ORIGIN, }, @@ -166,13 +169,12 @@ export class SentryMastraExporter implements MastraObservabilityExporter { const evaluation = span.type === 'classifier_evaluation' ? takeStartingClassifierEvaluation() : undefined; if (evaluation) { - const { recordInputs, recordOutputs } = resolveAIRecordingOptions(this._options); Object.assign(evaluation, { span: sentrySpan, mastraSpan: span, spanOutputProcessors: this._spanOutputProcessors, - recordInputs, - recordOutputs, + recordInputs: recordingOptions.recordInputs, + recordOutputs: recordingOptions.recordOutputs, }); } @@ -203,14 +205,14 @@ export class SentryMastraExporter implements MastraObservabilityExporter { return this._spans.remove(id); } - private _onSpanUpdated(span: MastraExportedSpan): void { + private _onSpanUpdated(span: MastraExportedSpan, recordingOptions: AttributeRecordingOptions): void { const tracked = this._spans.get(span.id); if (tracked) { - tracked.span.setAttributes(this._attributesFor(span)); + tracked.span.setAttributes(getSpanAttributes(span, recordingOptions)); } } - private _onSpanEnded(span: MastraExportedSpan): void { + private _onSpanEnded(span: MastraExportedSpan, recordingOptions: AttributeRecordingOptions): void { const tracked = this._spans.get(span.id); if (!tracked) { DEBUG_BUILD && debug.warn(`[Mastra] no Sentry span open for ended span ${span.id} (${span.name})`); @@ -218,7 +220,7 @@ export class SentryMastraExporter implements MastraObservabilityExporter { } const { span: sentrySpan } = tracked; - sentrySpan.setAttributes(this._attributesFor(span)); + sentrySpan.setAttributes(getSpanAttributes(span, recordingOptions)); sentrySpan.updateName(getSpanName(span)); if (MODEL_SPAN_TYPES.has(span.type)) { @@ -275,9 +277,4 @@ export class SentryMastraExporter implements MastraObservabilityExporter { } return undefined; } - - private _attributesFor(span: MastraExportedSpan): ReturnType { - // Per event: the exporter can be constructed before `Sentry.init()`, when `dataCollection.genAI` does not exist yet. - return getSpanAttributes(span, resolveAIRecordingOptions(this._options)); - } } From 55997335fb11df95b60d1e217638376149978b73 Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Thu, 8 Oct 2026 13:14:26 +0200 Subject: [PATCH 11/12] Move evaluation message helpers to shared AI utils --- packages/server-utils/src/ai/core/utils.ts | 10 ++++++++++ packages/server-utils/src/ai/typesafe/index.ts | 17 ++--------------- .../server-utils/src/ai/workers-ai/utils.ts | 9 +++++++-- .../src/integrations/mastra-classifier.ts | 2 +- 4 files changed, 20 insertions(+), 18 deletions(-) diff --git a/packages/server-utils/src/ai/core/utils.ts b/packages/server-utils/src/ai/core/utils.ts index 49f76fb18998..93949ec40eca 100644 --- a/packages/server-utils/src/ai/core/utils.ts +++ b/packages/server-utils/src/ai/core/utils.ts @@ -136,6 +136,16 @@ export function getTokenUsageAttributes( return attributes; } +/** Serialize the `state` and `questions` of an evaluation request (TypeSafe, Workers AI, Mastra classifiers). */ +export function getEvaluationInputMessages(request: Record): string | undefined { + return stringify([{ type: 'evaluation', state: request.state, questions: request.questions }]); +} + +/** Serialize the `answers` of an evaluation result (TypeSafe, Workers AI, Mastra classifiers). */ +export function getEvaluationOutputMessages(answers: unknown): string | undefined { + return stringify([{ type: 'evaluation', answers }]); +} + /** One assistant turn for {@link setOutputMessagesAttribute}. */ export interface GenAiOutputMessage { /** The message's text content, already flattened out of any content-part array. */ diff --git a/packages/server-utils/src/ai/typesafe/index.ts b/packages/server-utils/src/ai/typesafe/index.ts index 702d26dc2ead..99280c6843ad 100644 --- a/packages/server-utils/src/ai/typesafe/index.ts +++ b/packages/server-utils/src/ai/typesafe/index.ts @@ -12,9 +12,9 @@ import { } from '@sentry/conventions/attributes'; import { GEN_AI_EVALUATE } from '@sentry/conventions/op'; import type { Span, SpanAttributes } from '@sentry/core'; -import { isObjectLike, SPAN_STATUS_ERROR, startInactiveSpan, stringify, withActiveSpan } from '@sentry/core'; +import { isObjectLike, SPAN_STATUS_ERROR, startInactiveSpan, withActiveSpan } from '@sentry/core'; import type { GenAiOptions } from '../core/utils'; -import { resolveAIRecordingOptions } from '../core/utils'; +import { getEvaluationInputMessages, getEvaluationOutputMessages, resolveAIRecordingOptions } from '../core/utils'; import { TYPESAFE_ORIGIN, TYPESAFE_PROVIDER_NAME } from './constants'; /** @@ -54,14 +54,6 @@ function getRequestAttributes( }; } -/** - * Serialize the `state` and `questions` of an evaluation request. Also used for TypeSafe models on Workers AI - * and for Mastra classifier evaluations. - */ -export function getEvaluationInputMessages(request: Record): string | undefined { - return stringify([{ type: 'evaluation', state: request.state, questions: request.questions }]); -} - /** * Add the response model, token usage and (optionally) the answers of a `systemOne` result. * Also used for TypeSafe models on Workers AI. @@ -93,11 +85,6 @@ export function addResponseAttributes(span: Span, result: unknown, recordOutputs } } -/** Serialize the `answers` of an evaluation result. Also used for Mastra classifier evaluations. */ -export function getEvaluationOutputMessages(answers: unknown): string | undefined { - return stringify([{ type: 'evaluation', answers }]); -} - /** * `systemOne` returns a lazy `APIPromise` before the request settles, and parses the body on the first * `.then()`. Wait on `asResponse()` (the raw fetch, which does not parse the body) and read a clone of diff --git a/packages/server-utils/src/ai/workers-ai/utils.ts b/packages/server-utils/src/ai/workers-ai/utils.ts index 66dd3d567c8c..73da7883b12e 100644 --- a/packages/server-utils/src/ai/workers-ai/utils.ts +++ b/packages/server-utils/src/ai/workers-ai/utils.ts @@ -20,8 +20,13 @@ import { GEN_AI_CHAT, GEN_AI_EMBEDDINGS, GEN_AI_EVALUATE } from '@sentry/convent import { isObjectLike, stringify } from '@sentry/core'; import type { Span, SpanAttributeValue } from '@sentry/core'; import { GEN_AI_REQUEST_STREAM_ATTRIBUTE } from '../core/gen-ai-attributes'; -import { extractSystemInstructions, getTokenUsageAttributes, setOutputMessagesAttribute } from '../core/utils'; -import { addResponseAttributes as addEvaluateResponseAttributes, getEvaluationInputMessages } from '../typesafe'; +import { + extractSystemInstructions, + getEvaluationInputMessages, + getTokenUsageAttributes, + setOutputMessagesAttribute, +} from '../core/utils'; +import { addResponseAttributes as addEvaluateResponseAttributes } from '../typesafe'; // Re-exported so `workers-ai/streaming.ts` keeps importing it from this module. export { setOutputMessagesAttribute }; import { WORKERS_AI_ORIGIN, WORKERS_AI_PROVIDER_NAME } from './constants'; diff --git a/packages/server-utils/src/integrations/mastra-classifier.ts b/packages/server-utils/src/integrations/mastra-classifier.ts index 310651bff301..d25434bb2c9d 100644 --- a/packages/server-utils/src/integrations/mastra-classifier.ts +++ b/packages/server-utils/src/integrations/mastra-classifier.ts @@ -2,7 +2,7 @@ import { GEN_AI_INPUT_MESSAGES, GEN_AI_OUTPUT_MESSAGES } from '@sentry/conventio import { isObjectLike } from '@sentry/core'; import type { ClassifierEvaluationCall } from '../ai/mastra/classifier-evaluation'; import { processClassifierEvaluationData, setStartingClassifierEvaluation } from '../ai/mastra/classifier-evaluation'; -import { getEvaluationInputMessages, getEvaluationOutputMessages } from '../ai/typesafe'; +import { getEvaluationInputMessages, getEvaluationOutputMessages } from '../ai/core/utils'; import { CHANNELS } from '../orchestrion/channels'; import { safeChannelCallback } from '../tracing-channel'; import * as diagnosticsChannel from '../utils/diagnosticsChannel'; From 17acb4bc322507ad6d02af1c89f4535daa26c0be Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Thu, 8 Oct 2026 14:15:59 +0200 Subject: [PATCH 12/12] Assert classifier tests on streamed spans --- .../suites/tracing/mastra/test.ts | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/dev-packages/node-integration-tests/suites/tracing/mastra/test.ts b/dev-packages/node-integration-tests/suites/tracing/mastra/test.ts index cbbaf3b128f2..d26ea2281428 100644 --- a/dev-packages/node-integration-tests/suites/tracing/mastra/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/mastra/test.ts @@ -223,12 +223,13 @@ conditionalTest({ min: 22 })('Mastra integration', () => { (createRunner, test) => { test('maps a classifier evaluation to a gen_ai.evaluate span', async () => { await createRunner() - .expect({ transaction: { transaction: 'mastra-test' } }) .expect({ span: container => { - expect(container.items.map(span => span.name)).toEqual(['evaluate jev-latest']); + expect(container.items.find(span => span.is_segment && span.name === 'mastra-test')).toBeDefined(); + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.mastra'); + expect(spans.map(span => span.name)).toEqual(['evaluate jev-latest']); - const evaluateSpan = container.items[0]!; + const evaluateSpan = spans[0]!; expect(evaluateSpan.attributes[SENTRY_OP].value).toBe(GEN_AI_EVALUATE); expect(evaluateSpan.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.mastra'); expect(evaluateSpan.attributes[GEN_AI_OPERATION_NAME].value).toBe('evaluate'); @@ -255,9 +256,9 @@ conditionalTest({ min: 22 })('Mastra integration', () => { (createRunner, test) => { test('records the evaluated state, questions and answers when genAI recording is on', async () => { await createRunner() - .expect({ transaction: { transaction: 'mastra-test' } }) .expect({ span: container => { + expect(container.items.find(span => span.is_segment && span.name === 'mastra-test')).toBeDefined(); const evaluateSpan = container.items.find(span => span.name === 'evaluate jev-latest')!; expect(JSON.parse(evaluateSpan.attributes[GEN_AI_INPUT_MESSAGES].value)).toEqual([ { @@ -288,12 +289,13 @@ conditionalTest({ min: 22 })('Mastra integration', () => { test('ends a failed classifier evaluation span with an error status', async () => { await createRunner() .ignore('event') - .expect({ transaction: { transaction: 'mastra-test' } }) .expect({ span: container => { - expect(container.items.map(span => span.name)).toEqual(['evaluate jev-latest']); + expect(container.items.find(span => span.is_segment && span.name === 'mastra-test')).toBeDefined(); + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.mastra'); + expect(spans.map(span => span.name)).toEqual(['evaluate jev-latest']); - const evaluateSpan = container.items[0]!; + const evaluateSpan = spans[0]!; expect(evaluateSpan.status).toBe('error'); expect(evaluateSpan.attributes[GEN_AI_INPUT_MESSAGES].value).toContain('I was charged twice.'); expect(evaluateSpan.attributes[GEN_AI_OUTPUT_MESSAGES]).toBeUndefined();