From 6ef18d2a2f09e269ea2ae8ba28440d494beb48be Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Tue, 6 Oct 2026 10:49:43 +0200 Subject: [PATCH 1/8] test(node): Port OpenAI integration tests to span streaming Co-Authored-By: GPT-6 --- .../node-suites/excludes.ts | 1 + .../openai/instrument-span-streaming.mjs | 11 - .../openai/instrument-with-options.mjs | 7 - .../tracing/openai/instrument-with-pii.mjs | 7 - .../suites/tracing/openai/instrument.mjs | 7 - .../openai-tool-calls/instrument-with-pii.mjs | 1 - .../openai/openai-tool-calls/instrument.mjs | 1 - .../tracing/openai/openai-tool-calls/test.ts | 28 +- .../instrument.mjs} | 0 .../suites/tracing/openai/static/test.ts | 63 +++ .../suites/tracing/openai/test.ts | 412 +++++++----------- .../openai/v6/instrument-root-span.mjs | 11 - .../openai/v6/instrument-with-options.mjs | 7 - .../tracing/openai/v6/instrument-with-pii.mjs | 7 - .../suites/tracing/openai/v6/instrument.mjs | 7 - .../suites/tracing/openai/v6/test.ts | 207 +++++---- .../openai/v7/instrument-span-streaming.mjs | 11 - .../suites/tracing/openai/v7/instrument.mjs | 7 - .../suites/tracing/openai/v7/test.ts | 69 +-- 19 files changed, 346 insertions(+), 518 deletions(-) delete mode 100644 dev-packages/node-integration-tests/suites/tracing/openai/instrument-span-streaming.mjs rename dev-packages/node-integration-tests/suites/tracing/openai/{instrument-root-span.mjs => static/instrument.mjs} (100%) create mode 100644 dev-packages/node-integration-tests/suites/tracing/openai/static/test.ts delete mode 100644 dev-packages/node-integration-tests/suites/tracing/openai/v6/instrument-root-span.mjs delete mode 100644 dev-packages/node-integration-tests/suites/tracing/openai/v7/instrument-span-streaming.mjs diff --git a/dev-packages/bun-integration-tests/node-suites/excludes.ts b/dev-packages/bun-integration-tests/node-suites/excludes.ts index d27790a79a0f..177cdf83d94f 100644 --- a/dev-packages/bun-integration-tests/node-suites/excludes.ts +++ b/dev-packages/bun-integration-tests/node-suites/excludes.ts @@ -121,6 +121,7 @@ export const NO_AUTO_INSTRUMENTATION = [ 'suites/tracing/mysql2-tracing-channel/test.ts', 'suites/tracing/mysql2/test.ts', 'suites/tracing/openai/test.ts', + 'suites/tracing/openai/static/test.ts', 'suites/tracing/openai/v6/test.ts', 'suites/tracing/openai/v7/test.ts', 'suites/tracing/orchestrion-lazy-registration/test.ts', diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/instrument-span-streaming.mjs b/dev-packages/node-integration-tests/suites/tracing/openai/instrument-span-streaming.mjs deleted file mode 100644 index 657bed0a3a8c..000000000000 --- a/dev-packages/node-integration-tests/suites/tracing/openai/instrument-span-streaming.mjs +++ /dev/null @@ -1,11 +0,0 @@ -import * as Sentry from '@sentry/node'; -import { loggingTransport } from '@sentry-internal/node-integration-tests'; - -Sentry.init({ - dsn: 'https://public@dsn.ingest.sentry.io/1337', - release: '1.0', - tracesSampleRate: 1.0, - dataCollection: { genAI: { inputs: true, outputs: true } }, - transport: loggingTransport, - traceLifecycle: 'stream', -}); diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/instrument-with-options.mjs b/dev-packages/node-integration-tests/suites/tracing/openai/instrument-with-options.mjs index 0d6548af1291..913758cd6078 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/instrument-with-options.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/openai/instrument-with-options.mjs @@ -2,7 +2,6 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; Sentry.init({ - traceLifecycle: 'static', dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, @@ -14,10 +13,4 @@ Sentry.init({ recordOutputs: true, }), ], - beforeSendTransaction: event => { - if (event.transaction.includes('/openai/')) { - return null; - } - return event; - }, }); diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/instrument-with-pii.mjs b/dev-packages/node-integration-tests/suites/tracing/openai/instrument-with-pii.mjs index 2b78a3ea1987..1c507ba84b9c 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/instrument-with-pii.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/openai/instrument-with-pii.mjs @@ -2,16 +2,9 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; Sentry.init({ - traceLifecycle: 'static', dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, dataCollection: { genAI: { inputs: true, outputs: true } }, transport: loggingTransport, - beforeSendTransaction: event => { - if (event.transaction.includes('/openai/')) { - return null; - } - return event; - }, }); diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/openai/instrument.mjs index 2097d76a4eff..cc192fb89834 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/instrument.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/openai/instrument.mjs @@ -2,16 +2,9 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; Sentry.init({ - traceLifecycle: 'static', dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, dataCollection: { genAI: { inputs: false, outputs: false } }, transport: loggingTransport, - beforeSendTransaction: event => { - if (event.transaction.includes('/openai/')) { - return null; - } - return event; - }, }); diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/openai-tool-calls/instrument-with-pii.mjs b/dev-packages/node-integration-tests/suites/tracing/openai/openai-tool-calls/instrument-with-pii.mjs index bdf24d6bbd0e..1c507ba84b9c 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/openai-tool-calls/instrument-with-pii.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/openai/openai-tool-calls/instrument-with-pii.mjs @@ -2,7 +2,6 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; Sentry.init({ - traceLifecycle: 'static', dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/openai-tool-calls/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/openai/openai-tool-calls/instrument.mjs index 42052d281304..cc192fb89834 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/openai-tool-calls/instrument.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/openai/openai-tool-calls/instrument.mjs @@ -2,7 +2,6 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; Sentry.init({ - traceLifecycle: 'static', dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/openai-tool-calls/test.ts b/dev-packages/node-integration-tests/suites/tracing/openai/openai-tool-calls/test.ts index 5dfd13c21f7f..f5ca83f3b3f8 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/openai-tool-calls/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/openai/openai-tool-calls/test.ts @@ -79,11 +79,13 @@ describe('OpenAI Tool Calls integration', () => { createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument.mjs', (createRunner, test) => { test('creates openai tool calls related spans with genAI recording disabled', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - expect(container.items).toHaveLength(4); - const chatToolsSpan = container.items.find( + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + expect(spans).toHaveLength(4); + const chatToolsSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-tools-123', ); expect(chatToolsSpan).toBeDefined(); @@ -133,7 +135,7 @@ describe('OpenAI Tool Calls integration', () => { value: 40, }); - const streamingChatToolsSpan = container.items.find( + const streamingChatToolsSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-stream-tools-123', ); expect(streamingChatToolsSpan).toBeDefined(); @@ -193,7 +195,7 @@ describe('OpenAI Tool Calls integration', () => { value: 40, }); - const responsesToolsSpan = container.items.find( + const responsesToolsSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_tools_789', ); expect(responsesToolsSpan).toBeDefined(); @@ -246,7 +248,7 @@ describe('OpenAI Tool Calls integration', () => { value: 20, }); - const streamingResponsesToolsSpan = container.items.find( + const streamingResponsesToolsSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_stream_tools_789', ); expect(streamingResponsesToolsSpan).toBeDefined(); @@ -315,11 +317,13 @@ describe('OpenAI Tool Calls integration', () => { createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { test('creates openai tool calls related spans with genAI recording enabled', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - expect(container.items).toHaveLength(4); - const chatToolsSpan = container.items.find( + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + expect(spans).toHaveLength(4); + const chatToolsSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-tools-123', ); expect(chatToolsSpan).toBeDefined(); @@ -383,7 +387,7 @@ describe('OpenAI Tool Calls integration', () => { value: 40, }); - const streamingChatToolsSpan = container.items.find( + const streamingChatToolsSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-stream-tools-123', ); expect(streamingChatToolsSpan).toBeDefined(); @@ -454,7 +458,7 @@ describe('OpenAI Tool Calls integration', () => { value: 40, }); - const responsesToolsSpan = container.items.find( + const responsesToolsSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_tools_789', ); expect(responsesToolsSpan).toBeDefined(); @@ -518,7 +522,7 @@ describe('OpenAI Tool Calls integration', () => { value: 20, }); - const streamingResponsesToolsSpan = container.items.find( + const streamingResponsesToolsSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_stream_tools_789', ); expect(streamingResponsesToolsSpan).toBeDefined(); diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/instrument-root-span.mjs b/dev-packages/node-integration-tests/suites/tracing/openai/static/instrument.mjs similarity index 100% rename from dev-packages/node-integration-tests/suites/tracing/openai/instrument-root-span.mjs rename to dev-packages/node-integration-tests/suites/tracing/openai/static/instrument.mjs diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/static/test.ts b/dev-packages/node-integration-tests/suites/tracing/openai/static/test.ts new file mode 100644 index 000000000000..b25100db6add --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/openai/static/test.ts @@ -0,0 +1,63 @@ +import { afterAll, describe, expect } from 'vitest'; +import { + GEN_AI_OPERATION_NAME, + GEN_AI_PROVIDER_NAME, + GEN_AI_REQUEST_MODEL, + GEN_AI_REQUEST_TEMPERATURE, + GEN_AI_RESPONSE_FINISH_REASONS, + GEN_AI_RESPONSE_ID, + GEN_AI_RESPONSE_MODEL, + GEN_AI_USAGE_INPUT_TOKENS, + GEN_AI_USAGE_OUTPUT_TOKENS, + GEN_AI_USAGE_TOTAL_TOKENS, + SENTRY_OP, + SENTRY_ORIGIN, +} from '@sentry/conventions/attributes'; +import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/runner'; + +describe('OpenAI integration > static lifecycle', () => { + afterAll(() => { + cleanupChildProcesses(); + }); + createEsmAndCjsTests(__dirname, '../scenario-root-span.mjs', 'instrument.mjs', (createRunner, test) => { + test('it works without a wrapping span', async () => { + await createRunner() + // First the span that our mock express server is emitting, unrelated to this test + .expect({ + transaction: { + transaction: 'POST /openai/chat/completions', + }, + }) + .expect({ + transaction: { + transaction: 'chat gpt-3.5-turbo', + contexts: { + trace: { + span_id: expect.any(String), + trace_id: expect.any(String), + data: { + [GEN_AI_OPERATION_NAME]: 'chat', + [SENTRY_OP]: 'gen_ai.chat', + [SENTRY_ORIGIN]: 'auto.ai.openai', + [GEN_AI_PROVIDER_NAME]: 'openai', + [GEN_AI_REQUEST_MODEL]: 'gpt-3.5-turbo', + [GEN_AI_REQUEST_TEMPERATURE]: 0.7, + [GEN_AI_RESPONSE_MODEL]: 'gpt-3.5-turbo', + [GEN_AI_RESPONSE_ID]: 'chatcmpl-mock123', + [GEN_AI_RESPONSE_FINISH_REASONS]: '["stop"]', + [GEN_AI_USAGE_INPUT_TOKENS]: 10, + [GEN_AI_USAGE_OUTPUT_TOKENS]: 15, + [GEN_AI_USAGE_TOTAL_TOKENS]: 25, + }, + op: 'gen_ai.chat', + origin: 'auto.ai.openai', + status: 'ok', + }, + }, + }, + }) + .start() + .completed(); + }); + }); +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/test.ts b/dev-packages/node-integration-tests/suites/tracing/openai/test.ts index 2da282ebabb4..bb10be74099c 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/openai/test.ts @@ -34,12 +34,18 @@ describe('OpenAI integration', () => { createEsmAndCjsTests(__dirname, 'scenario-chat.mjs', 'instrument.mjs', (createRunner, test) => { test('creates openai related spans with genAI recording disabled', async () => { + const allSpans: SerializedStreamedSpanContainer['items'] = []; + await createRunner() - .expect({ transaction: { transaction: 'main' } }) + .unordered() .expect({ span: container => { - expect(container.items).toHaveLength(8); - const chatCompletionSpan = container.items.find( + allSpans.push(...container.items); + const segment = allSpans.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + expect(spans).toHaveLength(8); + const chatCompletionSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123' && span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-3.5-turbo', @@ -96,7 +102,7 @@ describe('OpenAI integration', () => { value: 25, }); - const responsesSpan = container.items.find( + const responsesSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_mock456' && span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-3.5-turbo', @@ -146,7 +152,7 @@ describe('OpenAI integration', () => { value: 13, }); - const nonStreamingErrorSpan = container.items.find( + const nonStreamingErrorSpan = spans.find( span => span.name === 'chat error-model' && span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE] === undefined, ); @@ -175,7 +181,7 @@ describe('OpenAI integration', () => { value: 'error-model', }); - const streamingChatCompletionSpan = container.items.find( + const streamingChatCompletionSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-stream-123', ); expect(streamingChatCompletionSpan).toBeDefined(); @@ -238,7 +244,7 @@ describe('OpenAI integration', () => { value: true, }); - const streamingResponsesSpan = container.items.find( + const streamingResponsesSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_stream_456', ); expect(streamingResponsesSpan).toBeDefined(); @@ -297,7 +303,7 @@ describe('OpenAI integration', () => { value: true, }); - const streamingErrorSpan = container.items.find( + const streamingErrorSpan = spans.find( span => span.name === 'chat error-model' && span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]?.value === true, ); @@ -329,7 +335,7 @@ describe('OpenAI integration', () => { value: 'auto.ai.openai', }); - const parsedChatCompletionSpan = container.items.find( + const parsedChatCompletionSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123' && span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-4o', @@ -382,7 +388,7 @@ describe('OpenAI integration', () => { value: 25, }); - const parsedResponsesSpan = container.items.find( + const parsedResponsesSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_mock456' && span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-4o', @@ -433,9 +439,9 @@ describe('OpenAI integration', () => { }); // GenAI inputs and outputs must never be recorded when `dataCollection.genAI` disables them. - // Asserting over every span in the envelope catches attributes leaking onto + // Asserting over every GenAI span catches attributes leaking onto // spans which are not individually inspected above. - for (const span of container.items) { + for (const span of spans) { expect(span.attributes[GEN_AI_INPUT_MESSAGES]).toBeUndefined(); expect(span.attributes[GEN_AI_SYSTEM_INSTRUCTIONS]).toBeUndefined(); expect(span.attributes[GEN_AI_RESPONSE_TEXT]).toBeUndefined(); @@ -450,12 +456,18 @@ describe('OpenAI integration', () => { createEsmAndCjsTests(__dirname, 'scenario-chat.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { test('creates openai related spans with genAI recording enabled', async () => { + const allSpans: SerializedStreamedSpanContainer['items'] = []; + await createRunner() - .expect({ transaction: { transaction: 'main' } }) + .unordered() .expect({ span: container => { - expect(container.items).toHaveLength(8); - const chatCompletionSpan = container.items.find( + allSpans.push(...container.items); + const segment = allSpans.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + expect(spans).toHaveLength(8); + const chatCompletionSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123' && span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-3.5-turbo', @@ -524,7 +536,7 @@ describe('OpenAI integration', () => { value: 25, }); - const responsesSpan = container.items.find( + const responsesSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_mock456' && span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-3.5-turbo', @@ -582,7 +594,7 @@ describe('OpenAI integration', () => { value: 13, }); - const nonStreamingErrorSpan = container.items.find( + const nonStreamingErrorSpan = spans.find( span => span.name === 'chat error-model' && span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE] === undefined, ); @@ -615,7 +627,7 @@ describe('OpenAI integration', () => { value: '[{"role":"user","content":"This will fail"}]', }); - const streamingChatCompletionSpan = container.items.find( + const streamingChatCompletionSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-stream-123', ); expect(streamingChatCompletionSpan).toBeDefined(); @@ -690,7 +702,7 @@ describe('OpenAI integration', () => { value: true, }); - const streamingResponsesSpan = container.items.find( + const streamingResponsesSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_stream_456', ); expect(streamingResponsesSpan).toBeDefined(); @@ -757,7 +769,7 @@ describe('OpenAI integration', () => { value: true, }); - const streamingErrorSpan = container.items.find( + const streamingErrorSpan = spans.find( span => span.name === 'chat error-model' && span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]?.value === true, ); @@ -793,7 +805,7 @@ describe('OpenAI integration', () => { value: 'auto.ai.openai', }); - const parsedChatCompletionSpan = container.items.find( + const parsedChatCompletionSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123' && span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-4o', @@ -855,7 +867,7 @@ describe('OpenAI integration', () => { value: 25, }); - const parsedResponsesSpan = container.items.find( + const parsedResponsesSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_mock456' && span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-4o', @@ -921,12 +933,18 @@ describe('OpenAI integration', () => { createEsmAndCjsTests(__dirname, 'scenario-chat.mjs', 'instrument-with-options.mjs', (createRunner, test) => { test('creates openai related spans with custom options', async () => { + const allSpans: SerializedStreamedSpanContainer['items'] = []; + await createRunner() - .expect({ transaction: { transaction: 'main' } }) + .unordered() .expect({ span: container => { - expect(container.items).toHaveLength(8); - const chatCompletionSpan = container.items.find( + allSpans.push(...container.items); + const segment = allSpans.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + expect(spans).toHaveLength(8); + const chatCompletionSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123' && span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-3.5-turbo', @@ -942,7 +960,7 @@ describe('OpenAI integration', () => { value: expect.any(String), }); - const streamingChatCompletionSpan = container.items.find( + const streamingChatCompletionSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-stream-123', ); expect(streamingChatCompletionSpan).toBeDefined(); @@ -959,7 +977,7 @@ describe('OpenAI integration', () => { value: expect.any(String), }); - const parsedChatCompletionSpan = container.items.find( + const parsedChatCompletionSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123' && span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-4o', @@ -975,7 +993,7 @@ describe('OpenAI integration', () => { value: expect.any(String), }); - const parsedResponsesSpan = container.items.find( + const parsedResponsesSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_mock456' && span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-4o', @@ -1000,15 +1018,13 @@ describe('OpenAI integration', () => { createEsmAndCjsTests(__dirname, 'scenario-embeddings.mjs', 'instrument.mjs', (createRunner, test) => { test('creates openai related spans with genAI recording disabled', async () => { await createRunner() - .expect({ - transaction: { - transaction: 'main', - }, - }) .expect({ span: container => { - expect(container.items).toHaveLength(4); - const singleEmbeddingSpan = container.items.find( + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + expect(spans).toHaveLength(4); + const singleEmbeddingSpan = spans.find( span => span.name === 'embeddings text-embedding-3-small' && span.attributes[GEN_AI_REQUEST_DIMENSIONS_ATTRIBUTE] !== undefined, @@ -1057,7 +1073,7 @@ describe('OpenAI integration', () => { value: 10, }); - const errorEmbeddingSpan = container.items.find(span => span.name === 'embeddings error-model'); + const errorEmbeddingSpan = spans.find(span => span.name === 'embeddings error-model'); expect(errorEmbeddingSpan).toBeDefined(); expect(errorEmbeddingSpan!.name).toBe('embeddings error-model'); expect(errorEmbeddingSpan!.status).toBe('error'); @@ -1082,7 +1098,7 @@ describe('OpenAI integration', () => { value: 'error-model', }); - const multiEmbeddingSpan = container.items.find( + const multiEmbeddingSpan = spans.find( span => span.name === 'embeddings text-embedding-3-small' && span.attributes[GEN_AI_REQUEST_DIMENSIONS_ATTRIBUTE] === undefined, @@ -1126,14 +1142,14 @@ describe('OpenAI integration', () => { // GenAI inputs and outputs must never be recorded when `dataCollection.genAI` disables them. // Asserting over every span in the envelope catches attributes leaking onto // spans which are not individually inspected above. - for (const span of container.items) { + for (const span of spans) { expect(span.attributes[GEN_AI_INPUT_MESSAGES]).toBeUndefined(); expect(span.attributes[GEN_AI_SYSTEM_INSTRUCTIONS]).toBeUndefined(); expect(span.attributes[GEN_AI_RESPONSE_TEXT]).toBeUndefined(); expect(span.attributes[GEN_AI_EMBEDDINGS_INPUT]).toBeUndefined(); } - const rawEmbeddingSpan = container.items.find(span => span.name === 'embeddings text-embedding-3-large'); + const rawEmbeddingSpan = spans.find(span => span.name === 'embeddings text-embedding-3-large'); expect(rawEmbeddingSpan).toBeDefined(); expect(rawEmbeddingSpan!.name).toBe('embeddings text-embedding-3-large'); expect(rawEmbeddingSpan!.status).toBe('ok'); @@ -1170,15 +1186,13 @@ describe('OpenAI integration', () => { createEsmAndCjsTests(__dirname, 'scenario-embeddings.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { test('creates openai related spans with genAI recording enabled', async () => { await createRunner() - .expect({ - transaction: { - transaction: 'main', - }, - }) .expect({ span: container => { - expect(container.items).toHaveLength(4); - const singleEmbeddingSpan = container.items.find( + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + expect(spans).toHaveLength(4); + const singleEmbeddingSpan = spans.find( span => span.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value === 'Embedding test!', ); expect(singleEmbeddingSpan).toBeDefined(); @@ -1229,7 +1243,7 @@ describe('OpenAI integration', () => { value: 10, }); - const errorEmbeddingSpan = container.items.find( + const errorEmbeddingSpan = spans.find( span => span.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value === 'Error embedding test!', ); expect(errorEmbeddingSpan).toBeDefined(); @@ -1260,7 +1274,7 @@ describe('OpenAI integration', () => { value: 'Error embedding test!', }); - const multiEmbeddingSpan = container.items.find( + const multiEmbeddingSpan = spans.find( span => span.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value === '["First input text","Second input text","Third input text"]', @@ -1305,7 +1319,7 @@ describe('OpenAI integration', () => { value: 10, }); - const rawEmbeddingSpan = container.items.find(span => span.name === 'embeddings text-embedding-3-large'); + const rawEmbeddingSpan = spans.find(span => span.name === 'embeddings text-embedding-3-large'); expect(rawEmbeddingSpan).toBeDefined(); expect(rawEmbeddingSpan!.name).toBe('embeddings text-embedding-3-large'); expect(rawEmbeddingSpan!.status).toBe('ok'); @@ -1343,41 +1357,31 @@ describe('OpenAI integration', () => { }); }); - createEsmAndCjsTests(__dirname, 'scenario-root-span.mjs', 'instrument-root-span.mjs', (createRunner, test) => { + createEsmAndCjsTests(__dirname, 'scenario-root-span.mjs', 'instrument.mjs', (createRunner, test) => { test('it works without a wrapping span', async () => { await createRunner() - // First the span that our mock express server is emitting, unrelated to this test .expect({ - transaction: { - transaction: 'POST /openai/chat/completions', - }, - }) - .expect({ - transaction: { - transaction: 'chat gpt-3.5-turbo', - contexts: { - trace: { - span_id: expect.any(String), - trace_id: expect.any(String), - data: { - [GEN_AI_OPERATION_NAME]: 'chat', - [SENTRY_OP]: 'gen_ai.chat', - [SENTRY_ORIGIN]: 'auto.ai.openai', - [GEN_AI_PROVIDER_NAME]: 'openai', - [GEN_AI_REQUEST_MODEL]: 'gpt-3.5-turbo', - [GEN_AI_REQUEST_TEMPERATURE]: 0.7, - [GEN_AI_RESPONSE_MODEL]: 'gpt-3.5-turbo', - [GEN_AI_RESPONSE_ID]: 'chatcmpl-mock123', - [GEN_AI_RESPONSE_FINISH_REASONS]: '["stop"]', - [GEN_AI_USAGE_INPUT_TOKENS]: 10, - [GEN_AI_USAGE_OUTPUT_TOKENS]: 15, - [GEN_AI_USAGE_TOTAL_TOKENS]: 25, - }, - op: 'gen_ai.chat', - origin: 'auto.ai.openai', - status: 'ok', - }, - }, + span: container => { + const serverSegment = container.items.find( + span => span.is_segment && span.name === 'POST /openai/chat/completions', + ); + expect(serverSegment).toBeDefined(); + const segment = container.items.find(span => span.is_segment && span.name === 'chat gpt-3.5-turbo'); + expect(segment?.span_id).toEqual(expect.any(String)); + expect(segment?.trace_id).toEqual(expect.any(String)); + expect(segment?.attributes[GEN_AI_OPERATION_NAME]).toEqual({ value: 'chat', type: 'string' }); + expect(segment?.attributes[SENTRY_OP]).toEqual({ value: 'gen_ai.chat', type: 'string' }); + expect(segment?.attributes[SENTRY_ORIGIN]).toEqual({ value: 'auto.ai.openai', type: 'string' }); + expect(segment?.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ value: 'openai', type: 'string' }); + expect(segment?.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ value: 'gpt-3.5-turbo', type: 'string' }); + expect(segment?.attributes[GEN_AI_REQUEST_TEMPERATURE]).toEqual({ value: 0.7, type: 'double' }); + expect(segment?.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ value: 'gpt-3.5-turbo', type: 'string' }); + expect(segment?.attributes[GEN_AI_RESPONSE_ID]).toEqual({ value: 'chatcmpl-mock123', type: 'string' }); + expect(segment?.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ value: '["stop"]', type: 'string' }); + expect(segment?.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ value: 10, type: 'integer' }); + expect(segment?.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ value: 15, type: 'integer' }); + expect(segment?.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ value: 25, type: 'integer' }); + expect(segment?.status).toEqual('ok'); }, }) .start() @@ -1388,38 +1392,28 @@ describe('OpenAI integration', () => { createEsmAndCjsTests(__dirname, 'scenario-azure-openai.mjs', 'instrument.mjs', (createRunner, test) => { test('it works with Azure OpenAI', async () => { await createRunner() - // First the span that our mock express server is emitting, unrelated to this test .expect({ - transaction: { - transaction: 'POST /azureopenai/deployments/:model/chat/completions', - }, - }) - .expect({ - transaction: { - transaction: 'chat gpt-3.5-turbo', - contexts: { - trace: { - span_id: expect.any(String), - trace_id: expect.any(String), - data: { - [GEN_AI_OPERATION_NAME]: 'chat', - [SENTRY_OP]: 'gen_ai.chat', - [SENTRY_ORIGIN]: 'auto.ai.openai', - [GEN_AI_PROVIDER_NAME]: 'openai', - [GEN_AI_REQUEST_MODEL]: 'gpt-3.5-turbo', - [GEN_AI_REQUEST_TEMPERATURE]: 0.7, - [GEN_AI_RESPONSE_MODEL]: 'gpt-3.5-turbo', - [GEN_AI_RESPONSE_ID]: 'chatcmpl-mock123', - [GEN_AI_RESPONSE_FINISH_REASONS]: '["stop"]', - [GEN_AI_USAGE_INPUT_TOKENS]: 10, - [GEN_AI_USAGE_OUTPUT_TOKENS]: 15, - [GEN_AI_USAGE_TOTAL_TOKENS]: 25, - }, - op: 'gen_ai.chat', - origin: 'auto.ai.openai', - status: 'ok', - }, - }, + span: container => { + const serverSegment = container.items.find( + span => span.is_segment && span.name === 'POST /azureopenai/deployments/:model/chat/completions', + ); + expect(serverSegment).toBeDefined(); + const segment = container.items.find(span => span.is_segment && span.name === 'chat gpt-3.5-turbo'); + expect(segment?.span_id).toEqual(expect.any(String)); + expect(segment?.trace_id).toEqual(expect.any(String)); + expect(segment?.attributes[GEN_AI_OPERATION_NAME]).toEqual({ value: 'chat', type: 'string' }); + expect(segment?.attributes[SENTRY_OP]).toEqual({ value: 'gen_ai.chat', type: 'string' }); + expect(segment?.attributes[SENTRY_ORIGIN]).toEqual({ value: 'auto.ai.openai', type: 'string' }); + expect(segment?.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ value: 'openai', type: 'string' }); + expect(segment?.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ value: 'gpt-3.5-turbo', type: 'string' }); + expect(segment?.attributes[GEN_AI_REQUEST_TEMPERATURE]).toEqual({ value: 0.7, type: 'double' }); + expect(segment?.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ value: 'gpt-3.5-turbo', type: 'string' }); + expect(segment?.attributes[GEN_AI_RESPONSE_ID]).toEqual({ value: 'chatcmpl-mock123', type: 'string' }); + expect(segment?.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ value: '["stop"]', type: 'string' }); + expect(segment?.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ value: 10, type: 'integer' }); + expect(segment?.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ value: 15, type: 'integer' }); + expect(segment?.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ value: 25, type: 'integer' }); + expect(segment?.status).toEqual('ok'); }, }) .start() @@ -1431,17 +1425,15 @@ describe('OpenAI integration', () => { createEsmAndCjsTests(__dirname, 'scenario-conversation.mjs', 'instrument.mjs', (createRunner, test) => { test('captures conversation ID from Conversations API and previous_response_id', async () => { await createRunner() - .expect({ - transaction: { - transaction: 'conversation-test', - }, - }) .expect({ span: container => { - expect(container.items).toHaveLength(5); - const conversationCreateSpan = container.items.find(span => span.name === 'chat unknown'); + const segment = container.items.find(span => span.is_segment && span.name === 'conversation-test'); + expect(segment).toBeDefined(); + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + expect(spans).toHaveLength(5); + const conversationCreateSpan = spans.find(span => span.name === 'chat'); expect(conversationCreateSpan).toBeDefined(); - expect(conversationCreateSpan!.name).toBe('chat unknown'); + expect(conversationCreateSpan!.name).toBe('chat'); expect(conversationCreateSpan!.status).toBe('ok'); expect(conversationCreateSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ type: 'string', @@ -1464,7 +1456,7 @@ describe('OpenAI integration', () => { value: 'conv_689667905b048191b4740501625afd940c7533ace33a2dab', }); - const conversationResponseSpan = container.items.find( + const conversationResponseSpan = spans.find( span => span.attributes[GEN_AI_CONVERSATION_ID]?.value === 'conv_689667905b048191b4740501625afd940c7533ace33a2dab' && @@ -1497,7 +1489,7 @@ describe('OpenAI integration', () => { value: 'conv_689667905b048191b4740501625afd940c7533ace33a2dab', }); - const unlinkedResponseSpan = container.items.find( + const unlinkedResponseSpan = spans.find( span => span.attributes[SENTRY_OP]?.value === 'gen_ai.chat' && span.attributes[GEN_AI_CONVERSATION_ID] === undefined, @@ -1510,7 +1502,7 @@ describe('OpenAI integration', () => { }); expect(unlinkedResponseSpan!.attributes[GEN_AI_CONVERSATION_ID]).toBeUndefined(); - const previousResponseSpan = container.items.find( + const previousResponseSpan = spans.find( span => span.attributes[GEN_AI_CONVERSATION_ID]?.value === 'resp_mock_conv_123', ); expect(previousResponseSpan).toBeDefined(); @@ -1540,11 +1532,11 @@ describe('OpenAI integration', () => { value: 'resp_mock_conv_123', }); - const rawConversationSpan = container.items.find( - span => span.name === 'chat unknown' && span.attributes[GEN_AI_CONVERSATION_ID] === undefined, + const rawConversationSpan = spans.find( + span => span.name === 'chat' && span.attributes[GEN_AI_CONVERSATION_ID] === undefined, ); expect(rawConversationSpan).toBeDefined(); - expect(rawConversationSpan!.name).toBe('chat unknown'); + expect(rawConversationSpan!.name).toBe('chat'); expect(rawConversationSpan!.status).toBe('ok'); expect(rawConversationSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ type: 'string', @@ -1574,17 +1566,17 @@ describe('OpenAI integration', () => { createEsmAndCjsTests(__dirname, 'scenario-manual-conversation-id.mjs', 'instrument.mjs', (createRunner, test) => { test('attaches manual conversation ID set via setConversationId() to all chat spans', async () => { await createRunner() - .expect({ - transaction: { - transaction: 'chat-with-manual-conversation-id', - }, - }) .expect({ span: container => { - expect(container.items).toHaveLength(3); + const segment = container.items.find( + span => span.is_segment && span.name === 'chat-with-manual-conversation-id', + ); + expect(segment).toBeDefined(); + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + expect(spans).toHaveLength(3); // All three chat completion spans should have the same manually-set conversation ID - for (const span of container.items) { + for (const span of spans) { expect(span!.name).toBe('chat gpt-4'); expect(span!.status).toBe('ok'); expect(span!.attributes['gen_ai.conversation.id']).toEqual({ @@ -1606,17 +1598,15 @@ describe('OpenAI integration', () => { createEsmAndCjsTests(__dirname, 'scenario-separate-scope-1.mjs', 'instrument.mjs', (createRunner, test) => { test('isolates conversation IDs across separate scopes - conversation 1', async () => { await createRunner() - .expect({ - transaction: { - transaction: 'GET /chat/conversation-1', - }, - }) .expect({ span: container => { - expect(container.items).toHaveLength(2); + const segment = container.items.find(span => span.is_segment && span.name === 'GET /chat/conversation-1'); + expect(segment).toBeDefined(); + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + expect(spans).toHaveLength(2); // Both chat completion spans should have the expected conversation ID - for (const span of container.items) { + for (const span of spans) { expect(span!.name).toBe('chat gpt-4'); expect(span!.status).toBe('ok'); expect(span!.attributes['gen_ai.conversation.id']).toEqual({ @@ -1637,17 +1627,15 @@ describe('OpenAI integration', () => { createEsmAndCjsTests(__dirname, 'scenario-separate-scope-2.mjs', 'instrument.mjs', (createRunner, test) => { test('isolates conversation IDs across separate scopes - conversation 2', async () => { await createRunner() - .expect({ - transaction: { - transaction: 'GET /chat/conversation-2', - }, - }) .expect({ span: container => { - expect(container.items).toHaveLength(2); + const segment = container.items.find(span => span.is_segment && span.name === 'GET /chat/conversation-2'); + expect(segment).toBeDefined(); + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + expect(spans).toHaveLength(2); // Both chat completion spans should have the expected conversation ID - for (const span of container.items) { + for (const span of spans) { expect(span!.name).toBe('chat gpt-4'); expect(span!.status).toBe('ok'); expect(span!.attributes['gen_ai.conversation.id']).toEqual({ @@ -1672,15 +1660,13 @@ describe('OpenAI integration', () => { (createRunner, test) => { test('extracts system instructions from messages', async () => { await createRunner() - .expect({ - transaction: { - transaction: 'main', - }, - }) .expect({ span: container => { - expect(container.items).toHaveLength(1); - const [firstSpan] = container.items; + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + expect(spans).toHaveLength(1); + const [firstSpan] = spans; // [0] chat completion with system instructions extracted from messages expect(firstSpan!.attributes[GEN_AI_SYSTEM_INSTRUCTIONS]).toEqual({ @@ -1698,30 +1684,28 @@ describe('OpenAI integration', () => { createEsmAndCjsTests(__dirname, 'scenario-with-response.mjs', 'instrument.mjs', (createRunner, test) => { test('preserves .withResponse() method and works correctly', async () => { await createRunner() - .expect({ - transaction: { - transaction: 'main', - }, - }) .expect({ span: container => { - expect(container.items).toHaveLength(2); + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + expect(spans).toHaveLength(2); - for (const span of container.items) { + for (const span of spans) { expect(span!.name).toBe('chat gpt-4'); expect(span!.status).toBe('ok'); expect(span!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ type: 'string', value: 'chat' }); expect(span!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ type: 'string', value: 'gpt-4' }); } - const parsedSpans = container.items.filter(span => span.attributes[GEN_AI_RESPONSE_ID] !== undefined); + const parsedSpans = spans.filter(span => span.attributes[GEN_AI_RESPONSE_ID] !== undefined); expect(parsedSpans).toHaveLength(1); expect(parsedSpans[0]!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ type: 'string', value: 'chatcmpl-withresponse', }); - const rawSpans = container.items.filter(span => span.attributes[GEN_AI_RESPONSE_ID] === undefined); + const rawSpans = spans.filter(span => span.attributes[GEN_AI_RESPONSE_ID] === undefined); expect(rawSpans).toHaveLength(1); expect(rawSpans[0]!.attributes[GEN_AI_RESPONSE_TEXT]).toBeUndefined(); expect(rawSpans[0]!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toBeUndefined(); @@ -1731,106 +1715,4 @@ describe('OpenAI integration', () => { .completed(); }); }); - - createEsmAndCjsTests(__dirname, 'scenario-chat.mjs', 'instrument-span-streaming.mjs', (createRunner, test) => { - test('creates openai related spans with span streaming enabled', async () => { - const spans: SerializedStreamedSpanContainer['items'] = []; - - await createRunner() - .unordered() - .expect({ - span: container => { - spans.push(...container.items); - const chatCompletionSpan = spans.find( - span => - span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123' && - span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-3.5-turbo', - ); - expect(chatCompletionSpan).toBeDefined(); - expect(chatCompletionSpan!.name).toBe('chat gpt-3.5-turbo'); - expect(chatCompletionSpan!.status).toBe('ok'); - expect(chatCompletionSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ type: 'string', value: 'chat' }); - expect(chatCompletionSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(chatCompletionSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'gpt-3.5-turbo', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toEqual({ - type: 'string', - value: '[{"role":"user","content":"What is the capital of France?"}]', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ type: 'string', value: 'openai' }); - - const parsedChatCompletionSpan = spans.find( - span => - span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123' && - span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-4o', - ); - expect(parsedChatCompletionSpan).toBeDefined(); - expect(parsedChatCompletionSpan!.name).toBe('chat gpt-4o'); - expect(parsedChatCompletionSpan!.status).toBe('ok'); - expect(parsedChatCompletionSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'chat', - }); - expect(parsedChatCompletionSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(parsedChatCompletionSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(parsedChatCompletionSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'gpt-4o', - }); - expect(parsedChatCompletionSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toEqual({ - type: 'string', - value: '[{"role":"user","content":"What is the capital of France?"}]', - }); - expect(parsedChatCompletionSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - - const parsedResponsesSpan = spans.find( - span => - span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_mock456' && - span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-4o', - ); - expect(parsedResponsesSpan).toBeDefined(); - expect(parsedResponsesSpan!.name).toBe('chat gpt-4o'); - expect(parsedResponsesSpan!.status).toBe('ok'); - expect(parsedResponsesSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ type: 'string', value: 'chat' }); - expect(parsedResponsesSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(parsedResponsesSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(parsedResponsesSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'gpt-4o', - }); - expect(parsedResponsesSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toEqual({ - type: 'string', - value: 'What is the capital of France?', - }); - expect(parsedResponsesSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ type: 'string', value: 'openai' }); - }, - }) - .start() - .completed(); - }); - }); }); diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/v6/instrument-root-span.mjs b/dev-packages/node-integration-tests/suites/tracing/openai/v6/instrument-root-span.mjs deleted file mode 100644 index 42052d281304..000000000000 --- a/dev-packages/node-integration-tests/suites/tracing/openai/v6/instrument-root-span.mjs +++ /dev/null @@ -1,11 +0,0 @@ -import * as Sentry from '@sentry/node'; -import { loggingTransport } from '@sentry-internal/node-integration-tests'; - -Sentry.init({ - traceLifecycle: 'static', - dsn: 'https://public@dsn.ingest.sentry.io/1337', - release: '1.0', - tracesSampleRate: 1.0, - dataCollection: { genAI: { inputs: false, outputs: false } }, - transport: loggingTransport, -}); diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/v6/instrument-with-options.mjs b/dev-packages/node-integration-tests/suites/tracing/openai/v6/instrument-with-options.mjs index 0169719e5d59..8bdeb0ef483f 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/v6/instrument-with-options.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/openai/v6/instrument-with-options.mjs @@ -2,7 +2,6 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; Sentry.init({ - traceLifecycle: 'static', dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, @@ -15,10 +14,4 @@ Sentry.init({ }), ], debug: true, - beforeSendTransaction: event => { - if (event.transaction.includes('/openai/')) { - return null; - } - return event; - }, }); diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/v6/instrument-with-pii.mjs b/dev-packages/node-integration-tests/suites/tracing/openai/v6/instrument-with-pii.mjs index 2b78a3ea1987..1c507ba84b9c 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/v6/instrument-with-pii.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/openai/v6/instrument-with-pii.mjs @@ -2,16 +2,9 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; Sentry.init({ - traceLifecycle: 'static', dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, dataCollection: { genAI: { inputs: true, outputs: true } }, transport: loggingTransport, - beforeSendTransaction: event => { - if (event.transaction.includes('/openai/')) { - return null; - } - return event; - }, }); diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/v6/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/openai/v6/instrument.mjs index 2097d76a4eff..cc192fb89834 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/v6/instrument.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/openai/v6/instrument.mjs @@ -2,16 +2,9 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; Sentry.init({ - traceLifecycle: 'static', dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, dataCollection: { genAI: { inputs: false, outputs: false } }, transport: loggingTransport, - beforeSendTransaction: event => { - if (event.transaction.includes('/openai/')) { - return null; - } - return event; - }, }); diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/v6/test.ts b/dev-packages/node-integration-tests/suites/tracing/openai/v6/test.ts index 49d5cb48bd9d..56cf087de0e9 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/v6/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/openai/v6/test.ts @@ -1,3 +1,4 @@ +import type { SerializedStreamedSpanContainer } from '@sentry/core'; import { afterAll, describe, expect } from 'vitest'; import { GEN_AI_EMBEDDINGS_INPUT, @@ -36,12 +37,18 @@ describe('OpenAI integration (V6)', () => { 'instrument.mjs', (createRunner, test) => { test('creates openai related spans with genAI recording disabled (v6)', async () => { + const allSpans: SerializedStreamedSpanContainer['items'] = []; + await createRunner() - .expect({ transaction: { transaction: 'main' } }) + .unordered() .expect({ span: container => { - expect(container.items).toHaveLength(6); - const chatCompletionSpan = container.items.find( + allSpans.push(...container.items); + const segment = allSpans.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + expect(spans).toHaveLength(6); + const chatCompletionSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123', ); expect(chatCompletionSpan).toBeDefined(); @@ -96,9 +103,7 @@ describe('OpenAI integration (V6)', () => { value: 25, }); - const responsesSpan = container.items.find( - span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_mock456', - ); + const responsesSpan = spans.find(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_mock456'); expect(responsesSpan).toBeDefined(); expect(responsesSpan!.name).toBe('chat gpt-3.5-turbo'); expect(responsesSpan!.status).toBe('ok'); @@ -144,7 +149,7 @@ describe('OpenAI integration (V6)', () => { value: 13, }); - const nonStreamingErrorSpan = container.items.find( + const nonStreamingErrorSpan = spans.find( span => span.name === 'chat error-model' && span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE] === undefined, ); @@ -173,7 +178,7 @@ describe('OpenAI integration (V6)', () => { value: 'error-model', }); - const streamingChatCompletionSpan = container.items.find( + const streamingChatCompletionSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-stream-123', ); expect(streamingChatCompletionSpan).toBeDefined(); @@ -236,7 +241,7 @@ describe('OpenAI integration (V6)', () => { value: true, }); - const streamingResponsesSpan = container.items.find( + const streamingResponsesSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_stream_456', ); expect(streamingResponsesSpan).toBeDefined(); @@ -295,7 +300,7 @@ describe('OpenAI integration (V6)', () => { value: true, }); - const streamingErrorSpan = container.items.find( + const streamingErrorSpan = spans.find( span => span.name === 'chat error-model' && span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]?.value === true, ); @@ -345,12 +350,18 @@ describe('OpenAI integration (V6)', () => { 'instrument-with-pii.mjs', (createRunner, test) => { test('creates openai related spans with genAI recording enabled (v6)', async () => { + const allSpans: SerializedStreamedSpanContainer['items'] = []; + await createRunner() - .expect({ transaction: { transaction: 'main' } }) + .unordered() .expect({ span: container => { - expect(container.items).toHaveLength(6); - const chatCompletionSpan = container.items.find( + allSpans.push(...container.items); + const segment = allSpans.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + expect(spans).toHaveLength(6); + const chatCompletionSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123', ); expect(chatCompletionSpan).toBeDefined(); @@ -417,9 +428,7 @@ describe('OpenAI integration (V6)', () => { value: 25, }); - const responsesSpan = container.items.find( - span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_mock456', - ); + const responsesSpan = spans.find(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_mock456'); expect(responsesSpan).toBeDefined(); expect(responsesSpan!.name).toBe('chat gpt-3.5-turbo'); expect(responsesSpan!.status).toBe('ok'); @@ -473,7 +482,7 @@ describe('OpenAI integration (V6)', () => { value: 13, }); - const nonStreamingErrorSpan = container.items.find( + const nonStreamingErrorSpan = spans.find( span => span.name === 'chat error-model' && span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE] === undefined, ); @@ -506,7 +515,7 @@ describe('OpenAI integration (V6)', () => { value: '[{"role":"user","content":"This will fail"}]', }); - const streamingChatCompletionSpan = container.items.find( + const streamingChatCompletionSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-stream-123', ); expect(streamingChatCompletionSpan).toBeDefined(); @@ -581,7 +590,7 @@ describe('OpenAI integration (V6)', () => { value: true, }); - const streamingResponsesSpan = container.items.find( + const streamingResponsesSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_stream_456', ); expect(streamingResponsesSpan).toBeDefined(); @@ -648,7 +657,7 @@ describe('OpenAI integration (V6)', () => { value: true, }); - const streamingErrorSpan = container.items.find( + const streamingErrorSpan = spans.find( span => span.name === 'chat error-model' && span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]?.value === true, ); @@ -702,12 +711,18 @@ describe('OpenAI integration (V6)', () => { 'instrument-with-options.mjs', (createRunner, test) => { test('creates openai related spans with custom options (v6)', async () => { + const allSpans: SerializedStreamedSpanContainer['items'] = []; + await createRunner() - .expect({ transaction: { transaction: 'main' } }) + .unordered() .expect({ span: container => { - expect(container.items).toHaveLength(6); - const chatCompletionSpan = container.items.find( + allSpans.push(...container.items); + const segment = allSpans.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + expect(spans).toHaveLength(6); + const chatCompletionSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123', ); expect(chatCompletionSpan).toBeDefined(); @@ -725,7 +740,7 @@ describe('OpenAI integration (V6)', () => { value: expect.any(String), }); - const streamingChatCompletionSpan = container.items.find( + const streamingChatCompletionSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-stream-123', ); expect(streamingChatCompletionSpan).toBeDefined(); @@ -765,15 +780,13 @@ describe('OpenAI integration (V6)', () => { (createRunner, test) => { test('creates openai related spans with genAI recording disabled (v6)', async () => { await createRunner() - .expect({ - transaction: { - transaction: 'main', - }, - }) .expect({ span: container => { - expect(container.items).toHaveLength(3); - const singleEmbeddingSpan = container.items.find( + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + expect(spans).toHaveLength(3); + const singleEmbeddingSpan = spans.find( span => span.name === 'embeddings text-embedding-3-small' && span.attributes[GEN_AI_REQUEST_DIMENSIONS_ATTRIBUTE] !== undefined, @@ -822,7 +835,7 @@ describe('OpenAI integration (V6)', () => { value: 10, }); - const errorEmbeddingSpan = container.items.find(span => span.name === 'embeddings error-model'); + const errorEmbeddingSpan = spans.find(span => span.name === 'embeddings error-model'); expect(errorEmbeddingSpan).toBeDefined(); expect(errorEmbeddingSpan!.name).toBe('embeddings error-model'); expect(errorEmbeddingSpan!.status).toBe('error'); @@ -847,7 +860,7 @@ describe('OpenAI integration (V6)', () => { value: 'error-model', }); - const multiEmbeddingSpan = container.items.find( + const multiEmbeddingSpan = spans.find( span => span.name === 'embeddings text-embedding-3-small' && span.attributes[GEN_AI_REQUEST_DIMENSIONS_ATTRIBUTE] === undefined, @@ -907,15 +920,13 @@ describe('OpenAI integration (V6)', () => { (createRunner, test) => { test('creates openai related spans with genAI recording enabled (v6)', async () => { await createRunner() - .expect({ - transaction: { - transaction: 'main', - }, - }) .expect({ span: container => { - expect(container.items).toHaveLength(3); - const singleEmbeddingSpan = container.items.find( + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + expect(spans).toHaveLength(3); + const singleEmbeddingSpan = spans.find( span => span.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value === 'Embedding test!', ); expect(singleEmbeddingSpan).toBeDefined(); @@ -966,7 +977,7 @@ describe('OpenAI integration (V6)', () => { value: 10, }); - const errorEmbeddingSpan = container.items.find( + const errorEmbeddingSpan = spans.find( span => span.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value === 'Error embedding test!', ); expect(errorEmbeddingSpan).toBeDefined(); @@ -997,7 +1008,7 @@ describe('OpenAI integration (V6)', () => { value: 'Error embedding test!', }); - const multiEmbeddingSpan = container.items.find( + const multiEmbeddingSpan = spans.find( span => span.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value === '["First input text","Second input text","Third input text"]', @@ -1057,42 +1068,35 @@ describe('OpenAI integration (V6)', () => { createEsmAndCjsTests( __dirname, 'scenario-root-span.mjs', - 'instrument-root-span.mjs', + 'instrument.mjs', (createRunner, test) => { test('it works without a wrapping span (v6)', async () => { await createRunner() - // First the span that our mock express server is emitting, unrelated to this test - .expect({ - transaction: { - transaction: 'POST /openai/chat/completions', - }, - }) .expect({ - transaction: { - transaction: 'chat gpt-3.5-turbo', - contexts: { - trace: { - span_id: expect.any(String), - trace_id: expect.any(String), - data: { - [GEN_AI_OPERATION_NAME]: 'chat', - [SENTRY_OP]: 'gen_ai.chat', - [SENTRY_ORIGIN]: 'auto.ai.openai', - [GEN_AI_PROVIDER_NAME]: 'openai', - [GEN_AI_REQUEST_MODEL]: 'gpt-3.5-turbo', - [GEN_AI_REQUEST_TEMPERATURE]: 0.7, - [GEN_AI_RESPONSE_MODEL]: 'gpt-3.5-turbo', - [GEN_AI_RESPONSE_ID]: 'chatcmpl-mock123', - [GEN_AI_RESPONSE_FINISH_REASONS]: '["stop"]', - [GEN_AI_USAGE_INPUT_TOKENS]: 10, - [GEN_AI_USAGE_OUTPUT_TOKENS]: 15, - [GEN_AI_USAGE_TOTAL_TOKENS]: 25, - }, - op: 'gen_ai.chat', - origin: 'auto.ai.openai', - status: 'ok', - }, - }, + span: container => { + const serverSegment = container.items.find( + span => span.is_segment && span.name === 'POST /openai/chat/completions', + ); + expect(serverSegment).toBeDefined(); + const segment = container.items.find(span => span.is_segment && span.name === 'chat gpt-3.5-turbo'); + expect(segment?.span_id).toEqual(expect.any(String)); + expect(segment?.trace_id).toEqual(expect.any(String)); + expect(segment?.attributes[GEN_AI_OPERATION_NAME]).toEqual({ value: 'chat', type: 'string' }); + expect(segment?.attributes[SENTRY_OP]).toEqual({ value: 'gen_ai.chat', type: 'string' }); + expect(segment?.attributes[SENTRY_ORIGIN]).toEqual({ value: 'auto.ai.openai', type: 'string' }); + expect(segment?.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ value: 'openai', type: 'string' }); + expect(segment?.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ value: 'gpt-3.5-turbo', type: 'string' }); + expect(segment?.attributes[GEN_AI_REQUEST_TEMPERATURE]).toEqual({ value: 0.7, type: 'double' }); + expect(segment?.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ value: 'gpt-3.5-turbo', type: 'string' }); + expect(segment?.attributes[GEN_AI_RESPONSE_ID]).toEqual({ value: 'chatcmpl-mock123', type: 'string' }); + expect(segment?.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ + value: '["stop"]', + type: 'string', + }); + expect(segment?.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ value: 10, type: 'integer' }); + expect(segment?.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ value: 15, type: 'integer' }); + expect(segment?.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ value: 25, type: 'integer' }); + expect(segment?.status).toEqual('ok'); }, }) .start() @@ -1114,38 +1118,31 @@ describe('OpenAI integration (V6)', () => { (createRunner, test) => { test('it works with Azure OpenAI (v6)', async () => { await createRunner() - // First the span that our mock express server is emitting, unrelated to this test - .expect({ - transaction: { - transaction: 'POST /azureopenai/deployments/:model/chat/completions', - }, - }) .expect({ - transaction: { - transaction: 'chat gpt-3.5-turbo', - contexts: { - trace: { - span_id: expect.any(String), - trace_id: expect.any(String), - data: { - [GEN_AI_OPERATION_NAME]: 'chat', - [SENTRY_OP]: 'gen_ai.chat', - [SENTRY_ORIGIN]: 'auto.ai.openai', - [GEN_AI_PROVIDER_NAME]: 'openai', - [GEN_AI_REQUEST_MODEL]: 'gpt-3.5-turbo', - [GEN_AI_REQUEST_TEMPERATURE]: 0.7, - [GEN_AI_RESPONSE_MODEL]: 'gpt-3.5-turbo', - [GEN_AI_RESPONSE_ID]: 'chatcmpl-mock123', - [GEN_AI_RESPONSE_FINISH_REASONS]: '["stop"]', - [GEN_AI_USAGE_INPUT_TOKENS]: 10, - [GEN_AI_USAGE_OUTPUT_TOKENS]: 15, - [GEN_AI_USAGE_TOTAL_TOKENS]: 25, - }, - op: 'gen_ai.chat', - origin: 'auto.ai.openai', - status: 'ok', - }, - }, + span: container => { + const serverSegment = container.items.find( + span => span.is_segment && span.name === 'POST /azureopenai/deployments/:model/chat/completions', + ); + expect(serverSegment).toBeDefined(); + const segment = container.items.find(span => span.is_segment && span.name === 'chat gpt-3.5-turbo'); + expect(segment?.span_id).toEqual(expect.any(String)); + expect(segment?.trace_id).toEqual(expect.any(String)); + expect(segment?.attributes[GEN_AI_OPERATION_NAME]).toEqual({ value: 'chat', type: 'string' }); + expect(segment?.attributes[SENTRY_OP]).toEqual({ value: 'gen_ai.chat', type: 'string' }); + expect(segment?.attributes[SENTRY_ORIGIN]).toEqual({ value: 'auto.ai.openai', type: 'string' }); + expect(segment?.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ value: 'openai', type: 'string' }); + expect(segment?.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ value: 'gpt-3.5-turbo', type: 'string' }); + expect(segment?.attributes[GEN_AI_REQUEST_TEMPERATURE]).toEqual({ value: 0.7, type: 'double' }); + expect(segment?.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ value: 'gpt-3.5-turbo', type: 'string' }); + expect(segment?.attributes[GEN_AI_RESPONSE_ID]).toEqual({ value: 'chatcmpl-mock123', type: 'string' }); + expect(segment?.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ + value: '["stop"]', + type: 'string', + }); + expect(segment?.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ value: 10, type: 'integer' }); + expect(segment?.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ value: 15, type: 'integer' }); + expect(segment?.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ value: 25, type: 'integer' }); + expect(segment?.status).toEqual('ok'); }, }) .start() diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/v7/instrument-span-streaming.mjs b/dev-packages/node-integration-tests/suites/tracing/openai/v7/instrument-span-streaming.mjs deleted file mode 100644 index 5e0b6fb5592f..000000000000 --- a/dev-packages/node-integration-tests/suites/tracing/openai/v7/instrument-span-streaming.mjs +++ /dev/null @@ -1,11 +0,0 @@ -import * as Sentry from '@sentry/node'; -import { loggingTransport } from '@sentry-internal/node-integration-tests'; - -Sentry.init({ - dsn: 'https://public@dsn.ingest.sentry.io/1337', - release: '1.0', - tracesSampleRate: 1.0, - dataCollection: { genAI: { inputs: false, outputs: false } }, - transport: loggingTransport, - traceLifecycle: 'stream', -}); diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/v7/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/openai/v7/instrument.mjs index 2097d76a4eff..cc192fb89834 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/v7/instrument.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/openai/v7/instrument.mjs @@ -2,16 +2,9 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; Sentry.init({ - traceLifecycle: 'static', dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, dataCollection: { genAI: { inputs: false, outputs: false } }, transport: loggingTransport, - beforeSendTransaction: event => { - if (event.transaction.includes('/openai/')) { - return null; - } - return event; - }, }); diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts b/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts index fbea79d4f34c..a004c50ed844 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts @@ -1,3 +1,4 @@ +import type { SerializedStreamedSpanContainer } from '@sentry/core'; import { afterAll, expect } from 'vitest'; import { GEN_AI_OPERATION_NAME, @@ -32,14 +33,20 @@ conditionalTest({ min: 22 })('OpenAI integration (V7)', () => { 'instrument.mjs', (createRunner, test) => { test('instruments chat completions, the responses API and streaming on openai v7', async () => { + const allSpans: SerializedStreamedSpanContainer['items'] = []; + await createRunner() + .unordered() .ignore('event') - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - expect(container.items).toHaveLength(6); + allSpans.push(...container.items); + const segment = allSpans.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + expect(spans).toHaveLength(6); - const chatCompletionSpan = container.items.find( + const chatCompletionSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123', ); expect(chatCompletionSpan).toBeDefined(); @@ -70,9 +77,7 @@ conditionalTest({ min: 22 })('OpenAI integration (V7)', () => { expect(chatCompletionSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ type: 'integer', value: 25 }); // The responses API is a separate instrumented resource file from chat completions. - const responsesSpan = container.items.find( - span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_mock456', - ); + const responsesSpan = spans.find(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_mock456'); expect(responsesSpan).toBeDefined(); expect(responsesSpan!.name).toBe('chat gpt-3.5-turbo'); expect(responsesSpan!.status).toBe('ok'); @@ -84,7 +89,7 @@ conditionalTest({ min: 22 })('OpenAI integration (V7)', () => { // Streaming goes through the patched async iterator rather than `beforeSpanEnd`, so it // is the part most likely to break if the `Stream` shape changes across a major. - const streamingSpan = container.items.find( + const streamingSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-stream-123', ); expect(streamingSpan).toBeDefined(); @@ -95,7 +100,7 @@ conditionalTest({ min: 22 })('OpenAI integration (V7)', () => { expect(streamingSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ type: 'integer', value: 18 }); expect(streamingSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ type: 'integer', value: 30 }); - const errorSpan = container.items.find(span => span.name === 'chat error-model' && span.status !== 'ok'); + const errorSpan = spans.find(span => span.name === 'chat error-model' && span.status !== 'ok'); expect(errorSpan).toBeDefined(); expect(errorSpan!.attributes[SENTRY_ORIGIN]).toEqual({ type: 'string', @@ -124,10 +129,12 @@ conditionalTest({ min: 22 })('OpenAI integration (V7)', () => { test('instruments the embeddings API on openai v7', async () => { await createRunner() .ignore('event') - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - const embeddingSpans = container.items.filter( + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + const embeddingSpans = spans.filter( span => span.attributes[GEN_AI_OPERATION_NAME]?.value === 'embeddings', ); expect(embeddingSpans).toHaveLength(3); @@ -164,46 +171,4 @@ conditionalTest({ min: 22 })('OpenAI integration (V7)', () => { }, }, ); - // Span streaming is the default trace lifecycle, so cover it too. The span buffer flushes on a 5s - // timer per trace, which splits this scenario's spans across envelopes under CI load, and the - // runner asserts against one envelope at a time — so this only asserts on the first call's span, - // which is always in the first flush. The exhaustive assertions above stay on the static lifecycle, - // where every span arrives in a single envelope. - createEsmAndCjsTests( - __dirname, - 'scenario-chat.mjs', - 'instrument-span-streaming.mjs', - (createRunner, test) => { - test('instruments chat completions on openai v7 with span streaming enabled', async () => { - await createRunner() - .ignore('event') - .expect({ - span: container => { - const chatCompletionSpan = container.items.find( - span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123', - ); - expect(chatCompletionSpan).toBeDefined(); - expect(chatCompletionSpan!.name).toBe('chat gpt-3.5-turbo'); - expect(chatCompletionSpan!.status).toBe('ok'); - expect(chatCompletionSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(chatCompletionSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ type: 'string', value: 'openai' }); - }, - }) - .start() - .completed(); - }); - }, - { - additionalDependencies: { - openai: '7.5.0', - }, - }, - ); }); From 1587f5526b6ec109148f79054b12bb09b0771374 Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Tue, 6 Oct 2026 10:59:57 +0200 Subject: [PATCH 2/8] test(node): Remove added OpenAI static coverage Co-Authored-By: GPT-6 --- .../node-suites/excludes.ts | 1 - .../tracing/openai/static/instrument.mjs | 11 ---- .../suites/tracing/openai/static/test.ts | 63 ------------------- 3 files changed, 75 deletions(-) delete mode 100644 dev-packages/node-integration-tests/suites/tracing/openai/static/instrument.mjs delete mode 100644 dev-packages/node-integration-tests/suites/tracing/openai/static/test.ts diff --git a/dev-packages/bun-integration-tests/node-suites/excludes.ts b/dev-packages/bun-integration-tests/node-suites/excludes.ts index 177cdf83d94f..d27790a79a0f 100644 --- a/dev-packages/bun-integration-tests/node-suites/excludes.ts +++ b/dev-packages/bun-integration-tests/node-suites/excludes.ts @@ -121,7 +121,6 @@ export const NO_AUTO_INSTRUMENTATION = [ 'suites/tracing/mysql2-tracing-channel/test.ts', 'suites/tracing/mysql2/test.ts', 'suites/tracing/openai/test.ts', - 'suites/tracing/openai/static/test.ts', 'suites/tracing/openai/v6/test.ts', 'suites/tracing/openai/v7/test.ts', 'suites/tracing/orchestrion-lazy-registration/test.ts', diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/static/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/openai/static/instrument.mjs deleted file mode 100644 index 42052d281304..000000000000 --- a/dev-packages/node-integration-tests/suites/tracing/openai/static/instrument.mjs +++ /dev/null @@ -1,11 +0,0 @@ -import * as Sentry from '@sentry/node'; -import { loggingTransport } from '@sentry-internal/node-integration-tests'; - -Sentry.init({ - traceLifecycle: 'static', - dsn: 'https://public@dsn.ingest.sentry.io/1337', - release: '1.0', - tracesSampleRate: 1.0, - dataCollection: { genAI: { inputs: false, outputs: false } }, - transport: loggingTransport, -}); diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/static/test.ts b/dev-packages/node-integration-tests/suites/tracing/openai/static/test.ts deleted file mode 100644 index b25100db6add..000000000000 --- a/dev-packages/node-integration-tests/suites/tracing/openai/static/test.ts +++ /dev/null @@ -1,63 +0,0 @@ -import { afterAll, describe, expect } from 'vitest'; -import { - GEN_AI_OPERATION_NAME, - GEN_AI_PROVIDER_NAME, - GEN_AI_REQUEST_MODEL, - GEN_AI_REQUEST_TEMPERATURE, - GEN_AI_RESPONSE_FINISH_REASONS, - GEN_AI_RESPONSE_ID, - GEN_AI_RESPONSE_MODEL, - GEN_AI_USAGE_INPUT_TOKENS, - GEN_AI_USAGE_OUTPUT_TOKENS, - GEN_AI_USAGE_TOTAL_TOKENS, - SENTRY_OP, - SENTRY_ORIGIN, -} from '@sentry/conventions/attributes'; -import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/runner'; - -describe('OpenAI integration > static lifecycle', () => { - afterAll(() => { - cleanupChildProcesses(); - }); - createEsmAndCjsTests(__dirname, '../scenario-root-span.mjs', 'instrument.mjs', (createRunner, test) => { - test('it works without a wrapping span', async () => { - await createRunner() - // First the span that our mock express server is emitting, unrelated to this test - .expect({ - transaction: { - transaction: 'POST /openai/chat/completions', - }, - }) - .expect({ - transaction: { - transaction: 'chat gpt-3.5-turbo', - contexts: { - trace: { - span_id: expect.any(String), - trace_id: expect.any(String), - data: { - [GEN_AI_OPERATION_NAME]: 'chat', - [SENTRY_OP]: 'gen_ai.chat', - [SENTRY_ORIGIN]: 'auto.ai.openai', - [GEN_AI_PROVIDER_NAME]: 'openai', - [GEN_AI_REQUEST_MODEL]: 'gpt-3.5-turbo', - [GEN_AI_REQUEST_TEMPERATURE]: 0.7, - [GEN_AI_RESPONSE_MODEL]: 'gpt-3.5-turbo', - [GEN_AI_RESPONSE_ID]: 'chatcmpl-mock123', - [GEN_AI_RESPONSE_FINISH_REASONS]: '["stop"]', - [GEN_AI_USAGE_INPUT_TOKENS]: 10, - [GEN_AI_USAGE_OUTPUT_TOKENS]: 15, - [GEN_AI_USAGE_TOTAL_TOKENS]: 25, - }, - op: 'gen_ai.chat', - origin: 'auto.ai.openai', - status: 'ok', - }, - }, - }, - }) - .start() - .completed(); - }); - }); -}); From bf58d5541603ef0da270193f7002061cb87c7ebb Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Tue, 6 Oct 2026 11:24:50 +0200 Subject: [PATCH 3/8] test(node): Remove redundant openai assertions Co-Authored-By: GPT-6 --- .../tracing/openai/openai-tool-calls/test.ts | 8 ---- .../suites/tracing/openai/test.ts | 44 ------------------- .../suites/tracing/openai/v6/test.ts | 27 ------------ .../suites/tracing/openai/v7/test.ts | 5 --- 4 files changed, 84 deletions(-) diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/openai-tool-calls/test.ts b/dev-packages/node-integration-tests/suites/tracing/openai/openai-tool-calls/test.ts index f5ca83f3b3f8..59473fcea8fe 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/openai-tool-calls/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/openai/openai-tool-calls/test.ts @@ -88,7 +88,6 @@ describe('OpenAI Tool Calls integration', () => { const chatToolsSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-tools-123', ); - expect(chatToolsSpan).toBeDefined(); expect(chatToolsSpan!.name).toBe('chat gpt-4'); expect(chatToolsSpan!.status).toBe('ok'); expect(chatToolsSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ @@ -138,7 +137,6 @@ describe('OpenAI Tool Calls integration', () => { const streamingChatToolsSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-stream-tools-123', ); - expect(streamingChatToolsSpan).toBeDefined(); expect(streamingChatToolsSpan!.name).toBe('chat gpt-4'); expect(streamingChatToolsSpan!.status).toBe('ok'); expect(streamingChatToolsSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ @@ -198,7 +196,6 @@ describe('OpenAI Tool Calls integration', () => { const responsesToolsSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_tools_789', ); - expect(responsesToolsSpan).toBeDefined(); expect(responsesToolsSpan!.name).toBe('chat gpt-4'); expect(responsesToolsSpan!.status).toBe('ok'); expect(responsesToolsSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ @@ -251,7 +248,6 @@ describe('OpenAI Tool Calls integration', () => { const streamingResponsesToolsSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_stream_tools_789', ); - expect(streamingResponsesToolsSpan).toBeDefined(); expect(streamingResponsesToolsSpan!.name).toBe('chat gpt-4'); expect(streamingResponsesToolsSpan!.status).toBe('ok'); expect(streamingResponsesToolsSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ @@ -326,7 +322,6 @@ describe('OpenAI Tool Calls integration', () => { const chatToolsSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-tools-123', ); - expect(chatToolsSpan).toBeDefined(); expect(chatToolsSpan!.name).toBe('chat gpt-4'); expect(chatToolsSpan!.status).toBe('ok'); expect(chatToolsSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ @@ -390,7 +385,6 @@ describe('OpenAI Tool Calls integration', () => { const streamingChatToolsSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-stream-tools-123', ); - expect(streamingChatToolsSpan).toBeDefined(); expect(streamingChatToolsSpan!.name).toBe('chat gpt-4'); expect(streamingChatToolsSpan!.status).toBe('ok'); expect(streamingChatToolsSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ @@ -461,7 +455,6 @@ describe('OpenAI Tool Calls integration', () => { const responsesToolsSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_tools_789', ); - expect(responsesToolsSpan).toBeDefined(); expect(responsesToolsSpan!.name).toBe('chat gpt-4'); expect(responsesToolsSpan!.status).toBe('ok'); expect(responsesToolsSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ @@ -525,7 +518,6 @@ describe('OpenAI Tool Calls integration', () => { const streamingResponsesToolsSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_stream_tools_789', ); - expect(streamingResponsesToolsSpan).toBeDefined(); expect(streamingResponsesToolsSpan!.name).toBe('chat gpt-4'); expect(streamingResponsesToolsSpan!.status).toBe('ok'); expect(streamingResponsesToolsSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/test.ts b/dev-packages/node-integration-tests/suites/tracing/openai/test.ts index bb10be74099c..31c013f50e5c 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/openai/test.ts @@ -50,7 +50,6 @@ describe('OpenAI integration', () => { span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123' && span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-3.5-turbo', ); - expect(chatCompletionSpan).toBeDefined(); expect(chatCompletionSpan!.name).toBe('chat gpt-3.5-turbo'); expect(chatCompletionSpan!.status).toBe('ok'); expect(chatCompletionSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ @@ -107,7 +106,6 @@ describe('OpenAI integration', () => { span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_mock456' && span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-3.5-turbo', ); - expect(responsesSpan).toBeDefined(); expect(responsesSpan!.name).toBe('chat gpt-3.5-turbo'); expect(responsesSpan!.status).toBe('ok'); expect(responsesSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ @@ -156,8 +154,6 @@ describe('OpenAI integration', () => { span => span.name === 'chat error-model' && span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE] === undefined, ); - expect(nonStreamingErrorSpan).toBeDefined(); - expect(nonStreamingErrorSpan!.name).toBe('chat error-model'); expect(nonStreamingErrorSpan!.status).toBe('error'); expect(nonStreamingErrorSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toBeUndefined(); expect(nonStreamingErrorSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ @@ -184,7 +180,6 @@ describe('OpenAI integration', () => { const streamingChatCompletionSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-stream-123', ); - expect(streamingChatCompletionSpan).toBeDefined(); expect(streamingChatCompletionSpan!.name).toBe('chat gpt-4'); expect(streamingChatCompletionSpan!.status).toBe('ok'); expect(streamingChatCompletionSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ @@ -247,7 +242,6 @@ describe('OpenAI integration', () => { const streamingResponsesSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_stream_456', ); - expect(streamingResponsesSpan).toBeDefined(); expect(streamingResponsesSpan!.name).toBe('chat gpt-4'); expect(streamingResponsesSpan!.status).toBe('ok'); expect(streamingResponsesSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ @@ -307,8 +301,6 @@ describe('OpenAI integration', () => { span => span.name === 'chat error-model' && span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]?.value === true, ); - expect(streamingErrorSpan).toBeDefined(); - expect(streamingErrorSpan!.name).toBe('chat error-model'); expect(streamingErrorSpan!.status).toBe('error'); expect(streamingErrorSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ type: 'string', @@ -340,7 +332,6 @@ describe('OpenAI integration', () => { span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123' && span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-4o', ); - expect(parsedChatCompletionSpan).toBeDefined(); expect(parsedChatCompletionSpan!.name).toBe('chat gpt-4o'); expect(parsedChatCompletionSpan!.status).toBe('ok'); expect(parsedChatCompletionSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ @@ -393,7 +384,6 @@ describe('OpenAI integration', () => { span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_mock456' && span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-4o', ); - expect(parsedResponsesSpan).toBeDefined(); expect(parsedResponsesSpan!.name).toBe('chat gpt-4o'); expect(parsedResponsesSpan!.status).toBe('ok'); expect(parsedResponsesSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ @@ -472,7 +462,6 @@ describe('OpenAI integration', () => { span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123' && span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-3.5-turbo', ); - expect(chatCompletionSpan).toBeDefined(); expect(chatCompletionSpan!.name).toBe('chat gpt-3.5-turbo'); expect(chatCompletionSpan!.status).toBe('ok'); expect(chatCompletionSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ @@ -541,7 +530,6 @@ describe('OpenAI integration', () => { span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_mock456' && span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-3.5-turbo', ); - expect(responsesSpan).toBeDefined(); expect(responsesSpan!.name).toBe('chat gpt-3.5-turbo'); expect(responsesSpan!.status).toBe('ok'); expect(responsesSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ @@ -598,8 +586,6 @@ describe('OpenAI integration', () => { span => span.name === 'chat error-model' && span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE] === undefined, ); - expect(nonStreamingErrorSpan).toBeDefined(); - expect(nonStreamingErrorSpan!.name).toBe('chat error-model'); expect(nonStreamingErrorSpan!.status).toBe('error'); expect(nonStreamingErrorSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toBeUndefined(); expect(nonStreamingErrorSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ @@ -630,7 +616,6 @@ describe('OpenAI integration', () => { const streamingChatCompletionSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-stream-123', ); - expect(streamingChatCompletionSpan).toBeDefined(); expect(streamingChatCompletionSpan!.name).toBe('chat gpt-4'); expect(streamingChatCompletionSpan!.status).toBe('ok'); expect(streamingChatCompletionSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ @@ -705,7 +690,6 @@ describe('OpenAI integration', () => { const streamingResponsesSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_stream_456', ); - expect(streamingResponsesSpan).toBeDefined(); expect(streamingResponsesSpan!.name).toBe('chat gpt-4'); expect(streamingResponsesSpan!.status).toBe('ok'); expect(streamingResponsesSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ @@ -773,8 +757,6 @@ describe('OpenAI integration', () => { span => span.name === 'chat error-model' && span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]?.value === true, ); - expect(streamingErrorSpan).toBeDefined(); - expect(streamingErrorSpan!.name).toBe('chat error-model'); expect(streamingErrorSpan!.status).toBe('error'); expect(streamingErrorSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ type: 'string', @@ -810,7 +792,6 @@ describe('OpenAI integration', () => { span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123' && span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-4o', ); - expect(parsedChatCompletionSpan).toBeDefined(); expect(parsedChatCompletionSpan!.name).toBe('chat gpt-4o'); expect(parsedChatCompletionSpan!.status).toBe('ok'); expect(parsedChatCompletionSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ @@ -872,7 +853,6 @@ describe('OpenAI integration', () => { span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_mock456' && span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-4o', ); - expect(parsedResponsesSpan).toBeDefined(); expect(parsedResponsesSpan!.name).toBe('chat gpt-4o'); expect(parsedResponsesSpan!.status).toBe('ok'); expect(parsedResponsesSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ @@ -949,7 +929,6 @@ describe('OpenAI integration', () => { span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123' && span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-3.5-turbo', ); - expect(chatCompletionSpan).toBeDefined(); expect(chatCompletionSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toBeUndefined(); expect(chatCompletionSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toMatchObject({ type: 'string', @@ -963,7 +942,6 @@ describe('OpenAI integration', () => { const streamingChatCompletionSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-stream-123', ); - expect(streamingChatCompletionSpan).toBeDefined(); expect(streamingChatCompletionSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ type: 'boolean', value: true, @@ -982,7 +960,6 @@ describe('OpenAI integration', () => { span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123' && span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-4o', ); - expect(parsedChatCompletionSpan).toBeDefined(); expect(parsedChatCompletionSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toBeUndefined(); expect(parsedChatCompletionSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toMatchObject({ type: 'string', @@ -998,7 +975,6 @@ describe('OpenAI integration', () => { span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_mock456' && span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-4o', ); - expect(parsedResponsesSpan).toBeDefined(); expect(parsedResponsesSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toBeUndefined(); expect(parsedResponsesSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toMatchObject({ type: 'string', @@ -1029,8 +1005,6 @@ describe('OpenAI integration', () => { span.name === 'embeddings text-embedding-3-small' && span.attributes[GEN_AI_REQUEST_DIMENSIONS_ATTRIBUTE] !== undefined, ); - expect(singleEmbeddingSpan).toBeDefined(); - expect(singleEmbeddingSpan!.name).toBe('embeddings text-embedding-3-small'); expect(singleEmbeddingSpan!.status).toBe('ok'); expect(singleEmbeddingSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ type: 'string', @@ -1074,8 +1048,6 @@ describe('OpenAI integration', () => { }); const errorEmbeddingSpan = spans.find(span => span.name === 'embeddings error-model'); - expect(errorEmbeddingSpan).toBeDefined(); - expect(errorEmbeddingSpan!.name).toBe('embeddings error-model'); expect(errorEmbeddingSpan!.status).toBe('error'); expect(errorEmbeddingSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ type: 'string', @@ -1103,8 +1075,6 @@ describe('OpenAI integration', () => { span.name === 'embeddings text-embedding-3-small' && span.attributes[GEN_AI_REQUEST_DIMENSIONS_ATTRIBUTE] === undefined, ); - expect(multiEmbeddingSpan).toBeDefined(); - expect(multiEmbeddingSpan!.name).toBe('embeddings text-embedding-3-small'); expect(multiEmbeddingSpan!.status).toBe('ok'); expect(multiEmbeddingSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ type: 'string', @@ -1150,8 +1120,6 @@ describe('OpenAI integration', () => { } const rawEmbeddingSpan = spans.find(span => span.name === 'embeddings text-embedding-3-large'); - expect(rawEmbeddingSpan).toBeDefined(); - expect(rawEmbeddingSpan!.name).toBe('embeddings text-embedding-3-large'); expect(rawEmbeddingSpan!.status).toBe('ok'); expect(rawEmbeddingSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ type: 'string', @@ -1195,7 +1163,6 @@ describe('OpenAI integration', () => { const singleEmbeddingSpan = spans.find( span => span.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value === 'Embedding test!', ); - expect(singleEmbeddingSpan).toBeDefined(); expect(singleEmbeddingSpan!.name).toBe('embeddings text-embedding-3-small'); expect(singleEmbeddingSpan!.status).toBe('ok'); expect(singleEmbeddingSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ @@ -1246,7 +1213,6 @@ describe('OpenAI integration', () => { const errorEmbeddingSpan = spans.find( span => span.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value === 'Error embedding test!', ); - expect(errorEmbeddingSpan).toBeDefined(); expect(errorEmbeddingSpan!.name).toBe('embeddings error-model'); expect(errorEmbeddingSpan!.status).toBe('error'); expect(errorEmbeddingSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ @@ -1279,7 +1245,6 @@ describe('OpenAI integration', () => { span.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value === '["First input text","Second input text","Third input text"]', ); - expect(multiEmbeddingSpan).toBeDefined(); expect(multiEmbeddingSpan!.name).toBe('embeddings text-embedding-3-small'); expect(multiEmbeddingSpan!.status).toBe('ok'); expect(multiEmbeddingSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ @@ -1320,8 +1285,6 @@ describe('OpenAI integration', () => { }); const rawEmbeddingSpan = spans.find(span => span.name === 'embeddings text-embedding-3-large'); - expect(rawEmbeddingSpan).toBeDefined(); - expect(rawEmbeddingSpan!.name).toBe('embeddings text-embedding-3-large'); expect(rawEmbeddingSpan!.status).toBe('ok'); expect(rawEmbeddingSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ type: 'string', @@ -1432,8 +1395,6 @@ describe('OpenAI integration', () => { const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); expect(spans).toHaveLength(5); const conversationCreateSpan = spans.find(span => span.name === 'chat'); - expect(conversationCreateSpan).toBeDefined(); - expect(conversationCreateSpan!.name).toBe('chat'); expect(conversationCreateSpan!.status).toBe('ok'); expect(conversationCreateSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ type: 'string', @@ -1462,7 +1423,6 @@ describe('OpenAI integration', () => { 'conv_689667905b048191b4740501625afd940c7533ace33a2dab' && span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-4', ); - expect(conversationResponseSpan).toBeDefined(); expect(conversationResponseSpan!.status).toBe('ok'); expect(conversationResponseSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ type: 'string', @@ -1494,7 +1454,6 @@ describe('OpenAI integration', () => { span.attributes[SENTRY_OP]?.value === 'gen_ai.chat' && span.attributes[GEN_AI_CONVERSATION_ID] === undefined, ); - expect(unlinkedResponseSpan).toBeDefined(); expect(unlinkedResponseSpan!.status).toBe('ok'); expect(unlinkedResponseSpan!.attributes[SENTRY_OP]).toEqual({ type: 'string', @@ -1505,7 +1464,6 @@ describe('OpenAI integration', () => { const previousResponseSpan = spans.find( span => span.attributes[GEN_AI_CONVERSATION_ID]?.value === 'resp_mock_conv_123', ); - expect(previousResponseSpan).toBeDefined(); expect(previousResponseSpan!.status).toBe('ok'); expect(previousResponseSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ type: 'string', @@ -1535,8 +1493,6 @@ describe('OpenAI integration', () => { const rawConversationSpan = spans.find( span => span.name === 'chat' && span.attributes[GEN_AI_CONVERSATION_ID] === undefined, ); - expect(rawConversationSpan).toBeDefined(); - expect(rawConversationSpan!.name).toBe('chat'); expect(rawConversationSpan!.status).toBe('ok'); expect(rawConversationSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ type: 'string', diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/v6/test.ts b/dev-packages/node-integration-tests/suites/tracing/openai/v6/test.ts index 56cf087de0e9..66edeee9a282 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/v6/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/openai/v6/test.ts @@ -51,7 +51,6 @@ describe('OpenAI integration (V6)', () => { const chatCompletionSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123', ); - expect(chatCompletionSpan).toBeDefined(); expect(chatCompletionSpan!.name).toBe('chat gpt-3.5-turbo'); expect(chatCompletionSpan!.status).toBe('ok'); expect(chatCompletionSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ @@ -104,7 +103,6 @@ describe('OpenAI integration (V6)', () => { }); const responsesSpan = spans.find(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_mock456'); - expect(responsesSpan).toBeDefined(); expect(responsesSpan!.name).toBe('chat gpt-3.5-turbo'); expect(responsesSpan!.status).toBe('ok'); expect(responsesSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ @@ -153,8 +151,6 @@ describe('OpenAI integration (V6)', () => { span => span.name === 'chat error-model' && span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE] === undefined, ); - expect(nonStreamingErrorSpan).toBeDefined(); - expect(nonStreamingErrorSpan!.name).toBe('chat error-model'); expect(nonStreamingErrorSpan!.status).toBe('error'); expect(nonStreamingErrorSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toBeUndefined(); expect(nonStreamingErrorSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ @@ -181,7 +177,6 @@ describe('OpenAI integration (V6)', () => { const streamingChatCompletionSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-stream-123', ); - expect(streamingChatCompletionSpan).toBeDefined(); expect(streamingChatCompletionSpan!.name).toBe('chat gpt-4'); expect(streamingChatCompletionSpan!.status).toBe('ok'); expect(streamingChatCompletionSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ @@ -244,7 +239,6 @@ describe('OpenAI integration (V6)', () => { const streamingResponsesSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_stream_456', ); - expect(streamingResponsesSpan).toBeDefined(); expect(streamingResponsesSpan!.name).toBe('chat gpt-4'); expect(streamingResponsesSpan!.status).toBe('ok'); expect(streamingResponsesSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ @@ -304,8 +298,6 @@ describe('OpenAI integration (V6)', () => { span => span.name === 'chat error-model' && span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]?.value === true, ); - expect(streamingErrorSpan).toBeDefined(); - expect(streamingErrorSpan!.name).toBe('chat error-model'); expect(streamingErrorSpan!.status).toBe('error'); expect(streamingErrorSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ type: 'string', @@ -364,7 +356,6 @@ describe('OpenAI integration (V6)', () => { const chatCompletionSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123', ); - expect(chatCompletionSpan).toBeDefined(); expect(chatCompletionSpan!.name).toBe('chat gpt-3.5-turbo'); expect(chatCompletionSpan!.status).toBe('ok'); expect(chatCompletionSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ @@ -429,7 +420,6 @@ describe('OpenAI integration (V6)', () => { }); const responsesSpan = spans.find(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_mock456'); - expect(responsesSpan).toBeDefined(); expect(responsesSpan!.name).toBe('chat gpt-3.5-turbo'); expect(responsesSpan!.status).toBe('ok'); expect(responsesSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ @@ -486,8 +476,6 @@ describe('OpenAI integration (V6)', () => { span => span.name === 'chat error-model' && span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE] === undefined, ); - expect(nonStreamingErrorSpan).toBeDefined(); - expect(nonStreamingErrorSpan!.name).toBe('chat error-model'); expect(nonStreamingErrorSpan!.status).toBe('error'); expect(nonStreamingErrorSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toBeUndefined(); expect(nonStreamingErrorSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ @@ -518,7 +506,6 @@ describe('OpenAI integration (V6)', () => { const streamingChatCompletionSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-stream-123', ); - expect(streamingChatCompletionSpan).toBeDefined(); expect(streamingChatCompletionSpan!.name).toBe('chat gpt-4'); expect(streamingChatCompletionSpan!.status).toBe('ok'); expect(streamingChatCompletionSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ @@ -593,7 +580,6 @@ describe('OpenAI integration (V6)', () => { const streamingResponsesSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_stream_456', ); - expect(streamingResponsesSpan).toBeDefined(); expect(streamingResponsesSpan!.name).toBe('chat gpt-4'); expect(streamingResponsesSpan!.status).toBe('ok'); expect(streamingResponsesSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ @@ -661,8 +647,6 @@ describe('OpenAI integration (V6)', () => { span => span.name === 'chat error-model' && span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]?.value === true, ); - expect(streamingErrorSpan).toBeDefined(); - expect(streamingErrorSpan!.name).toBe('chat error-model'); expect(streamingErrorSpan!.status).toBe('error'); expect(streamingErrorSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ type: 'string', @@ -725,7 +709,6 @@ describe('OpenAI integration (V6)', () => { const chatCompletionSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123', ); - expect(chatCompletionSpan).toBeDefined(); expect(chatCompletionSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toBeUndefined(); expect(chatCompletionSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toMatchObject({ type: 'string', @@ -743,7 +726,6 @@ describe('OpenAI integration (V6)', () => { const streamingChatCompletionSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-stream-123', ); - expect(streamingChatCompletionSpan).toBeDefined(); expect(streamingChatCompletionSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ type: 'boolean', value: true, @@ -791,8 +773,6 @@ describe('OpenAI integration (V6)', () => { span.name === 'embeddings text-embedding-3-small' && span.attributes[GEN_AI_REQUEST_DIMENSIONS_ATTRIBUTE] !== undefined, ); - expect(singleEmbeddingSpan).toBeDefined(); - expect(singleEmbeddingSpan!.name).toBe('embeddings text-embedding-3-small'); expect(singleEmbeddingSpan!.status).toBe('ok'); expect(singleEmbeddingSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ type: 'string', @@ -836,8 +816,6 @@ describe('OpenAI integration (V6)', () => { }); const errorEmbeddingSpan = spans.find(span => span.name === 'embeddings error-model'); - expect(errorEmbeddingSpan).toBeDefined(); - expect(errorEmbeddingSpan!.name).toBe('embeddings error-model'); expect(errorEmbeddingSpan!.status).toBe('error'); expect(errorEmbeddingSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ type: 'string', @@ -865,8 +843,6 @@ describe('OpenAI integration (V6)', () => { span.name === 'embeddings text-embedding-3-small' && span.attributes[GEN_AI_REQUEST_DIMENSIONS_ATTRIBUTE] === undefined, ); - expect(multiEmbeddingSpan).toBeDefined(); - expect(multiEmbeddingSpan!.name).toBe('embeddings text-embedding-3-small'); expect(multiEmbeddingSpan!.status).toBe('ok'); expect(multiEmbeddingSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ type: 'string', @@ -929,7 +905,6 @@ describe('OpenAI integration (V6)', () => { const singleEmbeddingSpan = spans.find( span => span.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value === 'Embedding test!', ); - expect(singleEmbeddingSpan).toBeDefined(); expect(singleEmbeddingSpan!.name).toBe('embeddings text-embedding-3-small'); expect(singleEmbeddingSpan!.status).toBe('ok'); expect(singleEmbeddingSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ @@ -980,7 +955,6 @@ describe('OpenAI integration (V6)', () => { const errorEmbeddingSpan = spans.find( span => span.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value === 'Error embedding test!', ); - expect(errorEmbeddingSpan).toBeDefined(); expect(errorEmbeddingSpan!.name).toBe('embeddings error-model'); expect(errorEmbeddingSpan!.status).toBe('error'); expect(errorEmbeddingSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ @@ -1013,7 +987,6 @@ describe('OpenAI integration (V6)', () => { span.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value === '["First input text","Second input text","Third input text"]', ); - expect(multiEmbeddingSpan).toBeDefined(); expect(multiEmbeddingSpan!.name).toBe('embeddings text-embedding-3-small'); expect(multiEmbeddingSpan!.status).toBe('ok'); expect(multiEmbeddingSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts b/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts index a004c50ed844..d4b62a5d5f5f 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts @@ -49,7 +49,6 @@ conditionalTest({ min: 22 })('OpenAI integration (V7)', () => { const chatCompletionSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123', ); - expect(chatCompletionSpan).toBeDefined(); expect(chatCompletionSpan!.name).toBe('chat gpt-3.5-turbo'); expect(chatCompletionSpan!.status).toBe('ok'); expect(chatCompletionSpan!.attributes[SENTRY_OP]).toEqual({ @@ -78,7 +77,6 @@ conditionalTest({ min: 22 })('OpenAI integration (V7)', () => { // The responses API is a separate instrumented resource file from chat completions. const responsesSpan = spans.find(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_mock456'); - expect(responsesSpan).toBeDefined(); expect(responsesSpan!.name).toBe('chat gpt-3.5-turbo'); expect(responsesSpan!.status).toBe('ok'); expect(responsesSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ type: 'string', value: 'chat' }); @@ -92,7 +90,6 @@ conditionalTest({ min: 22 })('OpenAI integration (V7)', () => { const streamingSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-stream-123', ); - expect(streamingSpan).toBeDefined(); expect(streamingSpan!.name).toBe('chat gpt-4'); expect(streamingSpan!.status).toBe('ok'); expect(streamingSpan!.attributes[GEN_AI_RESPONSE_STREAMING]).toEqual({ type: 'boolean', value: true }); @@ -101,7 +98,6 @@ conditionalTest({ min: 22 })('OpenAI integration (V7)', () => { expect(streamingSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ type: 'integer', value: 30 }); const errorSpan = spans.find(span => span.name === 'chat error-model' && span.status !== 'ok'); - expect(errorSpan).toBeDefined(); expect(errorSpan!.attributes[SENTRY_ORIGIN]).toEqual({ type: 'string', value: 'auto.ai.openai', @@ -142,7 +138,6 @@ conditionalTest({ min: 22 })('OpenAI integration (V7)', () => { const singleEmbeddingSpan = embeddingSpans.find( span => span.name === 'embeddings text-embedding-3-small' && span.status === 'ok', ); - expect(singleEmbeddingSpan).toBeDefined(); expect(singleEmbeddingSpan!.attributes[SENTRY_OP]).toEqual({ type: 'string', value: 'gen_ai.embeddings', From 297df863816d08b630461b2089c715bd4bd05022 Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Tue, 6 Oct 2026 11:51:53 +0200 Subject: [PATCH 4/8] test(node): Collect openai spans across envelopes Co-Authored-By: GPT-6 --- .../suites/tracing/openai/test.ts | 82 ++++++++++++++----- .../suites/tracing/openai/v6/test.ts | 32 ++++++-- .../suites/tracing/openai/v7/test.ts | 8 +- 3 files changed, 90 insertions(+), 32 deletions(-) diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/test.ts b/dev-packages/node-integration-tests/suites/tracing/openai/test.ts index 31c013f50e5c..0d9430741532 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/openai/test.ts @@ -993,12 +993,16 @@ describe('OpenAI integration', () => { createEsmAndCjsTests(__dirname, 'scenario-embeddings.mjs', 'instrument.mjs', (createRunner, test) => { test('creates openai related spans with genAI recording disabled', async () => { + const allSpans: SerializedStreamedSpanContainer['items'] = []; + await createRunner() + .unordered() .expect({ span: container => { - const segment = container.items.find(span => span.is_segment && span.name === 'main'); + allSpans.push(...container.items); + const segment = allSpans.find(span => span.is_segment && span.name === 'main'); expect(segment).toBeDefined(); - const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); expect(spans).toHaveLength(4); const singleEmbeddingSpan = spans.find( span => @@ -1153,12 +1157,16 @@ describe('OpenAI integration', () => { createEsmAndCjsTests(__dirname, 'scenario-embeddings.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { test('creates openai related spans with genAI recording enabled', async () => { + const allSpans: SerializedStreamedSpanContainer['items'] = []; + await createRunner() + .unordered() .expect({ span: container => { - const segment = container.items.find(span => span.is_segment && span.name === 'main'); + allSpans.push(...container.items); + const segment = allSpans.find(span => span.is_segment && span.name === 'main'); expect(segment).toBeDefined(); - const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); expect(spans).toHaveLength(4); const singleEmbeddingSpan = spans.find( span => span.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value === 'Embedding test!', @@ -1322,14 +1330,18 @@ describe('OpenAI integration', () => { createEsmAndCjsTests(__dirname, 'scenario-root-span.mjs', 'instrument.mjs', (createRunner, test) => { test('it works without a wrapping span', async () => { + const allSpans: SerializedStreamedSpanContainer['items'] = []; + await createRunner() + .unordered() .expect({ span: container => { - const serverSegment = container.items.find( + allSpans.push(...container.items); + const serverSegment = allSpans.find( span => span.is_segment && span.name === 'POST /openai/chat/completions', ); expect(serverSegment).toBeDefined(); - const segment = container.items.find(span => span.is_segment && span.name === 'chat gpt-3.5-turbo'); + const segment = allSpans.find(span => span.is_segment && span.name === 'chat gpt-3.5-turbo'); expect(segment?.span_id).toEqual(expect.any(String)); expect(segment?.trace_id).toEqual(expect.any(String)); expect(segment?.attributes[GEN_AI_OPERATION_NAME]).toEqual({ value: 'chat', type: 'string' }); @@ -1354,14 +1366,18 @@ describe('OpenAI integration', () => { createEsmAndCjsTests(__dirname, 'scenario-azure-openai.mjs', 'instrument.mjs', (createRunner, test) => { test('it works with Azure OpenAI', async () => { + const allSpans: SerializedStreamedSpanContainer['items'] = []; + await createRunner() + .unordered() .expect({ span: container => { - const serverSegment = container.items.find( + allSpans.push(...container.items); + const serverSegment = allSpans.find( span => span.is_segment && span.name === 'POST /azureopenai/deployments/:model/chat/completions', ); expect(serverSegment).toBeDefined(); - const segment = container.items.find(span => span.is_segment && span.name === 'chat gpt-3.5-turbo'); + const segment = allSpans.find(span => span.is_segment && span.name === 'chat gpt-3.5-turbo'); expect(segment?.span_id).toEqual(expect.any(String)); expect(segment?.trace_id).toEqual(expect.any(String)); expect(segment?.attributes[GEN_AI_OPERATION_NAME]).toEqual({ value: 'chat', type: 'string' }); @@ -1387,12 +1403,16 @@ describe('OpenAI integration', () => { // Test for conversation ID support (Conversations API and previous_response_id) createEsmAndCjsTests(__dirname, 'scenario-conversation.mjs', 'instrument.mjs', (createRunner, test) => { test('captures conversation ID from Conversations API and previous_response_id', async () => { + const allSpans: SerializedStreamedSpanContainer['items'] = []; + await createRunner() + .unordered() .expect({ span: container => { - const segment = container.items.find(span => span.is_segment && span.name === 'conversation-test'); + allSpans.push(...container.items); + const segment = allSpans.find(span => span.is_segment && span.name === 'conversation-test'); expect(segment).toBeDefined(); - const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); expect(spans).toHaveLength(5); const conversationCreateSpan = spans.find(span => span.name === 'chat'); expect(conversationCreateSpan!.status).toBe('ok'); @@ -1521,14 +1541,16 @@ describe('OpenAI integration', () => { // Test for manual conversation ID setting using setConversationId() createEsmAndCjsTests(__dirname, 'scenario-manual-conversation-id.mjs', 'instrument.mjs', (createRunner, test) => { test('attaches manual conversation ID set via setConversationId() to all chat spans', async () => { + const allSpans: SerializedStreamedSpanContainer['items'] = []; + await createRunner() + .unordered() .expect({ span: container => { - const segment = container.items.find( - span => span.is_segment && span.name === 'chat-with-manual-conversation-id', - ); + allSpans.push(...container.items); + const segment = allSpans.find(span => span.is_segment && span.name === 'chat-with-manual-conversation-id'); expect(segment).toBeDefined(); - const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); expect(spans).toHaveLength(3); // All three chat completion spans should have the same manually-set conversation ID @@ -1553,12 +1575,16 @@ describe('OpenAI integration', () => { createEsmAndCjsTests(__dirname, 'scenario-separate-scope-1.mjs', 'instrument.mjs', (createRunner, test) => { test('isolates conversation IDs across separate scopes - conversation 1', async () => { + const allSpans: SerializedStreamedSpanContainer['items'] = []; + await createRunner() + .unordered() .expect({ span: container => { - const segment = container.items.find(span => span.is_segment && span.name === 'GET /chat/conversation-1'); + allSpans.push(...container.items); + const segment = allSpans.find(span => span.is_segment && span.name === 'GET /chat/conversation-1'); expect(segment).toBeDefined(); - const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); expect(spans).toHaveLength(2); // Both chat completion spans should have the expected conversation ID @@ -1582,12 +1608,16 @@ describe('OpenAI integration', () => { createEsmAndCjsTests(__dirname, 'scenario-separate-scope-2.mjs', 'instrument.mjs', (createRunner, test) => { test('isolates conversation IDs across separate scopes - conversation 2', async () => { + const allSpans: SerializedStreamedSpanContainer['items'] = []; + await createRunner() + .unordered() .expect({ span: container => { - const segment = container.items.find(span => span.is_segment && span.name === 'GET /chat/conversation-2'); + allSpans.push(...container.items); + const segment = allSpans.find(span => span.is_segment && span.name === 'GET /chat/conversation-2'); expect(segment).toBeDefined(); - const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); expect(spans).toHaveLength(2); // Both chat completion spans should have the expected conversation ID @@ -1615,12 +1645,16 @@ describe('OpenAI integration', () => { 'instrument-with-pii.mjs', (createRunner, test) => { test('extracts system instructions from messages', async () => { + const allSpans: SerializedStreamedSpanContainer['items'] = []; + await createRunner() + .unordered() .expect({ span: container => { - const segment = container.items.find(span => span.is_segment && span.name === 'main'); + allSpans.push(...container.items); + const segment = allSpans.find(span => span.is_segment && span.name === 'main'); expect(segment).toBeDefined(); - const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); expect(spans).toHaveLength(1); const [firstSpan] = spans; @@ -1639,12 +1673,16 @@ describe('OpenAI integration', () => { createEsmAndCjsTests(__dirname, 'scenario-with-response.mjs', 'instrument.mjs', (createRunner, test) => { test('preserves .withResponse() method and works correctly', async () => { + const allSpans: SerializedStreamedSpanContainer['items'] = []; + await createRunner() + .unordered() .expect({ span: container => { - const segment = container.items.find(span => span.is_segment && span.name === 'main'); + allSpans.push(...container.items); + const segment = allSpans.find(span => span.is_segment && span.name === 'main'); expect(segment).toBeDefined(); - const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); expect(spans).toHaveLength(2); for (const span of spans) { diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/v6/test.ts b/dev-packages/node-integration-tests/suites/tracing/openai/v6/test.ts index 66edeee9a282..7ffbdb5aae86 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/v6/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/openai/v6/test.ts @@ -761,12 +761,16 @@ describe('OpenAI integration (V6)', () => { 'instrument.mjs', (createRunner, test) => { test('creates openai related spans with genAI recording disabled (v6)', async () => { + const allSpans: SerializedStreamedSpanContainer['items'] = []; + await createRunner() + .unordered() .expect({ span: container => { - const segment = container.items.find(span => span.is_segment && span.name === 'main'); + allSpans.push(...container.items); + const segment = allSpans.find(span => span.is_segment && span.name === 'main'); expect(segment).toBeDefined(); - const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); expect(spans).toHaveLength(3); const singleEmbeddingSpan = spans.find( span => @@ -895,12 +899,16 @@ describe('OpenAI integration (V6)', () => { 'instrument-with-pii.mjs', (createRunner, test) => { test('creates openai related spans with genAI recording enabled (v6)', async () => { + const allSpans: SerializedStreamedSpanContainer['items'] = []; + await createRunner() + .unordered() .expect({ span: container => { - const segment = container.items.find(span => span.is_segment && span.name === 'main'); + allSpans.push(...container.items); + const segment = allSpans.find(span => span.is_segment && span.name === 'main'); expect(segment).toBeDefined(); - const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); expect(spans).toHaveLength(3); const singleEmbeddingSpan = spans.find( span => span.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value === 'Embedding test!', @@ -1044,14 +1052,18 @@ describe('OpenAI integration (V6)', () => { 'instrument.mjs', (createRunner, test) => { test('it works without a wrapping span (v6)', async () => { + const allSpans: SerializedStreamedSpanContainer['items'] = []; + await createRunner() + .unordered() .expect({ span: container => { - const serverSegment = container.items.find( + allSpans.push(...container.items); + const serverSegment = allSpans.find( span => span.is_segment && span.name === 'POST /openai/chat/completions', ); expect(serverSegment).toBeDefined(); - const segment = container.items.find(span => span.is_segment && span.name === 'chat gpt-3.5-turbo'); + const segment = allSpans.find(span => span.is_segment && span.name === 'chat gpt-3.5-turbo'); expect(segment?.span_id).toEqual(expect.any(String)); expect(segment?.trace_id).toEqual(expect.any(String)); expect(segment?.attributes[GEN_AI_OPERATION_NAME]).toEqual({ value: 'chat', type: 'string' }); @@ -1090,14 +1102,18 @@ describe('OpenAI integration (V6)', () => { 'instrument.mjs', (createRunner, test) => { test('it works with Azure OpenAI (v6)', async () => { + const allSpans: SerializedStreamedSpanContainer['items'] = []; + await createRunner() + .unordered() .expect({ span: container => { - const serverSegment = container.items.find( + allSpans.push(...container.items); + const serverSegment = allSpans.find( span => span.is_segment && span.name === 'POST /azureopenai/deployments/:model/chat/completions', ); expect(serverSegment).toBeDefined(); - const segment = container.items.find(span => span.is_segment && span.name === 'chat gpt-3.5-turbo'); + const segment = allSpans.find(span => span.is_segment && span.name === 'chat gpt-3.5-turbo'); expect(segment?.span_id).toEqual(expect.any(String)); expect(segment?.trace_id).toEqual(expect.any(String)); expect(segment?.attributes[GEN_AI_OPERATION_NAME]).toEqual({ value: 'chat', type: 'string' }); diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts b/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts index d4b62a5d5f5f..4910577c5012 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts @@ -123,13 +123,17 @@ conditionalTest({ min: 22 })('OpenAI integration (V7)', () => { 'instrument.mjs', (createRunner, test) => { test('instruments the embeddings API on openai v7', async () => { + const allSpans: SerializedStreamedSpanContainer['items'] = []; + await createRunner() + .unordered() .ignore('event') .expect({ span: container => { - const segment = container.items.find(span => span.is_segment && span.name === 'main'); + allSpans.push(...container.items); + const segment = allSpans.find(span => span.is_segment && span.name === 'main'); expect(segment).toBeDefined(); - const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); const embeddingSpans = spans.filter( span => span.attributes[GEN_AI_OPERATION_NAME]?.value === 'embeddings', ); From 7a7cc2e7a73be0891e6d446e1b119b05607bb7c6 Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Tue, 6 Oct 2026 12:12:30 +0200 Subject: [PATCH 5/8] test(node): Use convention constants for openai span attributes Co-Authored-By: GPT-6 --- .../suites/tracing/openai/test.ts | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/test.ts b/dev-packages/node-integration-tests/suites/tracing/openai/test.ts index 0d9430741532..0010916a3c8e 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/openai/test.ts @@ -1557,14 +1557,14 @@ describe('OpenAI integration', () => { for (const span of spans) { expect(span!.name).toBe('chat gpt-4'); expect(span!.status).toBe('ok'); - expect(span!.attributes['gen_ai.conversation.id']).toEqual({ + expect(span!.attributes[GEN_AI_CONVERSATION_ID]).toEqual({ type: 'string', value: 'user_chat_session_abc123', }); - expect(span!.attributes['gen_ai.provider.name']).toEqual({ type: 'string', value: 'openai' }); - expect(span!.attributes['gen_ai.request.model']).toEqual({ type: 'string', value: 'gpt-4' }); - expect(span!.attributes['gen_ai.operation.name']).toEqual({ type: 'string', value: 'chat' }); - expect(span!.attributes['sentry.op']).toEqual({ type: 'string', value: 'gen_ai.chat' }); + expect(span!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ type: 'string', value: 'openai' }); + expect(span!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ type: 'string', value: 'gpt-4' }); + expect(span!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ type: 'string', value: 'chat' }); + expect(span!.attributes[SENTRY_OP]).toEqual({ type: 'string', value: 'gen_ai.chat' }); } }, }) @@ -1591,13 +1591,13 @@ describe('OpenAI integration', () => { for (const span of spans) { expect(span!.name).toBe('chat gpt-4'); expect(span!.status).toBe('ok'); - expect(span!.attributes['gen_ai.conversation.id']).toEqual({ + expect(span!.attributes[GEN_AI_CONVERSATION_ID]).toEqual({ type: 'string', value: 'conv_user1_session_abc', }); - expect(span!.attributes['gen_ai.provider.name']).toEqual({ type: 'string', value: 'openai' }); - expect(span!.attributes['gen_ai.request.model']).toEqual({ type: 'string', value: 'gpt-4' }); - expect(span!.attributes['sentry.op']).toEqual({ type: 'string', value: 'gen_ai.chat' }); + expect(span!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ type: 'string', value: 'openai' }); + expect(span!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ type: 'string', value: 'gpt-4' }); + expect(span!.attributes[SENTRY_OP]).toEqual({ type: 'string', value: 'gen_ai.chat' }); } }, }) @@ -1624,13 +1624,13 @@ describe('OpenAI integration', () => { for (const span of spans) { expect(span!.name).toBe('chat gpt-4'); expect(span!.status).toBe('ok'); - expect(span!.attributes['gen_ai.conversation.id']).toEqual({ + expect(span!.attributes[GEN_AI_CONVERSATION_ID]).toEqual({ type: 'string', value: 'conv_user2_session_xyz', }); - expect(span!.attributes['gen_ai.provider.name']).toEqual({ type: 'string', value: 'openai' }); - expect(span!.attributes['gen_ai.request.model']).toEqual({ type: 'string', value: 'gpt-4' }); - expect(span!.attributes['sentry.op']).toEqual({ type: 'string', value: 'gen_ai.chat' }); + expect(span!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ type: 'string', value: 'openai' }); + expect(span!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ type: 'string', value: 'gpt-4' }); + expect(span!.attributes[SENTRY_OP]).toEqual({ type: 'string', value: 'gen_ai.chat' }); } }, }) From d3168434a7be36b641d3cd8bacd99cc55507bc78 Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Wed, 7 Oct 2026 09:29:33 +0200 Subject: [PATCH 6/8] test(node): Assert OpenAI spans after selecting their container Co-Authored-By: GPT-6 --- .../suites/tracing/openai/test.ts | 1156 +++++++++-------- .../suites/tracing/openai/v6/test.ts | 590 ++++----- .../suites/tracing/openai/v7/test.ts | 56 +- 3 files changed, 908 insertions(+), 894 deletions(-) diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/test.ts b/dev-packages/node-integration-tests/suites/tracing/openai/test.ts index 0010916a3c8e..ce0a96822e31 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/openai/test.ts @@ -993,649 +993,661 @@ describe('OpenAI integration', () => { createEsmAndCjsTests(__dirname, 'scenario-embeddings.mjs', 'instrument.mjs', (createRunner, test) => { test('creates openai related spans with genAI recording disabled', async () => { - const allSpans: SerializedStreamedSpanContainer['items'] = []; + let receivedSpans: SerializedStreamedSpanContainer['items'] = []; await createRunner() .unordered() .expect({ span: container => { - allSpans.push(...container.items); - const segment = allSpans.find(span => span.is_segment && span.name === 'main'); - expect(segment).toBeDefined(); - const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); - expect(spans).toHaveLength(4); - const singleEmbeddingSpan = spans.find( - span => - span.name === 'embeddings text-embedding-3-small' && - span.attributes[GEN_AI_REQUEST_DIMENSIONS_ATTRIBUTE] !== undefined, - ); - expect(singleEmbeddingSpan!.status).toBe('ok'); - expect(singleEmbeddingSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'embeddings', - }); - expect(singleEmbeddingSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.embeddings', - }); - expect(singleEmbeddingSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(singleEmbeddingSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(singleEmbeddingSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'text-embedding-3-small', - }); - expect(singleEmbeddingSpan!.attributes[GEN_AI_REQUEST_ENCODING_FORMAT_ATTRIBUTE]).toEqual({ - type: 'string', - value: 'float', - }); - expect(singleEmbeddingSpan!.attributes[GEN_AI_REQUEST_DIMENSIONS_ATTRIBUTE]).toEqual({ - type: 'integer', - value: 1536, - }); - expect(singleEmbeddingSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ - type: 'string', - value: 'text-embedding-3-small', - }); - expect(singleEmbeddingSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ - type: 'integer', - value: 10, - }); - expect(singleEmbeddingSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ - type: 'integer', - value: 10, - }); - - const errorEmbeddingSpan = spans.find(span => span.name === 'embeddings error-model'); - expect(errorEmbeddingSpan!.status).toBe('error'); - expect(errorEmbeddingSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'embeddings', - }); - expect(errorEmbeddingSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.embeddings', - }); - expect(errorEmbeddingSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(errorEmbeddingSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(errorEmbeddingSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'error-model', - }); - - const multiEmbeddingSpan = spans.find( - span => - span.name === 'embeddings text-embedding-3-small' && - span.attributes[GEN_AI_REQUEST_DIMENSIONS_ATTRIBUTE] === undefined, - ); - expect(multiEmbeddingSpan!.status).toBe('ok'); - expect(multiEmbeddingSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'embeddings', - }); - expect(multiEmbeddingSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.embeddings', - }); - expect(multiEmbeddingSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(multiEmbeddingSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(multiEmbeddingSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'text-embedding-3-small', - }); - expect(multiEmbeddingSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ - type: 'string', - value: 'text-embedding-3-small', - }); - expect(multiEmbeddingSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ - type: 'integer', - value: 10, - }); - expect(multiEmbeddingSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ - type: 'integer', - value: 10, - }); - - // GenAI inputs and outputs must never be recorded when `dataCollection.genAI` disables them. - // Asserting over every span in the envelope catches attributes leaking onto - // spans which are not individually inspected above. - for (const span of spans) { - expect(span.attributes[GEN_AI_INPUT_MESSAGES]).toBeUndefined(); - expect(span.attributes[GEN_AI_SYSTEM_INSTRUCTIONS]).toBeUndefined(); - expect(span.attributes[GEN_AI_RESPONSE_TEXT]).toBeUndefined(); - expect(span.attributes[GEN_AI_EMBEDDINGS_INPUT]).toBeUndefined(); - } - - const rawEmbeddingSpan = spans.find(span => span.name === 'embeddings text-embedding-3-large'); - expect(rawEmbeddingSpan!.status).toBe('ok'); - expect(rawEmbeddingSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'embeddings', - }); - expect(rawEmbeddingSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.embeddings', - }); - expect(rawEmbeddingSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(rawEmbeddingSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(rawEmbeddingSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'text-embedding-3-large', - }); - expect(rawEmbeddingSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toBeUndefined(); - expect(rawEmbeddingSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toBeUndefined(); - expect(rawEmbeddingSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toBeUndefined(); + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + receivedSpans = container.items; }, }) .start() .completed(); + + const spans = receivedSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + expect(spans).toHaveLength(4); + const singleEmbeddingSpan = spans.find( + span => + span.name === 'embeddings text-embedding-3-small' && + span.attributes[GEN_AI_REQUEST_DIMENSIONS_ATTRIBUTE] !== undefined, + ); + expect(singleEmbeddingSpan!.status).toBe('ok'); + expect(singleEmbeddingSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'embeddings', + }); + expect(singleEmbeddingSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.embeddings', + }); + expect(singleEmbeddingSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(singleEmbeddingSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(singleEmbeddingSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'text-embedding-3-small', + }); + expect(singleEmbeddingSpan!.attributes[GEN_AI_REQUEST_ENCODING_FORMAT_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'float', + }); + expect(singleEmbeddingSpan!.attributes[GEN_AI_REQUEST_DIMENSIONS_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 1536, + }); + expect(singleEmbeddingSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'text-embedding-3-small', + }); + expect(singleEmbeddingSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 10, + }); + expect(singleEmbeddingSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 10, + }); + + const errorEmbeddingSpan = spans.find(span => span.name === 'embeddings error-model'); + expect(errorEmbeddingSpan!.status).toBe('error'); + expect(errorEmbeddingSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'embeddings', + }); + expect(errorEmbeddingSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.embeddings', + }); + expect(errorEmbeddingSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(errorEmbeddingSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(errorEmbeddingSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'error-model', + }); + + const multiEmbeddingSpan = spans.find( + span => + span.name === 'embeddings text-embedding-3-small' && + span.attributes[GEN_AI_REQUEST_DIMENSIONS_ATTRIBUTE] === undefined, + ); + expect(multiEmbeddingSpan!.status).toBe('ok'); + expect(multiEmbeddingSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'embeddings', + }); + expect(multiEmbeddingSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.embeddings', + }); + expect(multiEmbeddingSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(multiEmbeddingSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(multiEmbeddingSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'text-embedding-3-small', + }); + expect(multiEmbeddingSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'text-embedding-3-small', + }); + expect(multiEmbeddingSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 10, + }); + expect(multiEmbeddingSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 10, + }); + + // GenAI inputs and outputs must never be recorded when `dataCollection.genAI` disables them. + // Asserting over every span in the envelope catches attributes leaking onto + // spans which are not individually inspected above. + for (const span of spans) { + expect(span.attributes[GEN_AI_INPUT_MESSAGES]).toBeUndefined(); + expect(span.attributes[GEN_AI_SYSTEM_INSTRUCTIONS]).toBeUndefined(); + expect(span.attributes[GEN_AI_RESPONSE_TEXT]).toBeUndefined(); + expect(span.attributes[GEN_AI_EMBEDDINGS_INPUT]).toBeUndefined(); + } + + const rawEmbeddingSpan = spans.find(span => span.name === 'embeddings text-embedding-3-large'); + expect(rawEmbeddingSpan!.status).toBe('ok'); + expect(rawEmbeddingSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'embeddings', + }); + expect(rawEmbeddingSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.embeddings', + }); + expect(rawEmbeddingSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(rawEmbeddingSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(rawEmbeddingSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'text-embedding-3-large', + }); + expect(rawEmbeddingSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toBeUndefined(); + expect(rawEmbeddingSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toBeUndefined(); + expect(rawEmbeddingSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toBeUndefined(); }); }); createEsmAndCjsTests(__dirname, 'scenario-embeddings.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { test('creates openai related spans with genAI recording enabled', async () => { - const allSpans: SerializedStreamedSpanContainer['items'] = []; + let receivedSpans: SerializedStreamedSpanContainer['items'] = []; await createRunner() .unordered() .expect({ span: container => { - allSpans.push(...container.items); - const segment = allSpans.find(span => span.is_segment && span.name === 'main'); - expect(segment).toBeDefined(); - const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); - expect(spans).toHaveLength(4); - const singleEmbeddingSpan = spans.find( - span => span.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value === 'Embedding test!', - ); - expect(singleEmbeddingSpan!.name).toBe('embeddings text-embedding-3-small'); - expect(singleEmbeddingSpan!.status).toBe('ok'); - expect(singleEmbeddingSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'embeddings', - }); - expect(singleEmbeddingSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.embeddings', - }); - expect(singleEmbeddingSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(singleEmbeddingSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(singleEmbeddingSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'text-embedding-3-small', - }); - expect(singleEmbeddingSpan!.attributes[GEN_AI_REQUEST_ENCODING_FORMAT_ATTRIBUTE]).toEqual({ - type: 'string', - value: 'float', - }); - expect(singleEmbeddingSpan!.attributes[GEN_AI_REQUEST_DIMENSIONS_ATTRIBUTE]).toEqual({ - type: 'integer', - value: 1536, - }); - expect(singleEmbeddingSpan!.attributes[GEN_AI_EMBEDDINGS_INPUT]).toEqual({ - type: 'string', - value: 'Embedding test!', - }); - expect(singleEmbeddingSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ - type: 'string', - value: 'text-embedding-3-small', - }); - expect(singleEmbeddingSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ - type: 'integer', - value: 10, - }); - expect(singleEmbeddingSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ - type: 'integer', - value: 10, - }); - - const errorEmbeddingSpan = spans.find( - span => span.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value === 'Error embedding test!', - ); - expect(errorEmbeddingSpan!.name).toBe('embeddings error-model'); - expect(errorEmbeddingSpan!.status).toBe('error'); - expect(errorEmbeddingSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'embeddings', - }); - expect(errorEmbeddingSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.embeddings', - }); - expect(errorEmbeddingSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(errorEmbeddingSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(errorEmbeddingSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'error-model', - }); - expect(errorEmbeddingSpan!.attributes[GEN_AI_EMBEDDINGS_INPUT]).toEqual({ - type: 'string', - value: 'Error embedding test!', - }); - - const multiEmbeddingSpan = spans.find( - span => - span.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value === - '["First input text","Second input text","Third input text"]', - ); - expect(multiEmbeddingSpan!.name).toBe('embeddings text-embedding-3-small'); - expect(multiEmbeddingSpan!.status).toBe('ok'); - expect(multiEmbeddingSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'embeddings', - }); - expect(multiEmbeddingSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.embeddings', - }); - expect(multiEmbeddingSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(multiEmbeddingSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(multiEmbeddingSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'text-embedding-3-small', - }); - expect(multiEmbeddingSpan!.attributes[GEN_AI_EMBEDDINGS_INPUT]).toEqual({ - type: 'string', - value: '["First input text","Second input text","Third input text"]', - }); - expect(multiEmbeddingSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ - type: 'string', - value: 'text-embedding-3-small', - }); - expect(multiEmbeddingSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ - type: 'integer', - value: 10, - }); - expect(multiEmbeddingSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ - type: 'integer', - value: 10, - }); - - const rawEmbeddingSpan = spans.find(span => span.name === 'embeddings text-embedding-3-large'); - expect(rawEmbeddingSpan!.status).toBe('ok'); - expect(rawEmbeddingSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'embeddings', - }); - expect(rawEmbeddingSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.embeddings', - }); - expect(rawEmbeddingSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(rawEmbeddingSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(rawEmbeddingSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'text-embedding-3-large', - }); - expect(rawEmbeddingSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toBeUndefined(); - expect(rawEmbeddingSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toBeUndefined(); - expect(rawEmbeddingSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toBeUndefined(); - expect(rawEmbeddingSpan!.attributes[GEN_AI_EMBEDDINGS_INPUT]).toEqual({ - type: 'string', - value: 'Raw embedding test!', - }); + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + receivedSpans = container.items; }, }) .start() .completed(); + + const spans = receivedSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + expect(spans).toHaveLength(4); + const singleEmbeddingSpan = spans.find( + span => span.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value === 'Embedding test!', + ); + expect(singleEmbeddingSpan!.name).toBe('embeddings text-embedding-3-small'); + expect(singleEmbeddingSpan!.status).toBe('ok'); + expect(singleEmbeddingSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'embeddings', + }); + expect(singleEmbeddingSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.embeddings', + }); + expect(singleEmbeddingSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(singleEmbeddingSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(singleEmbeddingSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'text-embedding-3-small', + }); + expect(singleEmbeddingSpan!.attributes[GEN_AI_REQUEST_ENCODING_FORMAT_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'float', + }); + expect(singleEmbeddingSpan!.attributes[GEN_AI_REQUEST_DIMENSIONS_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 1536, + }); + expect(singleEmbeddingSpan!.attributes[GEN_AI_EMBEDDINGS_INPUT]).toEqual({ + type: 'string', + value: 'Embedding test!', + }); + expect(singleEmbeddingSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'text-embedding-3-small', + }); + expect(singleEmbeddingSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 10, + }); + expect(singleEmbeddingSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 10, + }); + + const errorEmbeddingSpan = spans.find( + span => span.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value === 'Error embedding test!', + ); + expect(errorEmbeddingSpan!.name).toBe('embeddings error-model'); + expect(errorEmbeddingSpan!.status).toBe('error'); + expect(errorEmbeddingSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'embeddings', + }); + expect(errorEmbeddingSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.embeddings', + }); + expect(errorEmbeddingSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(errorEmbeddingSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(errorEmbeddingSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'error-model', + }); + expect(errorEmbeddingSpan!.attributes[GEN_AI_EMBEDDINGS_INPUT]).toEqual({ + type: 'string', + value: 'Error embedding test!', + }); + + const multiEmbeddingSpan = spans.find( + span => + span.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value === + '["First input text","Second input text","Third input text"]', + ); + expect(multiEmbeddingSpan!.name).toBe('embeddings text-embedding-3-small'); + expect(multiEmbeddingSpan!.status).toBe('ok'); + expect(multiEmbeddingSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'embeddings', + }); + expect(multiEmbeddingSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.embeddings', + }); + expect(multiEmbeddingSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(multiEmbeddingSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(multiEmbeddingSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'text-embedding-3-small', + }); + expect(multiEmbeddingSpan!.attributes[GEN_AI_EMBEDDINGS_INPUT]).toEqual({ + type: 'string', + value: '["First input text","Second input text","Third input text"]', + }); + expect(multiEmbeddingSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'text-embedding-3-small', + }); + expect(multiEmbeddingSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 10, + }); + expect(multiEmbeddingSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 10, + }); + + const rawEmbeddingSpan = spans.find(span => span.name === 'embeddings text-embedding-3-large'); + expect(rawEmbeddingSpan!.status).toBe('ok'); + expect(rawEmbeddingSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'embeddings', + }); + expect(rawEmbeddingSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.embeddings', + }); + expect(rawEmbeddingSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(rawEmbeddingSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(rawEmbeddingSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'text-embedding-3-large', + }); + expect(rawEmbeddingSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toBeUndefined(); + expect(rawEmbeddingSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toBeUndefined(); + expect(rawEmbeddingSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toBeUndefined(); + expect(rawEmbeddingSpan!.attributes[GEN_AI_EMBEDDINGS_INPUT]).toEqual({ + type: 'string', + value: 'Raw embedding test!', + }); }); }); createEsmAndCjsTests(__dirname, 'scenario-root-span.mjs', 'instrument.mjs', (createRunner, test) => { test('it works without a wrapping span', async () => { - const allSpans: SerializedStreamedSpanContainer['items'] = []; + let receivedSpans: SerializedStreamedSpanContainer['items'] = []; await createRunner() .unordered() .expect({ span: container => { - allSpans.push(...container.items); - const serverSegment = allSpans.find( - span => span.is_segment && span.name === 'POST /openai/chat/completions', - ); - expect(serverSegment).toBeDefined(); - const segment = allSpans.find(span => span.is_segment && span.name === 'chat gpt-3.5-turbo'); - expect(segment?.span_id).toEqual(expect.any(String)); - expect(segment?.trace_id).toEqual(expect.any(String)); - expect(segment?.attributes[GEN_AI_OPERATION_NAME]).toEqual({ value: 'chat', type: 'string' }); - expect(segment?.attributes[SENTRY_OP]).toEqual({ value: 'gen_ai.chat', type: 'string' }); - expect(segment?.attributes[SENTRY_ORIGIN]).toEqual({ value: 'auto.ai.openai', type: 'string' }); - expect(segment?.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ value: 'openai', type: 'string' }); - expect(segment?.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ value: 'gpt-3.5-turbo', type: 'string' }); - expect(segment?.attributes[GEN_AI_REQUEST_TEMPERATURE]).toEqual({ value: 0.7, type: 'double' }); - expect(segment?.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ value: 'gpt-3.5-turbo', type: 'string' }); - expect(segment?.attributes[GEN_AI_RESPONSE_ID]).toEqual({ value: 'chatcmpl-mock123', type: 'string' }); - expect(segment?.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ value: '["stop"]', type: 'string' }); - expect(segment?.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ value: 10, type: 'integer' }); - expect(segment?.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ value: 15, type: 'integer' }); - expect(segment?.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ value: 25, type: 'integer' }); - expect(segment?.status).toEqual('ok'); + expect(container.items.find(span => span.is_segment && span.name === 'chat gpt-3.5-turbo')).toBeDefined(); + receivedSpans = container.items; }, }) .start() .completed(); + + const serverSegment = receivedSpans.find( + span => span.is_segment && span.name === 'POST /openai/chat/completions', + ); + expect(serverSegment).toBeDefined(); + const segment = receivedSpans.find(span => span.is_segment && span.name === 'chat gpt-3.5-turbo'); + expect(segment?.span_id).toEqual(expect.any(String)); + expect(segment?.trace_id).toEqual(expect.any(String)); + expect(segment?.attributes[GEN_AI_OPERATION_NAME]).toEqual({ value: 'chat', type: 'string' }); + expect(segment?.attributes[SENTRY_OP]).toEqual({ value: 'gen_ai.chat', type: 'string' }); + expect(segment?.attributes[SENTRY_ORIGIN]).toEqual({ value: 'auto.ai.openai', type: 'string' }); + expect(segment?.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ value: 'openai', type: 'string' }); + expect(segment?.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ value: 'gpt-3.5-turbo', type: 'string' }); + expect(segment?.attributes[GEN_AI_REQUEST_TEMPERATURE]).toEqual({ value: 0.7, type: 'double' }); + expect(segment?.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ value: 'gpt-3.5-turbo', type: 'string' }); + expect(segment?.attributes[GEN_AI_RESPONSE_ID]).toEqual({ value: 'chatcmpl-mock123', type: 'string' }); + expect(segment?.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ value: '["stop"]', type: 'string' }); + expect(segment?.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ value: 10, type: 'integer' }); + expect(segment?.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ value: 15, type: 'integer' }); + expect(segment?.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ value: 25, type: 'integer' }); + expect(segment?.status).toEqual('ok'); }); }); createEsmAndCjsTests(__dirname, 'scenario-azure-openai.mjs', 'instrument.mjs', (createRunner, test) => { test('it works with Azure OpenAI', async () => { - const allSpans: SerializedStreamedSpanContainer['items'] = []; + let receivedSpans: SerializedStreamedSpanContainer['items'] = []; await createRunner() .unordered() .expect({ span: container => { - allSpans.push(...container.items); - const serverSegment = allSpans.find( - span => span.is_segment && span.name === 'POST /azureopenai/deployments/:model/chat/completions', - ); - expect(serverSegment).toBeDefined(); - const segment = allSpans.find(span => span.is_segment && span.name === 'chat gpt-3.5-turbo'); - expect(segment?.span_id).toEqual(expect.any(String)); - expect(segment?.trace_id).toEqual(expect.any(String)); - expect(segment?.attributes[GEN_AI_OPERATION_NAME]).toEqual({ value: 'chat', type: 'string' }); - expect(segment?.attributes[SENTRY_OP]).toEqual({ value: 'gen_ai.chat', type: 'string' }); - expect(segment?.attributes[SENTRY_ORIGIN]).toEqual({ value: 'auto.ai.openai', type: 'string' }); - expect(segment?.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ value: 'openai', type: 'string' }); - expect(segment?.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ value: 'gpt-3.5-turbo', type: 'string' }); - expect(segment?.attributes[GEN_AI_REQUEST_TEMPERATURE]).toEqual({ value: 0.7, type: 'double' }); - expect(segment?.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ value: 'gpt-3.5-turbo', type: 'string' }); - expect(segment?.attributes[GEN_AI_RESPONSE_ID]).toEqual({ value: 'chatcmpl-mock123', type: 'string' }); - expect(segment?.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ value: '["stop"]', type: 'string' }); - expect(segment?.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ value: 10, type: 'integer' }); - expect(segment?.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ value: 15, type: 'integer' }); - expect(segment?.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ value: 25, type: 'integer' }); - expect(segment?.status).toEqual('ok'); + expect(container.items.find(span => span.is_segment && span.name === 'chat gpt-3.5-turbo')).toBeDefined(); + receivedSpans = container.items; }, }) .start() .completed(); + + const serverSegment = receivedSpans.find( + span => span.is_segment && span.name === 'POST /azureopenai/deployments/:model/chat/completions', + ); + expect(serverSegment).toBeDefined(); + const segment = receivedSpans.find(span => span.is_segment && span.name === 'chat gpt-3.5-turbo'); + expect(segment?.span_id).toEqual(expect.any(String)); + expect(segment?.trace_id).toEqual(expect.any(String)); + expect(segment?.attributes[GEN_AI_OPERATION_NAME]).toEqual({ value: 'chat', type: 'string' }); + expect(segment?.attributes[SENTRY_OP]).toEqual({ value: 'gen_ai.chat', type: 'string' }); + expect(segment?.attributes[SENTRY_ORIGIN]).toEqual({ value: 'auto.ai.openai', type: 'string' }); + expect(segment?.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ value: 'openai', type: 'string' }); + expect(segment?.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ value: 'gpt-3.5-turbo', type: 'string' }); + expect(segment?.attributes[GEN_AI_REQUEST_TEMPERATURE]).toEqual({ value: 0.7, type: 'double' }); + expect(segment?.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ value: 'gpt-3.5-turbo', type: 'string' }); + expect(segment?.attributes[GEN_AI_RESPONSE_ID]).toEqual({ value: 'chatcmpl-mock123', type: 'string' }); + expect(segment?.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ value: '["stop"]', type: 'string' }); + expect(segment?.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ value: 10, type: 'integer' }); + expect(segment?.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ value: 15, type: 'integer' }); + expect(segment?.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ value: 25, type: 'integer' }); + expect(segment?.status).toEqual('ok'); }); }); // Test for conversation ID support (Conversations API and previous_response_id) createEsmAndCjsTests(__dirname, 'scenario-conversation.mjs', 'instrument.mjs', (createRunner, test) => { test('captures conversation ID from Conversations API and previous_response_id', async () => { - const allSpans: SerializedStreamedSpanContainer['items'] = []; + let receivedSpans: SerializedStreamedSpanContainer['items'] = []; await createRunner() .unordered() .expect({ span: container => { - allSpans.push(...container.items); - const segment = allSpans.find(span => span.is_segment && span.name === 'conversation-test'); - expect(segment).toBeDefined(); - const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); - expect(spans).toHaveLength(5); - const conversationCreateSpan = spans.find(span => span.name === 'chat'); - expect(conversationCreateSpan!.status).toBe('ok'); - expect(conversationCreateSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'chat', - }); - expect(conversationCreateSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(conversationCreateSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(conversationCreateSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(conversationCreateSpan!.attributes[GEN_AI_CONVERSATION_ID]).toEqual({ - type: 'string', - value: 'conv_689667905b048191b4740501625afd940c7533ace33a2dab', - }); - - const conversationResponseSpan = spans.find( - span => - span.attributes[GEN_AI_CONVERSATION_ID]?.value === - 'conv_689667905b048191b4740501625afd940c7533ace33a2dab' && - span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-4', - ); - expect(conversationResponseSpan!.status).toBe('ok'); - expect(conversationResponseSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'chat', - }); - expect(conversationResponseSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(conversationResponseSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(conversationResponseSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(conversationResponseSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'gpt-4', - }); - expect(conversationResponseSpan!.attributes[GEN_AI_CONVERSATION_ID]).toEqual({ - type: 'string', - value: 'conv_689667905b048191b4740501625afd940c7533ace33a2dab', - }); - - const unlinkedResponseSpan = spans.find( - span => - span.attributes[SENTRY_OP]?.value === 'gen_ai.chat' && - span.attributes[GEN_AI_CONVERSATION_ID] === undefined, - ); - expect(unlinkedResponseSpan!.status).toBe('ok'); - expect(unlinkedResponseSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(unlinkedResponseSpan!.attributes[GEN_AI_CONVERSATION_ID]).toBeUndefined(); - - const previousResponseSpan = spans.find( - span => span.attributes[GEN_AI_CONVERSATION_ID]?.value === 'resp_mock_conv_123', - ); - expect(previousResponseSpan!.status).toBe('ok'); - expect(previousResponseSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'chat', - }); - expect(previousResponseSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(previousResponseSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(previousResponseSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(previousResponseSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'gpt-4', - }); - expect(previousResponseSpan!.attributes[GEN_AI_CONVERSATION_ID]).toEqual({ - type: 'string', - value: 'resp_mock_conv_123', - }); - - const rawConversationSpan = spans.find( - span => span.name === 'chat' && span.attributes[GEN_AI_CONVERSATION_ID] === undefined, - ); - expect(rawConversationSpan!.status).toBe('ok'); - expect(rawConversationSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'chat', - }); - expect(rawConversationSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(rawConversationSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(rawConversationSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(rawConversationSpan!.attributes[GEN_AI_CONVERSATION_ID]).toBeUndefined(); + expect(container.items.find(span => span.is_segment && span.name === 'conversation-test')).toBeDefined(); + receivedSpans = container.items; }, }) .start() .completed(); + + const spans = receivedSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + expect(spans).toHaveLength(5); + const conversationCreateSpan = spans.find( + span => span.name === 'chat' && span.attributes[GEN_AI_RESPONSE_ID] !== undefined, + ); + expect(conversationCreateSpan!.status).toBe('ok'); + expect(conversationCreateSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'chat', + }); + expect(conversationCreateSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(conversationCreateSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(conversationCreateSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(conversationCreateSpan!.attributes[GEN_AI_CONVERSATION_ID]).toEqual({ + type: 'string', + value: 'conv_689667905b048191b4740501625afd940c7533ace33a2dab', + }); + + const conversationResponseSpan = spans.find( + span => + span.attributes[GEN_AI_CONVERSATION_ID]?.value === 'conv_689667905b048191b4740501625afd940c7533ace33a2dab' && + span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-4', + ); + expect(conversationResponseSpan!.status).toBe('ok'); + expect(conversationResponseSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'chat', + }); + expect(conversationResponseSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(conversationResponseSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(conversationResponseSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(conversationResponseSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'gpt-4', + }); + expect(conversationResponseSpan!.attributes[GEN_AI_CONVERSATION_ID]).toEqual({ + type: 'string', + value: 'conv_689667905b048191b4740501625afd940c7533ace33a2dab', + }); + + const unlinkedResponseSpan = spans.find( + span => + span.attributes[SENTRY_OP]?.value === 'gen_ai.chat' && + span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-4' && + span.attributes[GEN_AI_CONVERSATION_ID] === undefined, + ); + expect(unlinkedResponseSpan!.status).toBe('ok'); + expect(unlinkedResponseSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(unlinkedResponseSpan!.attributes[GEN_AI_CONVERSATION_ID]).toBeUndefined(); + + const previousResponseSpan = spans.find( + span => span.attributes[GEN_AI_CONVERSATION_ID]?.value === 'resp_mock_conv_123', + ); + expect(previousResponseSpan!.status).toBe('ok'); + expect(previousResponseSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'chat', + }); + expect(previousResponseSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(previousResponseSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(previousResponseSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(previousResponseSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'gpt-4', + }); + expect(previousResponseSpan!.attributes[GEN_AI_CONVERSATION_ID]).toEqual({ + type: 'string', + value: 'resp_mock_conv_123', + }); + + const rawConversationSpan = spans.find( + span => span.name === 'chat' && span.attributes[GEN_AI_CONVERSATION_ID] === undefined, + ); + expect(rawConversationSpan!.status).toBe('ok'); + expect(rawConversationSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'chat', + }); + expect(rawConversationSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(rawConversationSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(rawConversationSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(rawConversationSpan!.attributes[GEN_AI_CONVERSATION_ID]).toBeUndefined(); }); }); // Test for manual conversation ID setting using setConversationId() createEsmAndCjsTests(__dirname, 'scenario-manual-conversation-id.mjs', 'instrument.mjs', (createRunner, test) => { test('attaches manual conversation ID set via setConversationId() to all chat spans', async () => { - const allSpans: SerializedStreamedSpanContainer['items'] = []; + let receivedSpans: SerializedStreamedSpanContainer['items'] = []; await createRunner() .unordered() .expect({ span: container => { - allSpans.push(...container.items); - const segment = allSpans.find(span => span.is_segment && span.name === 'chat-with-manual-conversation-id'); - expect(segment).toBeDefined(); - const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); - expect(spans).toHaveLength(3); - - // All three chat completion spans should have the same manually-set conversation ID - for (const span of spans) { - expect(span!.name).toBe('chat gpt-4'); - expect(span!.status).toBe('ok'); - expect(span!.attributes[GEN_AI_CONVERSATION_ID]).toEqual({ - type: 'string', - value: 'user_chat_session_abc123', - }); - expect(span!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ type: 'string', value: 'openai' }); - expect(span!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ type: 'string', value: 'gpt-4' }); - expect(span!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ type: 'string', value: 'chat' }); - expect(span!.attributes[SENTRY_OP]).toEqual({ type: 'string', value: 'gen_ai.chat' }); - } + expect( + container.items.find(span => span.is_segment && span.name === 'chat-with-manual-conversation-id'), + ).toBeDefined(); + receivedSpans = container.items; }, }) .start() .completed(); + + const spans = receivedSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + expect(spans).toHaveLength(3); + + // All three chat completion spans should have the same manually-set conversation ID + for (const span of spans) { + expect(span!.name).toBe('chat gpt-4'); + expect(span!.status).toBe('ok'); + expect(span!.attributes[GEN_AI_CONVERSATION_ID]).toEqual({ + type: 'string', + value: 'user_chat_session_abc123', + }); + expect(span!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ type: 'string', value: 'openai' }); + expect(span!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ type: 'string', value: 'gpt-4' }); + expect(span!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ type: 'string', value: 'chat' }); + expect(span!.attributes[SENTRY_OP]).toEqual({ type: 'string', value: 'gen_ai.chat' }); + } }); }); createEsmAndCjsTests(__dirname, 'scenario-separate-scope-1.mjs', 'instrument.mjs', (createRunner, test) => { test('isolates conversation IDs across separate scopes - conversation 1', async () => { - const allSpans: SerializedStreamedSpanContainer['items'] = []; + let receivedSpans: SerializedStreamedSpanContainer['items'] = []; await createRunner() .unordered() .expect({ span: container => { - allSpans.push(...container.items); - const segment = allSpans.find(span => span.is_segment && span.name === 'GET /chat/conversation-1'); - expect(segment).toBeDefined(); - const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); - expect(spans).toHaveLength(2); - - // Both chat completion spans should have the expected conversation ID - for (const span of spans) { - expect(span!.name).toBe('chat gpt-4'); - expect(span!.status).toBe('ok'); - expect(span!.attributes[GEN_AI_CONVERSATION_ID]).toEqual({ - type: 'string', - value: 'conv_user1_session_abc', - }); - expect(span!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ type: 'string', value: 'openai' }); - expect(span!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ type: 'string', value: 'gpt-4' }); - expect(span!.attributes[SENTRY_OP]).toEqual({ type: 'string', value: 'gen_ai.chat' }); - } + expect( + container.items.find(span => span.is_segment && span.name === 'GET /chat/conversation-1'), + ).toBeDefined(); + receivedSpans = container.items; }, }) .start() .completed(); + + const spans = receivedSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + expect(spans).toHaveLength(2); + + // Both chat completion spans should have the expected conversation ID + for (const span of spans) { + expect(span!.name).toBe('chat gpt-4'); + expect(span!.status).toBe('ok'); + expect(span!.attributes[GEN_AI_CONVERSATION_ID]).toEqual({ + type: 'string', + value: 'conv_user1_session_abc', + }); + expect(span!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ type: 'string', value: 'openai' }); + expect(span!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ type: 'string', value: 'gpt-4' }); + expect(span!.attributes[SENTRY_OP]).toEqual({ type: 'string', value: 'gen_ai.chat' }); + } }); }); createEsmAndCjsTests(__dirname, 'scenario-separate-scope-2.mjs', 'instrument.mjs', (createRunner, test) => { test('isolates conversation IDs across separate scopes - conversation 2', async () => { - const allSpans: SerializedStreamedSpanContainer['items'] = []; + let receivedSpans: SerializedStreamedSpanContainer['items'] = []; await createRunner() .unordered() .expect({ span: container => { - allSpans.push(...container.items); - const segment = allSpans.find(span => span.is_segment && span.name === 'GET /chat/conversation-2'); - expect(segment).toBeDefined(); - const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); - expect(spans).toHaveLength(2); - - // Both chat completion spans should have the expected conversation ID - for (const span of spans) { - expect(span!.name).toBe('chat gpt-4'); - expect(span!.status).toBe('ok'); - expect(span!.attributes[GEN_AI_CONVERSATION_ID]).toEqual({ - type: 'string', - value: 'conv_user2_session_xyz', - }); - expect(span!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ type: 'string', value: 'openai' }); - expect(span!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ type: 'string', value: 'gpt-4' }); - expect(span!.attributes[SENTRY_OP]).toEqual({ type: 'string', value: 'gen_ai.chat' }); - } + expect( + container.items.find(span => span.is_segment && span.name === 'GET /chat/conversation-2'), + ).toBeDefined(); + receivedSpans = container.items; }, }) .start() .completed(); + + const spans = receivedSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + expect(spans).toHaveLength(2); + + // Both chat completion spans should have the expected conversation ID + for (const span of spans) { + expect(span!.name).toBe('chat gpt-4'); + expect(span!.status).toBe('ok'); + expect(span!.attributes[GEN_AI_CONVERSATION_ID]).toEqual({ + type: 'string', + value: 'conv_user2_session_xyz', + }); + expect(span!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ type: 'string', value: 'openai' }); + expect(span!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ type: 'string', value: 'gpt-4' }); + expect(span!.attributes[SENTRY_OP]).toEqual({ type: 'string', value: 'gen_ai.chat' }); + } }); }); @@ -1645,68 +1657,68 @@ describe('OpenAI integration', () => { 'instrument-with-pii.mjs', (createRunner, test) => { test('extracts system instructions from messages', async () => { - const allSpans: SerializedStreamedSpanContainer['items'] = []; + let receivedSpans: SerializedStreamedSpanContainer['items'] = []; await createRunner() .unordered() .expect({ span: container => { - allSpans.push(...container.items); - const segment = allSpans.find(span => span.is_segment && span.name === 'main'); - expect(segment).toBeDefined(); - const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); - expect(spans).toHaveLength(1); - const [firstSpan] = spans; - - // [0] chat completion with system instructions extracted from messages - expect(firstSpan!.attributes[GEN_AI_SYSTEM_INSTRUCTIONS]).toEqual({ - type: 'string', - value: JSON.stringify([{ type: 'text', content: 'You are a helpful assistant' }]), - }); + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + receivedSpans = container.items; }, }) .start() .completed(); + + const spans = receivedSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + expect(spans).toHaveLength(1); + const [firstSpan] = spans; + + // [0] chat completion with system instructions extracted from messages + expect(firstSpan!.attributes[GEN_AI_SYSTEM_INSTRUCTIONS]).toEqual({ + type: 'string', + value: JSON.stringify([{ type: 'text', content: 'You are a helpful assistant' }]), + }); }); }, ); createEsmAndCjsTests(__dirname, 'scenario-with-response.mjs', 'instrument.mjs', (createRunner, test) => { test('preserves .withResponse() method and works correctly', async () => { - const allSpans: SerializedStreamedSpanContainer['items'] = []; + let receivedSpans: SerializedStreamedSpanContainer['items'] = []; await createRunner() .unordered() .expect({ span: container => { - allSpans.push(...container.items); - const segment = allSpans.find(span => span.is_segment && span.name === 'main'); - expect(segment).toBeDefined(); - const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); - expect(spans).toHaveLength(2); - - for (const span of spans) { - expect(span!.name).toBe('chat gpt-4'); - expect(span!.status).toBe('ok'); - expect(span!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ type: 'string', value: 'chat' }); - expect(span!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ type: 'string', value: 'gpt-4' }); - } - - const parsedSpans = spans.filter(span => span.attributes[GEN_AI_RESPONSE_ID] !== undefined); - expect(parsedSpans).toHaveLength(1); - expect(parsedSpans[0]!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ - type: 'string', - value: 'chatcmpl-withresponse', - }); - - const rawSpans = spans.filter(span => span.attributes[GEN_AI_RESPONSE_ID] === undefined); - expect(rawSpans).toHaveLength(1); - expect(rawSpans[0]!.attributes[GEN_AI_RESPONSE_TEXT]).toBeUndefined(); - expect(rawSpans[0]!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toBeUndefined(); + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + receivedSpans = container.items; }, }) .start() .completed(); + + const spans = receivedSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + expect(spans).toHaveLength(2); + + for (const span of spans) { + expect(span!.name).toBe('chat gpt-4'); + expect(span!.status).toBe('ok'); + expect(span!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ type: 'string', value: 'chat' }); + expect(span!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ type: 'string', value: 'gpt-4' }); + } + + const parsedSpans = spans.filter(span => span.attributes[GEN_AI_RESPONSE_ID] !== undefined); + expect(parsedSpans).toHaveLength(1); + expect(parsedSpans[0]!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ + type: 'string', + value: 'chatcmpl-withresponse', + }); + + const rawSpans = spans.filter(span => span.attributes[GEN_AI_RESPONSE_ID] === undefined); + expect(rawSpans).toHaveLength(1); + expect(rawSpans[0]!.attributes[GEN_AI_RESPONSE_TEXT]).toBeUndefined(); + expect(rawSpans[0]!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toBeUndefined(); }); }); }); diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/v6/test.ts b/dev-packages/node-integration-tests/suites/tracing/openai/v6/test.ts index 7ffbdb5aae86..56ad71bf7dae 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/v6/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/openai/v6/test.ts @@ -761,129 +761,129 @@ describe('OpenAI integration (V6)', () => { 'instrument.mjs', (createRunner, test) => { test('creates openai related spans with genAI recording disabled (v6)', async () => { - const allSpans: SerializedStreamedSpanContainer['items'] = []; + let receivedSpans: SerializedStreamedSpanContainer['items'] = []; await createRunner() .unordered() .expect({ span: container => { - allSpans.push(...container.items); - const segment = allSpans.find(span => span.is_segment && span.name === 'main'); - expect(segment).toBeDefined(); - const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); - expect(spans).toHaveLength(3); - const singleEmbeddingSpan = spans.find( - span => - span.name === 'embeddings text-embedding-3-small' && - span.attributes[GEN_AI_REQUEST_DIMENSIONS_ATTRIBUTE] !== undefined, - ); - expect(singleEmbeddingSpan!.status).toBe('ok'); - expect(singleEmbeddingSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'embeddings', - }); - expect(singleEmbeddingSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.embeddings', - }); - expect(singleEmbeddingSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(singleEmbeddingSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(singleEmbeddingSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'text-embedding-3-small', - }); - expect(singleEmbeddingSpan!.attributes[GEN_AI_REQUEST_ENCODING_FORMAT_ATTRIBUTE]).toEqual({ - type: 'string', - value: 'float', - }); - expect(singleEmbeddingSpan!.attributes[GEN_AI_REQUEST_DIMENSIONS_ATTRIBUTE]).toEqual({ - type: 'integer', - value: 1536, - }); - expect(singleEmbeddingSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ - type: 'string', - value: 'text-embedding-3-small', - }); - expect(singleEmbeddingSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ - type: 'integer', - value: 10, - }); - expect(singleEmbeddingSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ - type: 'integer', - value: 10, - }); - - const errorEmbeddingSpan = spans.find(span => span.name === 'embeddings error-model'); - expect(errorEmbeddingSpan!.status).toBe('error'); - expect(errorEmbeddingSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'embeddings', - }); - expect(errorEmbeddingSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.embeddings', - }); - expect(errorEmbeddingSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(errorEmbeddingSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(errorEmbeddingSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'error-model', - }); - - const multiEmbeddingSpan = spans.find( - span => - span.name === 'embeddings text-embedding-3-small' && - span.attributes[GEN_AI_REQUEST_DIMENSIONS_ATTRIBUTE] === undefined, - ); - expect(multiEmbeddingSpan!.status).toBe('ok'); - expect(multiEmbeddingSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'embeddings', - }); - expect(multiEmbeddingSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.embeddings', - }); - expect(multiEmbeddingSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(multiEmbeddingSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(multiEmbeddingSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'text-embedding-3-small', - }); - expect(multiEmbeddingSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ - type: 'string', - value: 'text-embedding-3-small', - }); - expect(multiEmbeddingSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ - type: 'integer', - value: 10, - }); - expect(multiEmbeddingSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ - type: 'integer', - value: 10, - }); + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + receivedSpans = container.items; }, }) .start() .completed(); + + const spans = receivedSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + expect(spans).toHaveLength(3); + const singleEmbeddingSpan = spans.find( + span => + span.name === 'embeddings text-embedding-3-small' && + span.attributes[GEN_AI_REQUEST_DIMENSIONS_ATTRIBUTE] !== undefined, + ); + expect(singleEmbeddingSpan!.status).toBe('ok'); + expect(singleEmbeddingSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'embeddings', + }); + expect(singleEmbeddingSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.embeddings', + }); + expect(singleEmbeddingSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(singleEmbeddingSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(singleEmbeddingSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'text-embedding-3-small', + }); + expect(singleEmbeddingSpan!.attributes[GEN_AI_REQUEST_ENCODING_FORMAT_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'float', + }); + expect(singleEmbeddingSpan!.attributes[GEN_AI_REQUEST_DIMENSIONS_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 1536, + }); + expect(singleEmbeddingSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'text-embedding-3-small', + }); + expect(singleEmbeddingSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 10, + }); + expect(singleEmbeddingSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 10, + }); + + const errorEmbeddingSpan = spans.find(span => span.name === 'embeddings error-model'); + expect(errorEmbeddingSpan!.status).toBe('error'); + expect(errorEmbeddingSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'embeddings', + }); + expect(errorEmbeddingSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.embeddings', + }); + expect(errorEmbeddingSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(errorEmbeddingSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(errorEmbeddingSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'error-model', + }); + + const multiEmbeddingSpan = spans.find( + span => + span.name === 'embeddings text-embedding-3-small' && + span.attributes[GEN_AI_REQUEST_DIMENSIONS_ATTRIBUTE] === undefined, + ); + expect(multiEmbeddingSpan!.status).toBe('ok'); + expect(multiEmbeddingSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'embeddings', + }); + expect(multiEmbeddingSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.embeddings', + }); + expect(multiEmbeddingSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(multiEmbeddingSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(multiEmbeddingSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'text-embedding-3-small', + }); + expect(multiEmbeddingSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'text-embedding-3-small', + }); + expect(multiEmbeddingSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 10, + }); + expect(multiEmbeddingSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 10, + }); }); }, { @@ -899,144 +899,144 @@ describe('OpenAI integration (V6)', () => { 'instrument-with-pii.mjs', (createRunner, test) => { test('creates openai related spans with genAI recording enabled (v6)', async () => { - const allSpans: SerializedStreamedSpanContainer['items'] = []; + let receivedSpans: SerializedStreamedSpanContainer['items'] = []; await createRunner() .unordered() .expect({ span: container => { - allSpans.push(...container.items); - const segment = allSpans.find(span => span.is_segment && span.name === 'main'); - expect(segment).toBeDefined(); - const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); - expect(spans).toHaveLength(3); - const singleEmbeddingSpan = spans.find( - span => span.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value === 'Embedding test!', - ); - expect(singleEmbeddingSpan!.name).toBe('embeddings text-embedding-3-small'); - expect(singleEmbeddingSpan!.status).toBe('ok'); - expect(singleEmbeddingSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'embeddings', - }); - expect(singleEmbeddingSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.embeddings', - }); - expect(singleEmbeddingSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(singleEmbeddingSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(singleEmbeddingSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'text-embedding-3-small', - }); - expect(singleEmbeddingSpan!.attributes[GEN_AI_REQUEST_ENCODING_FORMAT_ATTRIBUTE]).toEqual({ - type: 'string', - value: 'float', - }); - expect(singleEmbeddingSpan!.attributes[GEN_AI_REQUEST_DIMENSIONS_ATTRIBUTE]).toEqual({ - type: 'integer', - value: 1536, - }); - expect(singleEmbeddingSpan!.attributes[GEN_AI_EMBEDDINGS_INPUT]).toEqual({ - type: 'string', - value: 'Embedding test!', - }); - expect(singleEmbeddingSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ - type: 'string', - value: 'text-embedding-3-small', - }); - expect(singleEmbeddingSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ - type: 'integer', - value: 10, - }); - expect(singleEmbeddingSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ - type: 'integer', - value: 10, - }); - - const errorEmbeddingSpan = spans.find( - span => span.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value === 'Error embedding test!', - ); - expect(errorEmbeddingSpan!.name).toBe('embeddings error-model'); - expect(errorEmbeddingSpan!.status).toBe('error'); - expect(errorEmbeddingSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'embeddings', - }); - expect(errorEmbeddingSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.embeddings', - }); - expect(errorEmbeddingSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(errorEmbeddingSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(errorEmbeddingSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'error-model', - }); - expect(errorEmbeddingSpan!.attributes[GEN_AI_EMBEDDINGS_INPUT]).toEqual({ - type: 'string', - value: 'Error embedding test!', - }); - - const multiEmbeddingSpan = spans.find( - span => - span.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value === - '["First input text","Second input text","Third input text"]', - ); - expect(multiEmbeddingSpan!.name).toBe('embeddings text-embedding-3-small'); - expect(multiEmbeddingSpan!.status).toBe('ok'); - expect(multiEmbeddingSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'embeddings', - }); - expect(multiEmbeddingSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.embeddings', - }); - expect(multiEmbeddingSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(multiEmbeddingSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(multiEmbeddingSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'text-embedding-3-small', - }); - expect(multiEmbeddingSpan!.attributes[GEN_AI_EMBEDDINGS_INPUT]).toEqual({ - type: 'string', - value: '["First input text","Second input text","Third input text"]', - }); - expect(multiEmbeddingSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ - type: 'string', - value: 'text-embedding-3-small', - }); - expect(multiEmbeddingSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ - type: 'integer', - value: 10, - }); - expect(multiEmbeddingSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ - type: 'integer', - value: 10, - }); + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + receivedSpans = container.items; }, }) .start() .completed(); + + const spans = receivedSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + expect(spans).toHaveLength(3); + const singleEmbeddingSpan = spans.find( + span => span.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value === 'Embedding test!', + ); + expect(singleEmbeddingSpan!.name).toBe('embeddings text-embedding-3-small'); + expect(singleEmbeddingSpan!.status).toBe('ok'); + expect(singleEmbeddingSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'embeddings', + }); + expect(singleEmbeddingSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.embeddings', + }); + expect(singleEmbeddingSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(singleEmbeddingSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(singleEmbeddingSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'text-embedding-3-small', + }); + expect(singleEmbeddingSpan!.attributes[GEN_AI_REQUEST_ENCODING_FORMAT_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'float', + }); + expect(singleEmbeddingSpan!.attributes[GEN_AI_REQUEST_DIMENSIONS_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 1536, + }); + expect(singleEmbeddingSpan!.attributes[GEN_AI_EMBEDDINGS_INPUT]).toEqual({ + type: 'string', + value: 'Embedding test!', + }); + expect(singleEmbeddingSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'text-embedding-3-small', + }); + expect(singleEmbeddingSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 10, + }); + expect(singleEmbeddingSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 10, + }); + + const errorEmbeddingSpan = spans.find( + span => span.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value === 'Error embedding test!', + ); + expect(errorEmbeddingSpan!.name).toBe('embeddings error-model'); + expect(errorEmbeddingSpan!.status).toBe('error'); + expect(errorEmbeddingSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'embeddings', + }); + expect(errorEmbeddingSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.embeddings', + }); + expect(errorEmbeddingSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(errorEmbeddingSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(errorEmbeddingSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'error-model', + }); + expect(errorEmbeddingSpan!.attributes[GEN_AI_EMBEDDINGS_INPUT]).toEqual({ + type: 'string', + value: 'Error embedding test!', + }); + + const multiEmbeddingSpan = spans.find( + span => + span.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value === + '["First input text","Second input text","Third input text"]', + ); + expect(multiEmbeddingSpan!.name).toBe('embeddings text-embedding-3-small'); + expect(multiEmbeddingSpan!.status).toBe('ok'); + expect(multiEmbeddingSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'embeddings', + }); + expect(multiEmbeddingSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.embeddings', + }); + expect(multiEmbeddingSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(multiEmbeddingSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(multiEmbeddingSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'text-embedding-3-small', + }); + expect(multiEmbeddingSpan!.attributes[GEN_AI_EMBEDDINGS_INPUT]).toEqual({ + type: 'string', + value: '["First input text","Second input text","Third input text"]', + }); + expect(multiEmbeddingSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'text-embedding-3-small', + }); + expect(multiEmbeddingSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 10, + }); + expect(multiEmbeddingSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 10, + }); }); }, { @@ -1052,40 +1052,42 @@ describe('OpenAI integration (V6)', () => { 'instrument.mjs', (createRunner, test) => { test('it works without a wrapping span (v6)', async () => { - const allSpans: SerializedStreamedSpanContainer['items'] = []; + let receivedSpans: SerializedStreamedSpanContainer['items'] = []; await createRunner() .unordered() .expect({ span: container => { - allSpans.push(...container.items); - const serverSegment = allSpans.find( - span => span.is_segment && span.name === 'POST /openai/chat/completions', - ); - expect(serverSegment).toBeDefined(); - const segment = allSpans.find(span => span.is_segment && span.name === 'chat gpt-3.5-turbo'); - expect(segment?.span_id).toEqual(expect.any(String)); - expect(segment?.trace_id).toEqual(expect.any(String)); - expect(segment?.attributes[GEN_AI_OPERATION_NAME]).toEqual({ value: 'chat', type: 'string' }); - expect(segment?.attributes[SENTRY_OP]).toEqual({ value: 'gen_ai.chat', type: 'string' }); - expect(segment?.attributes[SENTRY_ORIGIN]).toEqual({ value: 'auto.ai.openai', type: 'string' }); - expect(segment?.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ value: 'openai', type: 'string' }); - expect(segment?.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ value: 'gpt-3.5-turbo', type: 'string' }); - expect(segment?.attributes[GEN_AI_REQUEST_TEMPERATURE]).toEqual({ value: 0.7, type: 'double' }); - expect(segment?.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ value: 'gpt-3.5-turbo', type: 'string' }); - expect(segment?.attributes[GEN_AI_RESPONSE_ID]).toEqual({ value: 'chatcmpl-mock123', type: 'string' }); - expect(segment?.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ - value: '["stop"]', - type: 'string', - }); - expect(segment?.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ value: 10, type: 'integer' }); - expect(segment?.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ value: 15, type: 'integer' }); - expect(segment?.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ value: 25, type: 'integer' }); - expect(segment?.status).toEqual('ok'); + expect(container.items.find(span => span.is_segment && span.name === 'chat gpt-3.5-turbo')).toBeDefined(); + receivedSpans = container.items; }, }) .start() .completed(); + + const serverSegment = receivedSpans.find( + span => span.is_segment && span.name === 'POST /openai/chat/completions', + ); + expect(serverSegment).toBeDefined(); + const segment = receivedSpans.find(span => span.is_segment && span.name === 'chat gpt-3.5-turbo'); + expect(segment?.span_id).toEqual(expect.any(String)); + expect(segment?.trace_id).toEqual(expect.any(String)); + expect(segment?.attributes[GEN_AI_OPERATION_NAME]).toEqual({ value: 'chat', type: 'string' }); + expect(segment?.attributes[SENTRY_OP]).toEqual({ value: 'gen_ai.chat', type: 'string' }); + expect(segment?.attributes[SENTRY_ORIGIN]).toEqual({ value: 'auto.ai.openai', type: 'string' }); + expect(segment?.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ value: 'openai', type: 'string' }); + expect(segment?.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ value: 'gpt-3.5-turbo', type: 'string' }); + expect(segment?.attributes[GEN_AI_REQUEST_TEMPERATURE]).toEqual({ value: 0.7, type: 'double' }); + expect(segment?.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ value: 'gpt-3.5-turbo', type: 'string' }); + expect(segment?.attributes[GEN_AI_RESPONSE_ID]).toEqual({ value: 'chatcmpl-mock123', type: 'string' }); + expect(segment?.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ + value: '["stop"]', + type: 'string', + }); + expect(segment?.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ value: 10, type: 'integer' }); + expect(segment?.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ value: 15, type: 'integer' }); + expect(segment?.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ value: 25, type: 'integer' }); + expect(segment?.status).toEqual('ok'); }); }, { @@ -1102,40 +1104,42 @@ describe('OpenAI integration (V6)', () => { 'instrument.mjs', (createRunner, test) => { test('it works with Azure OpenAI (v6)', async () => { - const allSpans: SerializedStreamedSpanContainer['items'] = []; + let receivedSpans: SerializedStreamedSpanContainer['items'] = []; await createRunner() .unordered() .expect({ span: container => { - allSpans.push(...container.items); - const serverSegment = allSpans.find( - span => span.is_segment && span.name === 'POST /azureopenai/deployments/:model/chat/completions', - ); - expect(serverSegment).toBeDefined(); - const segment = allSpans.find(span => span.is_segment && span.name === 'chat gpt-3.5-turbo'); - expect(segment?.span_id).toEqual(expect.any(String)); - expect(segment?.trace_id).toEqual(expect.any(String)); - expect(segment?.attributes[GEN_AI_OPERATION_NAME]).toEqual({ value: 'chat', type: 'string' }); - expect(segment?.attributes[SENTRY_OP]).toEqual({ value: 'gen_ai.chat', type: 'string' }); - expect(segment?.attributes[SENTRY_ORIGIN]).toEqual({ value: 'auto.ai.openai', type: 'string' }); - expect(segment?.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ value: 'openai', type: 'string' }); - expect(segment?.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ value: 'gpt-3.5-turbo', type: 'string' }); - expect(segment?.attributes[GEN_AI_REQUEST_TEMPERATURE]).toEqual({ value: 0.7, type: 'double' }); - expect(segment?.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ value: 'gpt-3.5-turbo', type: 'string' }); - expect(segment?.attributes[GEN_AI_RESPONSE_ID]).toEqual({ value: 'chatcmpl-mock123', type: 'string' }); - expect(segment?.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ - value: '["stop"]', - type: 'string', - }); - expect(segment?.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ value: 10, type: 'integer' }); - expect(segment?.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ value: 15, type: 'integer' }); - expect(segment?.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ value: 25, type: 'integer' }); - expect(segment?.status).toEqual('ok'); + expect(container.items.find(span => span.is_segment && span.name === 'chat gpt-3.5-turbo')).toBeDefined(); + receivedSpans = container.items; }, }) .start() .completed(); + + const serverSegment = receivedSpans.find( + span => span.is_segment && span.name === 'POST /azureopenai/deployments/:model/chat/completions', + ); + expect(serverSegment).toBeDefined(); + const segment = receivedSpans.find(span => span.is_segment && span.name === 'chat gpt-3.5-turbo'); + expect(segment?.span_id).toEqual(expect.any(String)); + expect(segment?.trace_id).toEqual(expect.any(String)); + expect(segment?.attributes[GEN_AI_OPERATION_NAME]).toEqual({ value: 'chat', type: 'string' }); + expect(segment?.attributes[SENTRY_OP]).toEqual({ value: 'gen_ai.chat', type: 'string' }); + expect(segment?.attributes[SENTRY_ORIGIN]).toEqual({ value: 'auto.ai.openai', type: 'string' }); + expect(segment?.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ value: 'openai', type: 'string' }); + expect(segment?.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ value: 'gpt-3.5-turbo', type: 'string' }); + expect(segment?.attributes[GEN_AI_REQUEST_TEMPERATURE]).toEqual({ value: 0.7, type: 'double' }); + expect(segment?.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ value: 'gpt-3.5-turbo', type: 'string' }); + expect(segment?.attributes[GEN_AI_RESPONSE_ID]).toEqual({ value: 'chatcmpl-mock123', type: 'string' }); + expect(segment?.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ + value: '["stop"]', + type: 'string', + }); + expect(segment?.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ value: 10, type: 'integer' }); + expect(segment?.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ value: 15, type: 'integer' }); + expect(segment?.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ value: 25, type: 'integer' }); + expect(segment?.status).toEqual('ok'); }); }, { diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts b/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts index 4910577c5012..7f354ff8a133 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts @@ -123,45 +123,43 @@ conditionalTest({ min: 22 })('OpenAI integration (V7)', () => { 'instrument.mjs', (createRunner, test) => { test('instruments the embeddings API on openai v7', async () => { - const allSpans: SerializedStreamedSpanContainer['items'] = []; + let receivedSpans: SerializedStreamedSpanContainer['items'] = []; await createRunner() .unordered() .ignore('event') .expect({ span: container => { - allSpans.push(...container.items); - const segment = allSpans.find(span => span.is_segment && span.name === 'main'); - expect(segment).toBeDefined(); - const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); - const embeddingSpans = spans.filter( - span => span.attributes[GEN_AI_OPERATION_NAME]?.value === 'embeddings', - ); - expect(embeddingSpans).toHaveLength(3); - - const singleEmbeddingSpan = embeddingSpans.find( - span => span.name === 'embeddings text-embedding-3-small' && span.status === 'ok', - ); - expect(singleEmbeddingSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.embeddings', - }); - expect(singleEmbeddingSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(singleEmbeddingSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - - const errorEmbeddingSpan = embeddingSpans.find(span => span.name === 'embeddings error-model'); - expect(errorEmbeddingSpan).toBeDefined(); - expect(errorEmbeddingSpan!.status).not.toBe('ok'); + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + receivedSpans = container.items; }, }) .start() .completed(); + + const spans = receivedSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + const embeddingSpans = spans.filter(span => span.attributes[GEN_AI_OPERATION_NAME]?.value === 'embeddings'); + expect(embeddingSpans).toHaveLength(3); + + const singleEmbeddingSpan = embeddingSpans.find( + span => span.name === 'embeddings text-embedding-3-small' && span.status === 'ok', + ); + expect(singleEmbeddingSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.embeddings', + }); + expect(singleEmbeddingSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(singleEmbeddingSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + + const errorEmbeddingSpan = embeddingSpans.find(span => span.name === 'embeddings error-model'); + expect(errorEmbeddingSpan).toBeDefined(); + expect(errorEmbeddingSpan!.status).not.toBe('ok'); }); }, { From 9ec7a818e8f342ab3f45641f22dd9c74964b4608 Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Wed, 7 Oct 2026 14:11:50 +0200 Subject: [PATCH 7/8] test(node): Collect OpenAI chat spans until the main segment Co-Authored-By: GPT-6 --- .../suites/tracing/openai/test.ts | 1861 ++++++++--------- .../suites/tracing/openai/v6/test.ts | 1305 ++++++------ .../suites/tracing/openai/v7/test.ts | 126 +- 3 files changed, 1616 insertions(+), 1676 deletions(-) diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/test.ts b/dev-packages/node-integration-tests/suites/tracing/openai/test.ts index ce0a96822e31..7c1fb7e385ed 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/openai/test.ts @@ -34,960 +34,935 @@ describe('OpenAI integration', () => { createEsmAndCjsTests(__dirname, 'scenario-chat.mjs', 'instrument.mjs', (createRunner, test) => { test('creates openai related spans with genAI recording disabled', async () => { - const allSpans: SerializedStreamedSpanContainer['items'] = []; + const runner = createRunner(); + const spansPromise = runner.collectStreamedSpansUntilSegment('main'); - await createRunner() - .unordered() - .expect({ - span: container => { - allSpans.push(...container.items); - const segment = allSpans.find(span => span.is_segment && span.name === 'main'); - expect(segment).toBeDefined(); - const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); - expect(spans).toHaveLength(8); - const chatCompletionSpan = spans.find( - span => - span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123' && - span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-3.5-turbo', - ); - expect(chatCompletionSpan!.name).toBe('chat gpt-3.5-turbo'); - expect(chatCompletionSpan!.status).toBe('ok'); - expect(chatCompletionSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'chat', - }); - expect(chatCompletionSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(chatCompletionSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'gpt-3.5-turbo', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE]).toEqual({ - type: 'double', - value: 0.7, - }); - expect(chatCompletionSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ - type: 'string', - value: 'gpt-3.5-turbo', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ - type: 'string', - value: 'chatcmpl-mock123', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ - type: 'string', - value: '["stop"]', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ - type: 'integer', - value: 10, - }); - expect(chatCompletionSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ - type: 'integer', - value: 15, - }); - expect(chatCompletionSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ - type: 'integer', - value: 25, - }); - - const responsesSpan = spans.find( - span => - span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_mock456' && - span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-3.5-turbo', - ); - expect(responsesSpan!.name).toBe('chat gpt-3.5-turbo'); - expect(responsesSpan!.status).toBe('ok'); - expect(responsesSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'chat', - }); - expect(responsesSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(responsesSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(responsesSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ type: 'string', value: 'openai' }); - expect(responsesSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'gpt-3.5-turbo', - }); - expect(responsesSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ - type: 'string', - value: 'gpt-3.5-turbo', - }); - expect(responsesSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ - type: 'string', - value: 'resp_mock456', - }); - expect(responsesSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ - type: 'string', - value: '["completed"]', - }); - expect(responsesSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ - type: 'integer', - value: 5, - }); - expect(responsesSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ - type: 'integer', - value: 8, - }); - expect(responsesSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ - type: 'integer', - value: 13, - }); - - const nonStreamingErrorSpan = spans.find( - span => - span.name === 'chat error-model' && span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE] === undefined, - ); - expect(nonStreamingErrorSpan!.status).toBe('error'); - expect(nonStreamingErrorSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toBeUndefined(); - expect(nonStreamingErrorSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'chat', - }); - expect(nonStreamingErrorSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(nonStreamingErrorSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(nonStreamingErrorSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(nonStreamingErrorSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'error-model', - }); - - const streamingChatCompletionSpan = spans.find( - span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-stream-123', - ); - expect(streamingChatCompletionSpan!.name).toBe('chat gpt-4'); - expect(streamingChatCompletionSpan!.status).toBe('ok'); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'chat', - }); - expect(streamingChatCompletionSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(streamingChatCompletionSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'gpt-4', - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE]).toEqual({ - type: 'double', - value: 0.8, - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ - type: 'boolean', - value: true, - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ - type: 'string', - value: 'gpt-4', - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ - type: 'string', - value: 'chatcmpl-stream-123', - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ - type: 'string', - value: '["stop"]', - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ - type: 'integer', - value: 12, - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ - type: 'integer', - value: 18, - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ - type: 'integer', - value: 30, - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_STREAMING]).toEqual({ - type: 'boolean', - value: true, - }); - - const streamingResponsesSpan = spans.find( - span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_stream_456', - ); - expect(streamingResponsesSpan!.name).toBe('chat gpt-4'); - expect(streamingResponsesSpan!.status).toBe('ok'); - expect(streamingResponsesSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'chat', - }); - expect(streamingResponsesSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(streamingResponsesSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'gpt-4', - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ - type: 'boolean', - value: true, - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ - type: 'string', - value: 'gpt-4', - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ - type: 'string', - value: 'resp_stream_456', - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ - type: 'string', - value: '["in_progress","completed"]', - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ - type: 'integer', - value: 6, - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ - type: 'integer', - value: 10, - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ - type: 'integer', - value: 16, - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_STREAMING]).toEqual({ - type: 'boolean', - value: true, - }); - - const streamingErrorSpan = spans.find( - span => - span.name === 'chat error-model' && span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]?.value === true, - ); - expect(streamingErrorSpan!.status).toBe('error'); - expect(streamingErrorSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'chat', - }); - expect(streamingErrorSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'error-model', - }); - expect(streamingErrorSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ - type: 'boolean', - value: true, - }); - expect(streamingErrorSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(streamingErrorSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(streamingErrorSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - - const parsedChatCompletionSpan = spans.find( - span => - span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123' && - span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-4o', - ); - expect(parsedChatCompletionSpan!.name).toBe('chat gpt-4o'); - expect(parsedChatCompletionSpan!.status).toBe('ok'); - expect(parsedChatCompletionSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'chat', - }); - expect(parsedChatCompletionSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(parsedChatCompletionSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(parsedChatCompletionSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(parsedChatCompletionSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'gpt-4o', - }); - expect(parsedChatCompletionSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ - type: 'string', - value: 'gpt-4o', - }); - expect(parsedChatCompletionSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ - type: 'string', - value: 'chatcmpl-mock123', - }); - expect(parsedChatCompletionSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ - type: 'string', - value: '["stop"]', - }); - expect(parsedChatCompletionSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ - type: 'integer', - value: 10, - }); - expect(parsedChatCompletionSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ - type: 'integer', - value: 15, - }); - expect(parsedChatCompletionSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ - type: 'integer', - value: 25, - }); - - const parsedResponsesSpan = spans.find( - span => - span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_mock456' && - span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-4o', - ); - expect(parsedResponsesSpan!.name).toBe('chat gpt-4o'); - expect(parsedResponsesSpan!.status).toBe('ok'); - expect(parsedResponsesSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'chat', - }); - expect(parsedResponsesSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(parsedResponsesSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(parsedResponsesSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ type: 'string', value: 'openai' }); - expect(parsedResponsesSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'gpt-4o', - }); - expect(parsedResponsesSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ - type: 'string', - value: 'gpt-4o', - }); - expect(parsedResponsesSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ - type: 'string', - value: 'resp_mock456', - }); - expect(parsedResponsesSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ - type: 'string', - value: '["completed"]', - }); - expect(parsedResponsesSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ - type: 'integer', - value: 5, - }); - expect(parsedResponsesSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ - type: 'integer', - value: 8, - }); - expect(parsedResponsesSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ - type: 'integer', - value: 13, - }); - - // GenAI inputs and outputs must never be recorded when `dataCollection.genAI` disables them. - // Asserting over every GenAI span catches attributes leaking onto - // spans which are not individually inspected above. - for (const span of spans) { - expect(span.attributes[GEN_AI_INPUT_MESSAGES]).toBeUndefined(); - expect(span.attributes[GEN_AI_SYSTEM_INSTRUCTIONS]).toBeUndefined(); - expect(span.attributes[GEN_AI_RESPONSE_TEXT]).toBeUndefined(); - expect(span.attributes[GEN_AI_EMBEDDINGS_INPUT]).toBeUndefined(); - } - }, - }) - .start() - .completed(); + await runner.start().completed(); + + const allSpans = await spansPromise; + const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + expect(spans).toHaveLength(8); + const chatCompletionSpan = spans.find( + span => + span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123' && + span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-3.5-turbo', + ); + expect(chatCompletionSpan!.name).toBe('chat gpt-3.5-turbo'); + expect(chatCompletionSpan!.status).toBe('ok'); + expect(chatCompletionSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'chat', + }); + expect(chatCompletionSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(chatCompletionSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'gpt-3.5-turbo', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE]).toEqual({ + type: 'double', + value: 0.7, + }); + expect(chatCompletionSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'gpt-3.5-turbo', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ + type: 'string', + value: 'chatcmpl-mock123', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ + type: 'string', + value: '["stop"]', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 10, + }); + expect(chatCompletionSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 15, + }); + expect(chatCompletionSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 25, + }); + + const responsesSpan = spans.find( + span => + span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_mock456' && + span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-3.5-turbo', + ); + expect(responsesSpan!.name).toBe('chat gpt-3.5-turbo'); + expect(responsesSpan!.status).toBe('ok'); + expect(responsesSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'chat', + }); + expect(responsesSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(responsesSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(responsesSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ type: 'string', value: 'openai' }); + expect(responsesSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'gpt-3.5-turbo', + }); + expect(responsesSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'gpt-3.5-turbo', + }); + expect(responsesSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ + type: 'string', + value: 'resp_mock456', + }); + expect(responsesSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ + type: 'string', + value: '["completed"]', + }); + expect(responsesSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 5, + }); + expect(responsesSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 8, + }); + expect(responsesSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 13, + }); + + const nonStreamingErrorSpan = spans.find( + span => span.name === 'chat error-model' && span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE] === undefined, + ); + expect(nonStreamingErrorSpan!.status).toBe('error'); + expect(nonStreamingErrorSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toBeUndefined(); + expect(nonStreamingErrorSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'chat', + }); + expect(nonStreamingErrorSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(nonStreamingErrorSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(nonStreamingErrorSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(nonStreamingErrorSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'error-model', + }); + + const streamingChatCompletionSpan = spans.find( + span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-stream-123', + ); + expect(streamingChatCompletionSpan!.name).toBe('chat gpt-4'); + expect(streamingChatCompletionSpan!.status).toBe('ok'); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'chat', + }); + expect(streamingChatCompletionSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(streamingChatCompletionSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'gpt-4', + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE]).toEqual({ + type: 'double', + value: 0.8, + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ + type: 'boolean', + value: true, + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'gpt-4', + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ + type: 'string', + value: 'chatcmpl-stream-123', + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ + type: 'string', + value: '["stop"]', + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 12, + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 18, + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 30, + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_STREAMING]).toEqual({ + type: 'boolean', + value: true, + }); + + const streamingResponsesSpan = spans.find( + span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_stream_456', + ); + expect(streamingResponsesSpan!.name).toBe('chat gpt-4'); + expect(streamingResponsesSpan!.status).toBe('ok'); + expect(streamingResponsesSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'chat', + }); + expect(streamingResponsesSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(streamingResponsesSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'gpt-4', + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ + type: 'boolean', + value: true, + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'gpt-4', + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ + type: 'string', + value: 'resp_stream_456', + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ + type: 'string', + value: '["in_progress","completed"]', + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 6, + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 10, + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 16, + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_STREAMING]).toEqual({ + type: 'boolean', + value: true, + }); + + const streamingErrorSpan = spans.find( + span => span.name === 'chat error-model' && span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]?.value === true, + ); + expect(streamingErrorSpan!.status).toBe('error'); + expect(streamingErrorSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'chat', + }); + expect(streamingErrorSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'error-model', + }); + expect(streamingErrorSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ + type: 'boolean', + value: true, + }); + expect(streamingErrorSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(streamingErrorSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(streamingErrorSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + + const parsedChatCompletionSpan = spans.find( + span => + span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123' && + span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-4o', + ); + expect(parsedChatCompletionSpan!.name).toBe('chat gpt-4o'); + expect(parsedChatCompletionSpan!.status).toBe('ok'); + expect(parsedChatCompletionSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'chat', + }); + expect(parsedChatCompletionSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(parsedChatCompletionSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(parsedChatCompletionSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(parsedChatCompletionSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'gpt-4o', + }); + expect(parsedChatCompletionSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'gpt-4o', + }); + expect(parsedChatCompletionSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ + type: 'string', + value: 'chatcmpl-mock123', + }); + expect(parsedChatCompletionSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ + type: 'string', + value: '["stop"]', + }); + expect(parsedChatCompletionSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 10, + }); + expect(parsedChatCompletionSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 15, + }); + expect(parsedChatCompletionSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 25, + }); + + const parsedResponsesSpan = spans.find( + span => + span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_mock456' && + span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-4o', + ); + expect(parsedResponsesSpan!.name).toBe('chat gpt-4o'); + expect(parsedResponsesSpan!.status).toBe('ok'); + expect(parsedResponsesSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'chat', + }); + expect(parsedResponsesSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(parsedResponsesSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(parsedResponsesSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ type: 'string', value: 'openai' }); + expect(parsedResponsesSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'gpt-4o', + }); + expect(parsedResponsesSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'gpt-4o', + }); + expect(parsedResponsesSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ + type: 'string', + value: 'resp_mock456', + }); + expect(parsedResponsesSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ + type: 'string', + value: '["completed"]', + }); + expect(parsedResponsesSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 5, + }); + expect(parsedResponsesSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 8, + }); + expect(parsedResponsesSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 13, + }); + + // GenAI inputs and outputs must never be recorded when `dataCollection.genAI` disables them. + // Asserting over every GenAI span catches attributes leaking onto + // spans which are not individually inspected above. + for (const span of spans) { + expect(span.attributes[GEN_AI_INPUT_MESSAGES]).toBeUndefined(); + expect(span.attributes[GEN_AI_SYSTEM_INSTRUCTIONS]).toBeUndefined(); + expect(span.attributes[GEN_AI_RESPONSE_TEXT]).toBeUndefined(); + expect(span.attributes[GEN_AI_EMBEDDINGS_INPUT]).toBeUndefined(); + } }); }); - createEsmAndCjsTests(__dirname, 'scenario-chat.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { - test('creates openai related spans with genAI recording enabled', async () => { - const allSpans: SerializedStreamedSpanContainer['items'] = []; + createEsmAndCjsTests(__dirname, 'scenario-chat.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { + test('creates openai related spans with genAI recording enabled', async () => { + const runner = createRunner(); + const spansPromise = runner.collectStreamedSpansUntilSegment('main'); + + await runner.start().completed(); + + const allSpans = await spansPromise; + const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + expect(spans).toHaveLength(8); + const chatCompletionSpan = spans.find( + span => + span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123' && + span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-3.5-turbo', + ); + expect(chatCompletionSpan!.name).toBe('chat gpt-3.5-turbo'); + expect(chatCompletionSpan!.status).toBe('ok'); + expect(chatCompletionSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'chat', + }); + expect(chatCompletionSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(chatCompletionSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'gpt-3.5-turbo', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE]).toEqual({ + type: 'double', + value: 0.7, + }); + expect(chatCompletionSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toEqual({ + type: 'string', + value: '[{"role":"user","content":"What is the capital of France?"}]', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_SYSTEM_INSTRUCTIONS]).toEqual({ + type: 'string', + value: JSON.stringify([{ type: 'text', content: 'You are a helpful assistant.' }]), + }); + expect(chatCompletionSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'gpt-3.5-turbo', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ + type: 'string', + value: 'chatcmpl-mock123', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ + type: 'string', + value: '["stop"]', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toEqual({ + type: 'string', + value: '["Hello from OpenAI mock!"]', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 10, + }); + expect(chatCompletionSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 15, + }); + expect(chatCompletionSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 25, + }); + + const responsesSpan = spans.find( + span => + span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_mock456' && + span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-3.5-turbo', + ); + expect(responsesSpan!.name).toBe('chat gpt-3.5-turbo'); + expect(responsesSpan!.status).toBe('ok'); + expect(responsesSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'chat', + }); + expect(responsesSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(responsesSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(responsesSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ type: 'string', value: 'openai' }); + expect(responsesSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'gpt-3.5-turbo', + }); + expect(responsesSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toEqual({ + type: 'string', + value: 'Translate this to French: Hello', + }); + expect(responsesSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toEqual({ + type: 'string', + value: 'Response to: Translate this to French: Hello', + }); + expect(responsesSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ + type: 'string', + value: '["completed"]', + }); + expect(responsesSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'gpt-3.5-turbo', + }); + expect(responsesSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ + type: 'string', + value: 'resp_mock456', + }); + expect(responsesSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 5, + }); + expect(responsesSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 8, + }); + expect(responsesSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 13, + }); - await createRunner() - .unordered() - .expect({ - span: container => { - allSpans.push(...container.items); - const segment = allSpans.find(span => span.is_segment && span.name === 'main'); - expect(segment).toBeDefined(); - const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); - expect(spans).toHaveLength(8); - const chatCompletionSpan = spans.find( - span => - span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123' && - span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-3.5-turbo', - ); - expect(chatCompletionSpan!.name).toBe('chat gpt-3.5-turbo'); - expect(chatCompletionSpan!.status).toBe('ok'); - expect(chatCompletionSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'chat', - }); - expect(chatCompletionSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(chatCompletionSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'gpt-3.5-turbo', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE]).toEqual({ - type: 'double', - value: 0.7, - }); - expect(chatCompletionSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toEqual({ - type: 'string', - value: '[{"role":"user","content":"What is the capital of France?"}]', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_SYSTEM_INSTRUCTIONS]).toEqual({ - type: 'string', - value: JSON.stringify([{ type: 'text', content: 'You are a helpful assistant.' }]), - }); - expect(chatCompletionSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ - type: 'string', - value: 'gpt-3.5-turbo', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ - type: 'string', - value: 'chatcmpl-mock123', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ - type: 'string', - value: '["stop"]', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toEqual({ - type: 'string', - value: '["Hello from OpenAI mock!"]', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ - type: 'integer', - value: 10, - }); - expect(chatCompletionSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ - type: 'integer', - value: 15, - }); - expect(chatCompletionSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ - type: 'integer', - value: 25, - }); - - const responsesSpan = spans.find( - span => - span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_mock456' && - span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-3.5-turbo', - ); - expect(responsesSpan!.name).toBe('chat gpt-3.5-turbo'); - expect(responsesSpan!.status).toBe('ok'); - expect(responsesSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'chat', - }); - expect(responsesSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(responsesSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(responsesSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ type: 'string', value: 'openai' }); - expect(responsesSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'gpt-3.5-turbo', - }); - expect(responsesSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toEqual({ - type: 'string', - value: 'Translate this to French: Hello', - }); - expect(responsesSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toEqual({ - type: 'string', - value: 'Response to: Translate this to French: Hello', - }); - expect(responsesSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ - type: 'string', - value: '["completed"]', - }); - expect(responsesSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ - type: 'string', - value: 'gpt-3.5-turbo', - }); - expect(responsesSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ - type: 'string', - value: 'resp_mock456', - }); - expect(responsesSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ - type: 'integer', - value: 5, - }); - expect(responsesSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ - type: 'integer', - value: 8, - }); - expect(responsesSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ - type: 'integer', - value: 13, - }); - - const nonStreamingErrorSpan = spans.find( - span => - span.name === 'chat error-model' && span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE] === undefined, - ); - expect(nonStreamingErrorSpan!.status).toBe('error'); - expect(nonStreamingErrorSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toBeUndefined(); - expect(nonStreamingErrorSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'chat', - }); - expect(nonStreamingErrorSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(nonStreamingErrorSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(nonStreamingErrorSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(nonStreamingErrorSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'error-model', - }); - expect(nonStreamingErrorSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toEqual({ - type: 'string', - value: '[{"role":"user","content":"This will fail"}]', - }); - - const streamingChatCompletionSpan = spans.find( - span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-stream-123', - ); - expect(streamingChatCompletionSpan!.name).toBe('chat gpt-4'); - expect(streamingChatCompletionSpan!.status).toBe('ok'); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'chat', - }); - expect(streamingChatCompletionSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(streamingChatCompletionSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'gpt-4', - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE]).toEqual({ - type: 'double', - value: 0.8, - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ - type: 'boolean', - value: true, - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toEqual({ - type: 'string', - value: '[{"role":"user","content":"Tell me about streaming"}]', - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_SYSTEM_INSTRUCTIONS]).toEqual({ - type: 'string', - value: JSON.stringify([{ type: 'text', content: 'You are a helpful assistant.' }]), - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toEqual({ - type: 'string', - value: 'Hello from OpenAI streaming!', - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ - type: 'string', - value: '["stop"]', - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ - type: 'string', - value: 'chatcmpl-stream-123', - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ - type: 'string', - value: 'gpt-4', - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ - type: 'integer', - value: 12, - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ - type: 'integer', - value: 18, - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ - type: 'integer', - value: 30, - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_STREAMING]).toEqual({ - type: 'boolean', - value: true, - }); - - const streamingResponsesSpan = spans.find( - span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_stream_456', - ); - expect(streamingResponsesSpan!.name).toBe('chat gpt-4'); - expect(streamingResponsesSpan!.status).toBe('ok'); - expect(streamingResponsesSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'chat', - }); - expect(streamingResponsesSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(streamingResponsesSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'gpt-4', - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ - type: 'boolean', - value: true, - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toEqual({ - type: 'string', - value: 'Test streaming responses API', - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toEqual({ - type: 'string', - value: 'Streaming response to: Test streaming responses APITest streaming responses API', - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ - type: 'string', - value: '["in_progress","completed"]', - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ - type: 'string', - value: 'resp_stream_456', - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ - type: 'string', - value: 'gpt-4', - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ - type: 'integer', - value: 6, - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ - type: 'integer', - value: 10, - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ - type: 'integer', - value: 16, - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_STREAMING]).toEqual({ - type: 'boolean', - value: true, - }); - - const streamingErrorSpan = spans.find( - span => - span.name === 'chat error-model' && span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]?.value === true, - ); - expect(streamingErrorSpan!.status).toBe('error'); - expect(streamingErrorSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'chat', - }); - expect(streamingErrorSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'error-model', - }); - expect(streamingErrorSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ - type: 'boolean', - value: true, - }); - expect(streamingErrorSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toEqual({ - type: 'string', - value: '[{"role":"user","content":"This will fail"}]', - }); - expect(streamingErrorSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(streamingErrorSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(streamingErrorSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - - const parsedChatCompletionSpan = spans.find( - span => - span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123' && - span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-4o', - ); - expect(parsedChatCompletionSpan!.name).toBe('chat gpt-4o'); - expect(parsedChatCompletionSpan!.status).toBe('ok'); - expect(parsedChatCompletionSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'chat', - }); - expect(parsedChatCompletionSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(parsedChatCompletionSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(parsedChatCompletionSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(parsedChatCompletionSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'gpt-4o', - }); - expect(parsedChatCompletionSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toEqual({ - type: 'string', - value: '[{"role":"user","content":"What is the capital of France?"}]', - }); - expect(parsedChatCompletionSpan!.attributes[GEN_AI_SYSTEM_INSTRUCTIONS]).toBeUndefined(); - expect(parsedChatCompletionSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ - type: 'string', - value: 'gpt-4o', - }); - expect(parsedChatCompletionSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ - type: 'string', - value: 'chatcmpl-mock123', - }); - expect(parsedChatCompletionSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ - type: 'string', - value: '["stop"]', - }); - expect(parsedChatCompletionSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toEqual({ - type: 'string', - value: '["Hello from OpenAI mock!"]', - }); - expect(parsedChatCompletionSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ - type: 'integer', - value: 10, - }); - expect(parsedChatCompletionSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ - type: 'integer', - value: 15, - }); - expect(parsedChatCompletionSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ - type: 'integer', - value: 25, - }); - - const parsedResponsesSpan = spans.find( - span => - span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_mock456' && - span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-4o', - ); - expect(parsedResponsesSpan!.name).toBe('chat gpt-4o'); - expect(parsedResponsesSpan!.status).toBe('ok'); - expect(parsedResponsesSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'chat', - }); - expect(parsedResponsesSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(parsedResponsesSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(parsedResponsesSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ type: 'string', value: 'openai' }); - expect(parsedResponsesSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'gpt-4o', - }); - expect(parsedResponsesSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toEqual({ - type: 'string', - value: 'What is the capital of France?', - }); - expect(parsedResponsesSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toEqual({ - type: 'string', - value: 'Response to: What is the capital of France?', - }); - expect(parsedResponsesSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ - type: 'string', - value: '["completed"]', - }); - expect(parsedResponsesSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ - type: 'string', - value: 'gpt-4o', - }); - expect(parsedResponsesSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ - type: 'string', - value: 'resp_mock456', - }); - expect(parsedResponsesSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ - type: 'integer', - value: 5, - }); - expect(parsedResponsesSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ - type: 'integer', - value: 8, - }); - expect(parsedResponsesSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ - type: 'integer', - value: 13, - }); - }, - }) - .start() - .completed(); + const nonStreamingErrorSpan = spans.find( + span => span.name === 'chat error-model' && span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE] === undefined, + ); + expect(nonStreamingErrorSpan!.status).toBe('error'); + expect(nonStreamingErrorSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toBeUndefined(); + expect(nonStreamingErrorSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'chat', + }); + expect(nonStreamingErrorSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(nonStreamingErrorSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(nonStreamingErrorSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(nonStreamingErrorSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'error-model', + }); + expect(nonStreamingErrorSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toEqual({ + type: 'string', + value: '[{"role":"user","content":"This will fail"}]', + }); + + const streamingChatCompletionSpan = spans.find( + span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-stream-123', + ); + expect(streamingChatCompletionSpan!.name).toBe('chat gpt-4'); + expect(streamingChatCompletionSpan!.status).toBe('ok'); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'chat', + }); + expect(streamingChatCompletionSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(streamingChatCompletionSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'gpt-4', + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE]).toEqual({ + type: 'double', + value: 0.8, + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ + type: 'boolean', + value: true, + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toEqual({ + type: 'string', + value: '[{"role":"user","content":"Tell me about streaming"}]', + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_SYSTEM_INSTRUCTIONS]).toEqual({ + type: 'string', + value: JSON.stringify([{ type: 'text', content: 'You are a helpful assistant.' }]), + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toEqual({ + type: 'string', + value: 'Hello from OpenAI streaming!', + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ + type: 'string', + value: '["stop"]', + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ + type: 'string', + value: 'chatcmpl-stream-123', + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'gpt-4', + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 12, + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 18, + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 30, + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_STREAMING]).toEqual({ + type: 'boolean', + value: true, + }); + + const streamingResponsesSpan = spans.find( + span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_stream_456', + ); + expect(streamingResponsesSpan!.name).toBe('chat gpt-4'); + expect(streamingResponsesSpan!.status).toBe('ok'); + expect(streamingResponsesSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'chat', + }); + expect(streamingResponsesSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(streamingResponsesSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'gpt-4', + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ + type: 'boolean', + value: true, + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toEqual({ + type: 'string', + value: 'Test streaming responses API', + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toEqual({ + type: 'string', + value: 'Streaming response to: Test streaming responses APITest streaming responses API', + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ + type: 'string', + value: '["in_progress","completed"]', + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ + type: 'string', + value: 'resp_stream_456', + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'gpt-4', + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 6, + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 10, + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 16, + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_STREAMING]).toEqual({ + type: 'boolean', + value: true, + }); + + const streamingErrorSpan = spans.find( + span => span.name === 'chat error-model' && span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]?.value === true, + ); + expect(streamingErrorSpan!.status).toBe('error'); + expect(streamingErrorSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'chat', + }); + expect(streamingErrorSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'error-model', + }); + expect(streamingErrorSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ + type: 'boolean', + value: true, + }); + expect(streamingErrorSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toEqual({ + type: 'string', + value: '[{"role":"user","content":"This will fail"}]', + }); + expect(streamingErrorSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(streamingErrorSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(streamingErrorSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + + const parsedChatCompletionSpan = spans.find( + span => + span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123' && + span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-4o', + ); + expect(parsedChatCompletionSpan!.name).toBe('chat gpt-4o'); + expect(parsedChatCompletionSpan!.status).toBe('ok'); + expect(parsedChatCompletionSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'chat', + }); + expect(parsedChatCompletionSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(parsedChatCompletionSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(parsedChatCompletionSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(parsedChatCompletionSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'gpt-4o', + }); + expect(parsedChatCompletionSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toEqual({ + type: 'string', + value: '[{"role":"user","content":"What is the capital of France?"}]', + }); + expect(parsedChatCompletionSpan!.attributes[GEN_AI_SYSTEM_INSTRUCTIONS]).toBeUndefined(); + expect(parsedChatCompletionSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'gpt-4o', + }); + expect(parsedChatCompletionSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ + type: 'string', + value: 'chatcmpl-mock123', + }); + expect(parsedChatCompletionSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ + type: 'string', + value: '["stop"]', + }); + expect(parsedChatCompletionSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toEqual({ + type: 'string', + value: '["Hello from OpenAI mock!"]', + }); + expect(parsedChatCompletionSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 10, + }); + expect(parsedChatCompletionSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 15, + }); + expect(parsedChatCompletionSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 25, + }); + + const parsedResponsesSpan = spans.find( + span => + span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_mock456' && + span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-4o', + ); + expect(parsedResponsesSpan!.name).toBe('chat gpt-4o'); + expect(parsedResponsesSpan!.status).toBe('ok'); + expect(parsedResponsesSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'chat', + }); + expect(parsedResponsesSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(parsedResponsesSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(parsedResponsesSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ type: 'string', value: 'openai' }); + expect(parsedResponsesSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'gpt-4o', + }); + expect(parsedResponsesSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toEqual({ + type: 'string', + value: 'What is the capital of France?', + }); + expect(parsedResponsesSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toEqual({ + type: 'string', + value: 'Response to: What is the capital of France?', + }); + expect(parsedResponsesSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ + type: 'string', + value: '["completed"]', + }); + expect(parsedResponsesSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'gpt-4o', + }); + expect(parsedResponsesSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ + type: 'string', + value: 'resp_mock456', + }); + expect(parsedResponsesSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 5, + }); + expect(parsedResponsesSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 8, + }); + expect(parsedResponsesSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 13, + }); }); }); createEsmAndCjsTests(__dirname, 'scenario-chat.mjs', 'instrument-with-options.mjs', (createRunner, test) => { test('creates openai related spans with custom options', async () => { - const allSpans: SerializedStreamedSpanContainer['items'] = []; + const runner = createRunner(); + const spansPromise = runner.collectStreamedSpansUntilSegment('main'); - await createRunner() - .unordered() - .expect({ - span: container => { - allSpans.push(...container.items); - const segment = allSpans.find(span => span.is_segment && span.name === 'main'); - expect(segment).toBeDefined(); - const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); - expect(spans).toHaveLength(8); - const chatCompletionSpan = spans.find( - span => - span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123' && - span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-3.5-turbo', - ); - expect(chatCompletionSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toBeUndefined(); - expect(chatCompletionSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toMatchObject({ - type: 'string', - value: expect.any(String), - }); - expect(chatCompletionSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toMatchObject({ - type: 'string', - value: expect.any(String), - }); - - const streamingChatCompletionSpan = spans.find( - span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-stream-123', - ); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ - type: 'boolean', - value: true, - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toMatchObject({ - type: 'string', - value: expect.any(String), - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toMatchObject({ - type: 'string', - value: expect.any(String), - }); - - const parsedChatCompletionSpan = spans.find( - span => - span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123' && - span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-4o', - ); - expect(parsedChatCompletionSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toBeUndefined(); - expect(parsedChatCompletionSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toMatchObject({ - type: 'string', - value: expect.any(String), - }); - expect(parsedChatCompletionSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toMatchObject({ - type: 'string', - value: expect.any(String), - }); - - const parsedResponsesSpan = spans.find( - span => - span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_mock456' && - span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-4o', - ); - expect(parsedResponsesSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toBeUndefined(); - expect(parsedResponsesSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toMatchObject({ - type: 'string', - value: expect.any(String), - }); - expect(parsedResponsesSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toMatchObject({ - type: 'string', - value: expect.any(String), - }); - }, - }) - .start() - .completed(); + await runner.start().completed(); + + const allSpans = await spansPromise; + const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + expect(spans).toHaveLength(8); + const chatCompletionSpan = spans.find( + span => + span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123' && + span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-3.5-turbo', + ); + expect(chatCompletionSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toBeUndefined(); + expect(chatCompletionSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toMatchObject({ + type: 'string', + value: expect.any(String), + }); + expect(chatCompletionSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toMatchObject({ + type: 'string', + value: expect.any(String), + }); + + const streamingChatCompletionSpan = spans.find( + span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-stream-123', + ); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ + type: 'boolean', + value: true, + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toMatchObject({ + type: 'string', + value: expect.any(String), + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toMatchObject({ + type: 'string', + value: expect.any(String), + }); + + const parsedChatCompletionSpan = spans.find( + span => + span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123' && + span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-4o', + ); + expect(parsedChatCompletionSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toBeUndefined(); + expect(parsedChatCompletionSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toMatchObject({ + type: 'string', + value: expect.any(String), + }); + expect(parsedChatCompletionSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toMatchObject({ + type: 'string', + value: expect.any(String), + }); + + const parsedResponsesSpan = spans.find( + span => + span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_mock456' && + span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-4o', + ); + expect(parsedResponsesSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toBeUndefined(); + expect(parsedResponsesSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toMatchObject({ + type: 'string', + value: expect.any(String), + }); + expect(parsedResponsesSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toMatchObject({ + type: 'string', + value: expect.any(String), + }); }); }); diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/v6/test.ts b/dev-packages/node-integration-tests/suites/tracing/openai/v6/test.ts index 56ad71bf7dae..30d05aaf4fea 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/v6/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/openai/v6/test.ts @@ -37,296 +37,287 @@ describe('OpenAI integration (V6)', () => { 'instrument.mjs', (createRunner, test) => { test('creates openai related spans with genAI recording disabled (v6)', async () => { - const allSpans: SerializedStreamedSpanContainer['items'] = []; + const runner = createRunner(); + const spansPromise = runner.collectStreamedSpansUntilSegment('main'); - await createRunner() - .unordered() - .expect({ - span: container => { - allSpans.push(...container.items); - const segment = allSpans.find(span => span.is_segment && span.name === 'main'); - expect(segment).toBeDefined(); - const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); - expect(spans).toHaveLength(6); - const chatCompletionSpan = spans.find( - span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123', - ); - expect(chatCompletionSpan!.name).toBe('chat gpt-3.5-turbo'); - expect(chatCompletionSpan!.status).toBe('ok'); - expect(chatCompletionSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'chat', - }); - expect(chatCompletionSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(chatCompletionSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'gpt-3.5-turbo', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE]).toEqual({ - type: 'double', - value: 0.7, - }); - expect(chatCompletionSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ - type: 'string', - value: 'gpt-3.5-turbo', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ - type: 'string', - value: 'chatcmpl-mock123', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ - type: 'string', - value: '["stop"]', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ - type: 'integer', - value: 10, - }); - expect(chatCompletionSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ - type: 'integer', - value: 15, - }); - expect(chatCompletionSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ - type: 'integer', - value: 25, - }); + await runner.start().completed(); + + const allSpans = await spansPromise; + const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + expect(spans).toHaveLength(6); + const chatCompletionSpan = spans.find( + span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123', + ); + expect(chatCompletionSpan!.name).toBe('chat gpt-3.5-turbo'); + expect(chatCompletionSpan!.status).toBe('ok'); + expect(chatCompletionSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'chat', + }); + expect(chatCompletionSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(chatCompletionSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'gpt-3.5-turbo', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE]).toEqual({ + type: 'double', + value: 0.7, + }); + expect(chatCompletionSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'gpt-3.5-turbo', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ + type: 'string', + value: 'chatcmpl-mock123', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ + type: 'string', + value: '["stop"]', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 10, + }); + expect(chatCompletionSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 15, + }); + expect(chatCompletionSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 25, + }); - const responsesSpan = spans.find(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_mock456'); - expect(responsesSpan!.name).toBe('chat gpt-3.5-turbo'); - expect(responsesSpan!.status).toBe('ok'); - expect(responsesSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'chat', - }); - expect(responsesSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(responsesSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(responsesSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ type: 'string', value: 'openai' }); - expect(responsesSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'gpt-3.5-turbo', - }); - expect(responsesSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ - type: 'string', - value: 'gpt-3.5-turbo', - }); - expect(responsesSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ - type: 'string', - value: 'resp_mock456', - }); - expect(responsesSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ - type: 'string', - value: '["completed"]', - }); - expect(responsesSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ - type: 'integer', - value: 5, - }); - expect(responsesSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ - type: 'integer', - value: 8, - }); - expect(responsesSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ - type: 'integer', - value: 13, - }); + const responsesSpan = spans.find(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_mock456'); + expect(responsesSpan!.name).toBe('chat gpt-3.5-turbo'); + expect(responsesSpan!.status).toBe('ok'); + expect(responsesSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'chat', + }); + expect(responsesSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(responsesSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(responsesSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ type: 'string', value: 'openai' }); + expect(responsesSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'gpt-3.5-turbo', + }); + expect(responsesSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'gpt-3.5-turbo', + }); + expect(responsesSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ + type: 'string', + value: 'resp_mock456', + }); + expect(responsesSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ + type: 'string', + value: '["completed"]', + }); + expect(responsesSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 5, + }); + expect(responsesSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 8, + }); + expect(responsesSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 13, + }); - const nonStreamingErrorSpan = spans.find( - span => - span.name === 'chat error-model' && span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE] === undefined, - ); - expect(nonStreamingErrorSpan!.status).toBe('error'); - expect(nonStreamingErrorSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toBeUndefined(); - expect(nonStreamingErrorSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'chat', - }); - expect(nonStreamingErrorSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(nonStreamingErrorSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(nonStreamingErrorSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(nonStreamingErrorSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'error-model', - }); + const nonStreamingErrorSpan = spans.find( + span => span.name === 'chat error-model' && span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE] === undefined, + ); + expect(nonStreamingErrorSpan!.status).toBe('error'); + expect(nonStreamingErrorSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toBeUndefined(); + expect(nonStreamingErrorSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'chat', + }); + expect(nonStreamingErrorSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(nonStreamingErrorSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(nonStreamingErrorSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(nonStreamingErrorSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'error-model', + }); - const streamingChatCompletionSpan = spans.find( - span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-stream-123', - ); - expect(streamingChatCompletionSpan!.name).toBe('chat gpt-4'); - expect(streamingChatCompletionSpan!.status).toBe('ok'); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'chat', - }); - expect(streamingChatCompletionSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(streamingChatCompletionSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'gpt-4', - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE]).toEqual({ - type: 'double', - value: 0.8, - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ - type: 'boolean', - value: true, - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ - type: 'string', - value: 'gpt-4', - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ - type: 'string', - value: 'chatcmpl-stream-123', - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ - type: 'string', - value: '["stop"]', - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ - type: 'integer', - value: 12, - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ - type: 'integer', - value: 18, - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ - type: 'integer', - value: 30, - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_STREAMING]).toEqual({ - type: 'boolean', - value: true, - }); + const streamingChatCompletionSpan = spans.find( + span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-stream-123', + ); + expect(streamingChatCompletionSpan!.name).toBe('chat gpt-4'); + expect(streamingChatCompletionSpan!.status).toBe('ok'); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'chat', + }); + expect(streamingChatCompletionSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(streamingChatCompletionSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'gpt-4', + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE]).toEqual({ + type: 'double', + value: 0.8, + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ + type: 'boolean', + value: true, + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'gpt-4', + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ + type: 'string', + value: 'chatcmpl-stream-123', + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ + type: 'string', + value: '["stop"]', + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 12, + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 18, + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 30, + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_STREAMING]).toEqual({ + type: 'boolean', + value: true, + }); - const streamingResponsesSpan = spans.find( - span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_stream_456', - ); - expect(streamingResponsesSpan!.name).toBe('chat gpt-4'); - expect(streamingResponsesSpan!.status).toBe('ok'); - expect(streamingResponsesSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'chat', - }); - expect(streamingResponsesSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(streamingResponsesSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'gpt-4', - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ - type: 'boolean', - value: true, - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ - type: 'string', - value: 'gpt-4', - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ - type: 'string', - value: 'resp_stream_456', - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ - type: 'string', - value: '["in_progress","completed"]', - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ - type: 'integer', - value: 6, - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ - type: 'integer', - value: 10, - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ - type: 'integer', - value: 16, - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_STREAMING]).toEqual({ - type: 'boolean', - value: true, - }); + const streamingResponsesSpan = spans.find( + span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_stream_456', + ); + expect(streamingResponsesSpan!.name).toBe('chat gpt-4'); + expect(streamingResponsesSpan!.status).toBe('ok'); + expect(streamingResponsesSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'chat', + }); + expect(streamingResponsesSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(streamingResponsesSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'gpt-4', + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ + type: 'boolean', + value: true, + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'gpt-4', + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ + type: 'string', + value: 'resp_stream_456', + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ + type: 'string', + value: '["in_progress","completed"]', + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 6, + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 10, + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 16, + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_STREAMING]).toEqual({ + type: 'boolean', + value: true, + }); - const streamingErrorSpan = spans.find( - span => - span.name === 'chat error-model' && span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]?.value === true, - ); - expect(streamingErrorSpan!.status).toBe('error'); - expect(streamingErrorSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'chat', - }); - expect(streamingErrorSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'error-model', - }); - expect(streamingErrorSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ - type: 'boolean', - value: true, - }); - expect(streamingErrorSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(streamingErrorSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(streamingErrorSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - }, - }) - .start() - .completed(); + const streamingErrorSpan = spans.find( + span => span.name === 'chat error-model' && span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]?.value === true, + ); + expect(streamingErrorSpan!.status).toBe('error'); + expect(streamingErrorSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'chat', + }); + expect(streamingErrorSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'error-model', + }); + expect(streamingErrorSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ + type: 'boolean', + value: true, + }); + expect(streamingErrorSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(streamingErrorSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(streamingErrorSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); }); }, { @@ -342,344 +333,335 @@ describe('OpenAI integration (V6)', () => { 'instrument-with-pii.mjs', (createRunner, test) => { test('creates openai related spans with genAI recording enabled (v6)', async () => { - const allSpans: SerializedStreamedSpanContainer['items'] = []; + const runner = createRunner(); + const spansPromise = runner.collectStreamedSpansUntilSegment('main'); - await createRunner() - .unordered() - .expect({ - span: container => { - allSpans.push(...container.items); - const segment = allSpans.find(span => span.is_segment && span.name === 'main'); - expect(segment).toBeDefined(); - const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); - expect(spans).toHaveLength(6); - const chatCompletionSpan = spans.find( - span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123', - ); - expect(chatCompletionSpan!.name).toBe('chat gpt-3.5-turbo'); - expect(chatCompletionSpan!.status).toBe('ok'); - expect(chatCompletionSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'chat', - }); - expect(chatCompletionSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(chatCompletionSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'gpt-3.5-turbo', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE]).toEqual({ - type: 'double', - value: 0.7, - }); - expect(chatCompletionSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toEqual({ - type: 'string', - value: '[{"role":"user","content":"What is the capital of France?"}]', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_SYSTEM_INSTRUCTIONS]).toEqual({ - type: 'string', - value: '[{"type":"text","content":"You are a helpful assistant."}]', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ - type: 'string', - value: 'gpt-3.5-turbo', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ - type: 'string', - value: 'chatcmpl-mock123', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ - type: 'string', - value: '["stop"]', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toEqual({ - type: 'string', - value: '["Hello from OpenAI mock!"]', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ - type: 'integer', - value: 10, - }); - expect(chatCompletionSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ - type: 'integer', - value: 15, - }); - expect(chatCompletionSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ - type: 'integer', - value: 25, - }); + await runner.start().completed(); + + const allSpans = await spansPromise; + const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + expect(spans).toHaveLength(6); + const chatCompletionSpan = spans.find( + span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123', + ); + expect(chatCompletionSpan!.name).toBe('chat gpt-3.5-turbo'); + expect(chatCompletionSpan!.status).toBe('ok'); + expect(chatCompletionSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'chat', + }); + expect(chatCompletionSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(chatCompletionSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'gpt-3.5-turbo', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE]).toEqual({ + type: 'double', + value: 0.7, + }); + expect(chatCompletionSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toEqual({ + type: 'string', + value: '[{"role":"user","content":"What is the capital of France?"}]', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_SYSTEM_INSTRUCTIONS]).toEqual({ + type: 'string', + value: '[{"type":"text","content":"You are a helpful assistant."}]', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'gpt-3.5-turbo', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ + type: 'string', + value: 'chatcmpl-mock123', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ + type: 'string', + value: '["stop"]', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toEqual({ + type: 'string', + value: '["Hello from OpenAI mock!"]', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 10, + }); + expect(chatCompletionSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 15, + }); + expect(chatCompletionSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 25, + }); - const responsesSpan = spans.find(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_mock456'); - expect(responsesSpan!.name).toBe('chat gpt-3.5-turbo'); - expect(responsesSpan!.status).toBe('ok'); - expect(responsesSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'chat', - }); - expect(responsesSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(responsesSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(responsesSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ type: 'string', value: 'openai' }); - expect(responsesSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'gpt-3.5-turbo', - }); - expect(responsesSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toEqual({ - type: 'string', - value: 'Translate this to French: Hello', - }); - expect(responsesSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toEqual({ - type: 'string', - value: 'Response to: Translate this to French: Hello', - }); - expect(responsesSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ - type: 'string', - value: '["completed"]', - }); - expect(responsesSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ - type: 'string', - value: 'gpt-3.5-turbo', - }); - expect(responsesSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ - type: 'string', - value: 'resp_mock456', - }); - expect(responsesSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ - type: 'integer', - value: 5, - }); - expect(responsesSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ - type: 'integer', - value: 8, - }); - expect(responsesSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ - type: 'integer', - value: 13, - }); + const responsesSpan = spans.find(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_mock456'); + expect(responsesSpan!.name).toBe('chat gpt-3.5-turbo'); + expect(responsesSpan!.status).toBe('ok'); + expect(responsesSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'chat', + }); + expect(responsesSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(responsesSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(responsesSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ type: 'string', value: 'openai' }); + expect(responsesSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'gpt-3.5-turbo', + }); + expect(responsesSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toEqual({ + type: 'string', + value: 'Translate this to French: Hello', + }); + expect(responsesSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toEqual({ + type: 'string', + value: 'Response to: Translate this to French: Hello', + }); + expect(responsesSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ + type: 'string', + value: '["completed"]', + }); + expect(responsesSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'gpt-3.5-turbo', + }); + expect(responsesSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ + type: 'string', + value: 'resp_mock456', + }); + expect(responsesSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 5, + }); + expect(responsesSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 8, + }); + expect(responsesSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 13, + }); - const nonStreamingErrorSpan = spans.find( - span => - span.name === 'chat error-model' && span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE] === undefined, - ); - expect(nonStreamingErrorSpan!.status).toBe('error'); - expect(nonStreamingErrorSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toBeUndefined(); - expect(nonStreamingErrorSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'chat', - }); - expect(nonStreamingErrorSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(nonStreamingErrorSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(nonStreamingErrorSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(nonStreamingErrorSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'error-model', - }); - expect(nonStreamingErrorSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toEqual({ - type: 'string', - value: '[{"role":"user","content":"This will fail"}]', - }); + const nonStreamingErrorSpan = spans.find( + span => span.name === 'chat error-model' && span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE] === undefined, + ); + expect(nonStreamingErrorSpan!.status).toBe('error'); + expect(nonStreamingErrorSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toBeUndefined(); + expect(nonStreamingErrorSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'chat', + }); + expect(nonStreamingErrorSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(nonStreamingErrorSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(nonStreamingErrorSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(nonStreamingErrorSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'error-model', + }); + expect(nonStreamingErrorSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toEqual({ + type: 'string', + value: '[{"role":"user","content":"This will fail"}]', + }); - const streamingChatCompletionSpan = spans.find( - span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-stream-123', - ); - expect(streamingChatCompletionSpan!.name).toBe('chat gpt-4'); - expect(streamingChatCompletionSpan!.status).toBe('ok'); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'chat', - }); - expect(streamingChatCompletionSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(streamingChatCompletionSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'gpt-4', - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE]).toEqual({ - type: 'double', - value: 0.8, - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ - type: 'boolean', - value: true, - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toEqual({ - type: 'string', - value: '[{"role":"user","content":"Tell me about streaming"}]', - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_SYSTEM_INSTRUCTIONS]).toEqual({ - type: 'string', - value: '[{"type":"text","content":"You are a helpful assistant."}]', - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toEqual({ - type: 'string', - value: 'Hello from OpenAI streaming!', - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ - type: 'string', - value: '["stop"]', - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ - type: 'string', - value: 'chatcmpl-stream-123', - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ - type: 'string', - value: 'gpt-4', - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ - type: 'integer', - value: 12, - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ - type: 'integer', - value: 18, - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ - type: 'integer', - value: 30, - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_STREAMING]).toEqual({ - type: 'boolean', - value: true, - }); + const streamingChatCompletionSpan = spans.find( + span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-stream-123', + ); + expect(streamingChatCompletionSpan!.name).toBe('chat gpt-4'); + expect(streamingChatCompletionSpan!.status).toBe('ok'); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'chat', + }); + expect(streamingChatCompletionSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(streamingChatCompletionSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'gpt-4', + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE]).toEqual({ + type: 'double', + value: 0.8, + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ + type: 'boolean', + value: true, + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toEqual({ + type: 'string', + value: '[{"role":"user","content":"Tell me about streaming"}]', + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_SYSTEM_INSTRUCTIONS]).toEqual({ + type: 'string', + value: '[{"type":"text","content":"You are a helpful assistant."}]', + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toEqual({ + type: 'string', + value: 'Hello from OpenAI streaming!', + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ + type: 'string', + value: '["stop"]', + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ + type: 'string', + value: 'chatcmpl-stream-123', + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'gpt-4', + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 12, + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 18, + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 30, + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_STREAMING]).toEqual({ + type: 'boolean', + value: true, + }); - const streamingResponsesSpan = spans.find( - span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_stream_456', - ); - expect(streamingResponsesSpan!.name).toBe('chat gpt-4'); - expect(streamingResponsesSpan!.status).toBe('ok'); - expect(streamingResponsesSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'chat', - }); - expect(streamingResponsesSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(streamingResponsesSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'gpt-4', - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ - type: 'boolean', - value: true, - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toEqual({ - type: 'string', - value: 'Test streaming responses API', - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toEqual({ - type: 'string', - value: 'Streaming response to: Test streaming responses APITest streaming responses API', - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ - type: 'string', - value: '["in_progress","completed"]', - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ - type: 'string', - value: 'resp_stream_456', - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ - type: 'string', - value: 'gpt-4', - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ - type: 'integer', - value: 6, - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ - type: 'integer', - value: 10, - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ - type: 'integer', - value: 16, - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_STREAMING]).toEqual({ - type: 'boolean', - value: true, - }); + const streamingResponsesSpan = spans.find( + span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_stream_456', + ); + expect(streamingResponsesSpan!.name).toBe('chat gpt-4'); + expect(streamingResponsesSpan!.status).toBe('ok'); + expect(streamingResponsesSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'chat', + }); + expect(streamingResponsesSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(streamingResponsesSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'gpt-4', + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ + type: 'boolean', + value: true, + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toEqual({ + type: 'string', + value: 'Test streaming responses API', + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toEqual({ + type: 'string', + value: 'Streaming response to: Test streaming responses APITest streaming responses API', + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ + type: 'string', + value: '["in_progress","completed"]', + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ + type: 'string', + value: 'resp_stream_456', + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'gpt-4', + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 6, + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 10, + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 16, + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_STREAMING]).toEqual({ + type: 'boolean', + value: true, + }); - const streamingErrorSpan = spans.find( - span => - span.name === 'chat error-model' && span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]?.value === true, - ); - expect(streamingErrorSpan!.status).toBe('error'); - expect(streamingErrorSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'chat', - }); - expect(streamingErrorSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'error-model', - }); - expect(streamingErrorSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ - type: 'boolean', - value: true, - }); - expect(streamingErrorSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toEqual({ - type: 'string', - value: '[{"role":"user","content":"This will fail"}]', - }); - expect(streamingErrorSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(streamingErrorSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(streamingErrorSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - }, - }) - .start() - .completed(); + const streamingErrorSpan = spans.find( + span => span.name === 'chat error-model' && span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]?.value === true, + ); + expect(streamingErrorSpan!.status).toBe('error'); + expect(streamingErrorSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'chat', + }); + expect(streamingErrorSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'error-model', + }); + expect(streamingErrorSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ + type: 'boolean', + value: true, + }); + expect(streamingErrorSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toEqual({ + type: 'string', + value: '[{"role":"user","content":"This will fail"}]', + }); + expect(streamingErrorSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(streamingErrorSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(streamingErrorSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); }); }, { @@ -695,57 +677,50 @@ describe('OpenAI integration (V6)', () => { 'instrument-with-options.mjs', (createRunner, test) => { test('creates openai related spans with custom options (v6)', async () => { - const allSpans: SerializedStreamedSpanContainer['items'] = []; + const runner = createRunner(); + const spansPromise = runner.collectStreamedSpansUntilSegment('main'); - await createRunner() - .unordered() - .expect({ - span: container => { - allSpans.push(...container.items); - const segment = allSpans.find(span => span.is_segment && span.name === 'main'); - expect(segment).toBeDefined(); - const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); - expect(spans).toHaveLength(6); - const chatCompletionSpan = spans.find( - span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123', - ); - expect(chatCompletionSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toBeUndefined(); - expect(chatCompletionSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toMatchObject({ - type: 'string', - value: expect.any(String), - }); - expect(chatCompletionSpan!.attributes[GEN_AI_SYSTEM_INSTRUCTIONS]).toMatchObject({ - type: 'string', - value: expect.any(String), - }); - expect(chatCompletionSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toMatchObject({ - type: 'string', - value: expect.any(String), - }); + await runner.start().completed(); - const streamingChatCompletionSpan = spans.find( - span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-stream-123', - ); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ - type: 'boolean', - value: true, - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toMatchObject({ - type: 'string', - value: expect.any(String), - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_SYSTEM_INSTRUCTIONS]).toMatchObject({ - type: 'string', - value: expect.any(String), - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toMatchObject({ - type: 'string', - value: expect.any(String), - }); - }, - }) - .start() - .completed(); + const allSpans = await spansPromise; + const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + expect(spans).toHaveLength(6); + const chatCompletionSpan = spans.find( + span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123', + ); + expect(chatCompletionSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toBeUndefined(); + expect(chatCompletionSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toMatchObject({ + type: 'string', + value: expect.any(String), + }); + expect(chatCompletionSpan!.attributes[GEN_AI_SYSTEM_INSTRUCTIONS]).toMatchObject({ + type: 'string', + value: expect.any(String), + }); + expect(chatCompletionSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toMatchObject({ + type: 'string', + value: expect.any(String), + }); + + const streamingChatCompletionSpan = spans.find( + span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-stream-123', + ); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ + type: 'boolean', + value: true, + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toMatchObject({ + type: 'string', + value: expect.any(String), + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_SYSTEM_INSTRUCTIONS]).toMatchObject({ + type: 'string', + value: expect.any(String), + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toMatchObject({ + type: 'string', + value: expect.any(String), + }); }); }, { diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts b/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts index 7f354ff8a133..34199f155036 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts @@ -33,79 +33,69 @@ conditionalTest({ min: 22 })('OpenAI integration (V7)', () => { 'instrument.mjs', (createRunner, test) => { test('instruments chat completions, the responses API and streaming on openai v7', async () => { - const allSpans: SerializedStreamedSpanContainer['items'] = []; + const runner = createRunner().ignore('event'); + const spansPromise = runner.collectStreamedSpansUntilSegment('main'); - await createRunner() - .unordered() - .ignore('event') - .expect({ - span: container => { - allSpans.push(...container.items); - const segment = allSpans.find(span => span.is_segment && span.name === 'main'); - expect(segment).toBeDefined(); - const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); - expect(spans).toHaveLength(6); + await runner.start().completed(); + + const allSpans = await spansPromise; + const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + expect(spans).toHaveLength(6); - const chatCompletionSpan = spans.find( - span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123', - ); - expect(chatCompletionSpan!.name).toBe('chat gpt-3.5-turbo'); - expect(chatCompletionSpan!.status).toBe('ok'); - expect(chatCompletionSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(chatCompletionSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ type: 'string', value: 'openai' }); - expect(chatCompletionSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'gpt-3.5-turbo', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ - type: 'string', - value: '["stop"]', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ type: 'integer', value: 10 }); - expect(chatCompletionSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ - type: 'integer', - value: 15, - }); - expect(chatCompletionSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ type: 'integer', value: 25 }); + const chatCompletionSpan = spans.find( + span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123', + ); + expect(chatCompletionSpan!.name).toBe('chat gpt-3.5-turbo'); + expect(chatCompletionSpan!.status).toBe('ok'); + expect(chatCompletionSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(chatCompletionSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ type: 'string', value: 'openai' }); + expect(chatCompletionSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'gpt-3.5-turbo', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ + type: 'string', + value: '["stop"]', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ type: 'integer', value: 10 }); + expect(chatCompletionSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 15, + }); + expect(chatCompletionSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ type: 'integer', value: 25 }); - // The responses API is a separate instrumented resource file from chat completions. - const responsesSpan = spans.find(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_mock456'); - expect(responsesSpan!.name).toBe('chat gpt-3.5-turbo'); - expect(responsesSpan!.status).toBe('ok'); - expect(responsesSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ type: 'string', value: 'chat' }); - expect(responsesSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ - type: 'string', - value: 'gpt-3.5-turbo', - }); + // The responses API is a separate instrumented resource file from chat completions. + const responsesSpan = spans.find(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_mock456'); + expect(responsesSpan!.name).toBe('chat gpt-3.5-turbo'); + expect(responsesSpan!.status).toBe('ok'); + expect(responsesSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ type: 'string', value: 'chat' }); + expect(responsesSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'gpt-3.5-turbo', + }); - // Streaming goes through the patched async iterator rather than `beforeSpanEnd`, so it - // is the part most likely to break if the `Stream` shape changes across a major. - const streamingSpan = spans.find( - span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-stream-123', - ); - expect(streamingSpan!.name).toBe('chat gpt-4'); - expect(streamingSpan!.status).toBe('ok'); - expect(streamingSpan!.attributes[GEN_AI_RESPONSE_STREAMING]).toEqual({ type: 'boolean', value: true }); - expect(streamingSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ type: 'integer', value: 12 }); - expect(streamingSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ type: 'integer', value: 18 }); - expect(streamingSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ type: 'integer', value: 30 }); + // Streaming goes through the patched async iterator rather than `beforeSpanEnd`, so it + // is the part most likely to break if the `Stream` shape changes across a major. + const streamingSpan = spans.find(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-stream-123'); + expect(streamingSpan!.name).toBe('chat gpt-4'); + expect(streamingSpan!.status).toBe('ok'); + expect(streamingSpan!.attributes[GEN_AI_RESPONSE_STREAMING]).toEqual({ type: 'boolean', value: true }); + expect(streamingSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ type: 'integer', value: 12 }); + expect(streamingSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ type: 'integer', value: 18 }); + expect(streamingSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ type: 'integer', value: 30 }); - const errorSpan = spans.find(span => span.name === 'chat error-model' && span.status !== 'ok'); - expect(errorSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - }, - }) - .start() - .completed(); + const errorSpan = spans.find(span => span.name === 'chat error-model' && span.status !== 'ok'); + expect(errorSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); }); }, { From 868af2e0e3c9d084ec7f9bef3bd20545701f7eba Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Wed, 7 Oct 2026 14:22:15 +0200 Subject: [PATCH 8/8] test(node): Drop unrelated OpenAI mock server span checks Co-Authored-By: GPT-6 --- .../node-integration-tests/suites/tracing/openai/test.ts | 8 -------- .../suites/tracing/openai/v6/test.ts | 8 -------- 2 files changed, 16 deletions(-) diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/test.ts b/dev-packages/node-integration-tests/suites/tracing/openai/test.ts index 7c1fb7e385ed..754328af131a 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/openai/test.ts @@ -1318,10 +1318,6 @@ describe('OpenAI integration', () => { .start() .completed(); - const serverSegment = receivedSpans.find( - span => span.is_segment && span.name === 'POST /openai/chat/completions', - ); - expect(serverSegment).toBeDefined(); const segment = receivedSpans.find(span => span.is_segment && span.name === 'chat gpt-3.5-turbo'); expect(segment?.span_id).toEqual(expect.any(String)); expect(segment?.trace_id).toEqual(expect.any(String)); @@ -1356,10 +1352,6 @@ describe('OpenAI integration', () => { .start() .completed(); - const serverSegment = receivedSpans.find( - span => span.is_segment && span.name === 'POST /azureopenai/deployments/:model/chat/completions', - ); - expect(serverSegment).toBeDefined(); const segment = receivedSpans.find(span => span.is_segment && span.name === 'chat gpt-3.5-turbo'); expect(segment?.span_id).toEqual(expect.any(String)); expect(segment?.trace_id).toEqual(expect.any(String)); diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/v6/test.ts b/dev-packages/node-integration-tests/suites/tracing/openai/v6/test.ts index 30d05aaf4fea..c93586010b03 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/v6/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/openai/v6/test.ts @@ -1040,10 +1040,6 @@ describe('OpenAI integration (V6)', () => { .start() .completed(); - const serverSegment = receivedSpans.find( - span => span.is_segment && span.name === 'POST /openai/chat/completions', - ); - expect(serverSegment).toBeDefined(); const segment = receivedSpans.find(span => span.is_segment && span.name === 'chat gpt-3.5-turbo'); expect(segment?.span_id).toEqual(expect.any(String)); expect(segment?.trace_id).toEqual(expect.any(String)); @@ -1092,10 +1088,6 @@ describe('OpenAI integration (V6)', () => { .start() .completed(); - const serverSegment = receivedSpans.find( - span => span.is_segment && span.name === 'POST /azureopenai/deployments/:model/chat/completions', - ); - expect(serverSegment).toBeDefined(); const segment = receivedSpans.find(span => span.is_segment && span.name === 'chat gpt-3.5-turbo'); expect(segment?.span_id).toEqual(expect.any(String)); expect(segment?.trace_id).toEqual(expect.any(String));