diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/requirements.oldest.txt b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/requirements.oldest.txt index 7b1e5347..1d2c5e62 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/requirements.oldest.txt +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/requirements.oldest.txt @@ -20,6 +20,12 @@ # pyproject floors by UV_RESOLUTION=lowest-direct on the oldest tox factor, so they are NOT pinned # here to avoid drift between the declared bound and the tested version. +# Exception: this change depends on the fully-qualified error.type behavior added in +# opentelemetry-util-genai 1.1b0, which is not on PyPI yet. Install it from the workspace so the +# oldest env exercises the required behavior. Remove this once 1.1b0 is released and bump the +# pyproject floor to >= 1.1b0. +-e ./util/opentelemetry-util-genai + langchain-openai==0.2.0 langchain-aws==0.2.2 langchain-google-genai==2.0.0 diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_llm_call.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_llm_call.py index ebd585dc..be669736 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_llm_call.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_llm_call.py @@ -329,7 +329,9 @@ def assert_openai_completion_attributes_with_error( assert span is not None assert span.name == "chat gpt-3.5-turbo" attributes = span.attributes - assert attributes[error_attributes.ERROR_TYPE] == "AuthenticationError" + assert ( + attributes[error_attributes.ERROR_TYPE] == "openai.AuthenticationError" + ) assert attributes[gen_ai_attributes.GEN_AI_OPERATION_NAME] == "chat" assert ( attributes[gen_ai_attributes.GEN_AI_REQUEST_MODEL] == "gpt-3.5-turbo" @@ -490,7 +492,9 @@ def assert_duration_metric_when_error(metric, parent_span): def assert_duration_metric_attributes_when_error(attributes, parent_span): assert len(attributes) == 4 - assert attributes[error_attributes.ERROR_TYPE] == "AuthenticationError" + assert ( + attributes[error_attributes.ERROR_TYPE] == "openai.AuthenticationError" + ) assert attributes.get(gen_ai_attributes.GEN_AI_PROVIDER_NAME) == "openai" assert ( attributes.get(gen_ai_attributes.GEN_AI_OPERATION_NAME) diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/.changelog/282.fixed b/instrumentation/opentelemetry-instrumentation-genai-openai/.changelog/282.fixed new file mode 100644 index 00000000..ad0e8d4a --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/.changelog/282.fixed @@ -0,0 +1 @@ +Responses that fail now record the actual error type (`error` event code or the response's error code) instead of a generic `RuntimeError`, for both streaming and non-streaming calls. Responses that finish as incomplete (`max_output_tokens`/`content_filter`) are recorded as a finish reason rather than an error. diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/patch.py b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/patch.py index b38b3214..b0622561 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/patch.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/patch.py @@ -17,9 +17,6 @@ EmbeddingInvocation, InferenceInvocation, ) -from opentelemetry.util.genai.types import ( - Error, -) from .chat_wrappers import AsyncChatStreamWrapper, ChatStreamWrapper from .utils import ( @@ -89,7 +86,7 @@ def traced_method(wrapped, instance, args, kwargs): chat_invocation.stop() return result except Exception as error: - chat_invocation.fail(Error(type=type(error), message=str(error))) + chat_invocation.fail(error) raise return traced_method @@ -125,7 +122,7 @@ async def traced_method(wrapped, instance, args, kwargs): return result except Exception as error: - chat_invocation.fail(Error(type=type(error), message=str(error))) + chat_invocation.fail(error) raise return traced_method @@ -140,7 +137,7 @@ def traced_method(wrapped, instance, args, kwargs): try: result = wrapped(*args, **kwargs) except Exception as error: - invocation.fail(Error(type=type(error), message=str(error))) + invocation.fail(error) raise _safe_set_embeddings_response_properties(invocation, result) @@ -159,7 +156,7 @@ async def traced_method(wrapped, instance, args, kwargs): try: result = await wrapped(*args, **kwargs) except Exception as error: - invocation.fail(Error(type=type(error), message=str(error))) + invocation.fail(error) raise _safe_set_embeddings_response_properties(invocation, result) diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/patch_responses.py b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/patch_responses.py index 3eb4c443..704b66c5 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/patch_responses.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/patch_responses.py @@ -11,6 +11,7 @@ apply_request_attributes, extract_params, get_inference_creation_kwargs, + get_response_error, set_invocation_response_attributes, ) from .response_wrappers import ( @@ -115,12 +116,15 @@ def traced_method( capture_content, ) + response = cast("ResponseResult", parsed_result) set_invocation_response_attributes( - invocation, - cast("ResponseResult", parsed_result), - capture_content, + invocation, response, capture_content ) - invocation.stop() + error = get_response_error(response) + if error is not None: + invocation.fail(error) + else: + invocation.stop() return result except Exception as error: invocation.fail(error) @@ -194,12 +198,15 @@ async def traced_method( capture_content, ) + response = cast("ResponseResult", parsed_result) set_invocation_response_attributes( - invocation, - cast("ResponseResult", parsed_result), - capture_content, + invocation, response, capture_content ) - invocation.stop() + error = get_response_error(response) + if error is not None: + invocation.fail(error) + else: + invocation.stop() return result except Exception as error: invocation.fail(error) diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/response_extractors.py b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/response_extractors.py index 3b9bc7b9..0642e8d6 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/response_extractors.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/response_extractors.py @@ -25,6 +25,7 @@ from openai.types.responses.response_usage import ResponseUsage from opentelemetry.util.genai.types import ( + Error, InputMessage, OutputMessage, Text, @@ -57,6 +58,7 @@ try: from opentelemetry.util.genai.types import ( + Error, InputMessage, OutputMessage, Reasoning, @@ -66,6 +68,7 @@ ToolCallRequest as ToolCall, ) except ImportError: + Error = None InputMessage = None OutputMessage = None Reasoning = None @@ -365,6 +368,21 @@ def extract_finish_reasons(response: "Response | None") -> list[str]: return list(dict.fromkeys(finish_reasons)) +def get_response_error(response: "Response | None") -> "Error | None": + """Return an ``Error`` when the response failed, else ``None``. + + A failed response carries a ``ResponseError`` (``code`` + ``message``). + Incomplete responses (``incomplete_details``) are *not* errors — they + surface as a finish reason instead. + """ + if Response is None or Error is None or not isinstance(response, Response): + return None + error = response.error + if error is None or not getattr(error, "code", None): + return None + return Error(type=error.code, message=getattr(error, "message", None)) + + def get_inference_creation_kwargs( params: ResponseRequestParams, client_instance: object, diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/response_wrappers.py b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/response_wrappers.py index 8a3f0f11..d90543e2 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/response_wrappers.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/response_wrappers.py @@ -17,9 +17,11 @@ try: from opentelemetry.instrumentation.genai.openai.response_extractors import ( # pylint: disable=no-name-in-module + get_response_error, set_invocation_response_attributes, ) except ImportError: + get_response_error = None set_invocation_response_attributes = None if TYPE_CHECKING: @@ -140,10 +142,10 @@ def _stop( self._self_invocation.stop() self._self_response_telemetry_finalized = True - def _fail(self, message: str, error_type: type[BaseException]) -> None: + def _fail(self, error: Error) -> None: if self._self_response_telemetry_finalized: return - self._self_invocation.fail(Error(message=message, type=error_type)) + self._self_invocation.fail(error) self._self_response_telemetry_finalized = True def _process_chunk( @@ -155,7 +157,10 @@ def _on_stream_end(self) -> None: self._stop(None) def _on_stream_error(self, error: BaseException) -> None: - self._fail(str(error), type(error)) + if self._self_response_telemetry_finalized: + return + self._self_invocation.fail(error) + self._self_response_telemetry_finalized = True def get_final_response(self) -> "ParsedResponse[TextFormatT]": self.until_done() @@ -188,23 +193,26 @@ def process_event(self, event: "ResponseStreamEvent[TextFormatT]") -> None: if model: self._self_invocation.request_model = model - if event_type == "response.completed": + if event_type in {"response.completed", "response.incomplete"}: self._stop(response) return - if event_type in {"response.failed", "response.incomplete"}: + if event_type == "response.failed": _set_response_attributes( self._self_invocation, response, self._self_capture_content, ) - self._fail(event_type, RuntimeError) + error = ( + get_response_error(response) if get_response_error else None + ) + self._fail(error or Error(type=event_type, message=None)) return - if event_type == "response.error": - error_type = getattr(event, "code", None) or "response.error" - message = getattr(event, "message", None) or error_type - self._fail(message, RuntimeError) + if event_type == "error": + error_type = getattr(event, "code", None) or "error" + message = getattr(event, "message", None) + self._fail(Error(type=error_type, message=message)) class ResponseStreamWrapper( diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/requirements.oldest.txt b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/requirements.oldest.txt index af091b44..27d657a1 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/requirements.oldest.txt +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/requirements.oldest.txt @@ -21,6 +21,12 @@ # OpenTelemetry SDK and test utilities come transitively from opentelemetry-test-util-genai, which # every oldest env installs. Pin here only test-only deps that nothing else already provides. +# Exception: this change depends on the fully-qualified error.type behavior added in +# opentelemetry-util-genai 1.1b0, which is not on PyPI yet. Install it from the workspace so the +# oldest env exercises the required behavior. Remove this once 1.1b0 is released and bump the +# pyproject floor to >= 1.1b0. +-e ./util/opentelemetry-util-genai + numpy==2.3.1 ; python_version >= "3.11" numpy==2.1.0 ; python_version == "3.10" pydantic==2.12.5 diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_async_chat_completions.py b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_async_chat_completions.py index e04f9644..dd09f310 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_async_chat_completions.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_async_chat_completions.py @@ -161,7 +161,8 @@ async def test_async_chat_completion_bad_endpoint( ) assert 4242 == spans[0].attributes[ServerAttributes.SERVER_PORT] assert ( - "APIConnectionError" == spans[0].attributes[ErrorAttributes.ERROR_TYPE] + "openai.APIConnectionError" + == spans[0].attributes[ErrorAttributes.ERROR_TYPE] ) @@ -184,7 +185,10 @@ async def test_async_chat_completion_404( assert_all_attributes( spans[0], llm_model_value, latest_experimental_enabled ) - assert "NotFoundError" == spans[0].attributes[ErrorAttributes.ERROR_TYPE] + assert ( + "openai.NotFoundError" + == spans[0].attributes[ErrorAttributes.ERROR_TYPE] + ) @pytest.mark.asyncio() diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_async_embeddings.py b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_async_embeddings.py index da8099e4..3b397eed 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_async_embeddings.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_async_embeddings.py @@ -160,7 +160,10 @@ async def test_async_embeddings_error_handling( latest_experimental_enabled, operation_name="embeddings", ) - assert "NotFoundError" == spans[0].attributes[ErrorAttributes.ERROR_TYPE] + assert ( + "openai.NotFoundError" + == spans[0].attributes[ErrorAttributes.ERROR_TYPE] + ) @pytest.mark.asyncio diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_async_responses.py b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_async_responses.py index 4409048a..1010236b 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_async_responses.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_async_responses.py @@ -357,7 +357,10 @@ async def test_async_responses_create_connection_error( ) assert span.attributes[ServerAttributes.SERVER_ADDRESS] == "localhost" assert span.attributes[ServerAttributes.SERVER_PORT] == 4242 - assert span.attributes[ErrorAttributes.ERROR_TYPE] == "APIConnectionError" + assert ( + span.attributes[ErrorAttributes.ERROR_TYPE] + == "openai.APIConnectionError" + ) @pytest.mark.asyncio() @@ -381,7 +384,7 @@ async def test_async_responses_create_api_error( ) assert ( span.attributes[ErrorAttributes.ERROR_TYPE] - == type(exc_info.value).__name__ + == f"openai.{type(exc_info.value).__name__}" ) @@ -437,7 +440,10 @@ async def test_async_responses_stream_connection_error( assert ( span.attributes[GenAIAttributes.GEN_AI_REQUEST_MODEL] == DEFAULT_MODEL ) - assert span.attributes[ErrorAttributes.ERROR_TYPE] == "APIConnectionError" + assert ( + span.attributes[ErrorAttributes.ERROR_TYPE] + == "openai.APIConnectionError" + ) @pytest.mark.asyncio() @@ -655,7 +661,10 @@ async def test_async_responses_create_streaming_connection_error( assert ( span.attributes[GenAIAttributes.GEN_AI_REQUEST_MODEL] == DEFAULT_MODEL ) - assert span.attributes[ErrorAttributes.ERROR_TYPE] == "APIConnectionError" + assert ( + span.attributes[ErrorAttributes.ERROR_TYPE] + == "openai.APIConnectionError" + ) @pytest.mark.asyncio() diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_async_structured_outputs.py b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_async_structured_outputs.py index bf7797f9..6465775a 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_async_structured_outputs.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_async_structured_outputs.py @@ -187,4 +187,7 @@ async def test_async_structured_output_404( assert_all_attributes( spans[0], llm_model_value, latest_experimental_enabled ) - assert "NotFoundError" == spans[0].attributes[ErrorAttributes.ERROR_TYPE] + assert ( + "openai.NotFoundError" + == spans[0].attributes[ErrorAttributes.ERROR_TYPE] + ) diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_chat_completions.py b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_chat_completions.py index 0f3ea08d..bcb08768 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_chat_completions.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_chat_completions.py @@ -210,7 +210,8 @@ def test_chat_completion_bad_endpoint( ) assert 4242 == spans[0].attributes[ServerAttributes.SERVER_PORT] assert ( - "APIConnectionError" == spans[0].attributes[ErrorAttributes.ERROR_TYPE] + "openai.APIConnectionError" + == spans[0].attributes[ErrorAttributes.ERROR_TYPE] ) metrics = metric_reader.get_metrics_data().resource_metrics @@ -231,7 +232,7 @@ def test_chat_completion_bad_endpoint( duration_metric.data.data_points[0].attributes[ ErrorAttributes.ERROR_TYPE ] - == "APIConnectionError" + == "openai.APIConnectionError" ) @@ -253,7 +254,10 @@ def test_chat_completion_404( assert_all_attributes( spans[0], llm_model_value, latest_experimental_enabled ) - assert "NotFoundError" == spans[0].attributes[ErrorAttributes.ERROR_TYPE] + assert ( + "openai.NotFoundError" + == spans[0].attributes[ErrorAttributes.ERROR_TYPE] + ) metrics = metric_reader.get_metrics_data().resource_metrics assert len(metrics) == 1 @@ -273,7 +277,7 @@ def test_chat_completion_404( duration_metric.data.data_points[0].attributes[ ErrorAttributes.ERROR_TYPE ] - == "NotFoundError" + == "openai.NotFoundError" ) diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_embeddings.py b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_embeddings.py index 9bd87679..0964a6d2 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_embeddings.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_embeddings.py @@ -249,7 +249,8 @@ def test_embeddings_bad_endpoint( ) assert 4242 == spans[0].attributes[ServerAttributes.SERVER_PORT] assert ( - "APIConnectionError" == spans[0].attributes[ErrorAttributes.ERROR_TYPE] + "openai.APIConnectionError" + == spans[0].attributes[ErrorAttributes.ERROR_TYPE] ) # Verify metrics @@ -271,7 +272,7 @@ def test_embeddings_bad_endpoint( duration_metric.data.data_points[0].attributes[ ErrorAttributes.ERROR_TYPE ] - == "APIConnectionError" + == "openai.APIConnectionError" ) @@ -299,7 +300,10 @@ def test_embeddings_model_not_found( latest_experimental_enabled, operation_name="embeddings", ) - assert "NotFoundError" == spans[0].attributes[ErrorAttributes.ERROR_TYPE] + assert ( + "openai.NotFoundError" + == spans[0].attributes[ErrorAttributes.ERROR_TYPE] + ) # Verify metrics metrics = metric_reader.get_metrics_data().resource_metrics @@ -320,7 +324,7 @@ def test_embeddings_model_not_found( duration_metric.data.data_points[0].attributes[ ErrorAttributes.ERROR_TYPE ] - == "NotFoundError" + == "openai.NotFoundError" ) diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_response_extractors.py b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_response_extractors.py index 136892e5..a5edf21e 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_response_extractors.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_response_extractors.py @@ -210,6 +210,33 @@ def test_extract_finish_reasons_maps_terminal_message_and_tool_items( ] +def test_get_response_error_returns_error_for_failed_response(loaded_module): + response = _make_response( + status="failed", + error={"code": "server_error", "message": "boom"}, + ) + + error = loaded_module.get_response_error(response) + + assert error is not None + assert error.type == "server_error" + assert error.message == "boom" + + +def test_get_response_error_none_for_incomplete_response(loaded_module): + # Incomplete is a finish reason, not an error. + response = _make_response( + status="incomplete", + incomplete_details={"reason": "max_output_tokens"}, + ) + + assert loaded_module.get_response_error(response) is None + + +def test_get_response_error_none_for_successful_response(loaded_module): + assert loaded_module.get_response_error(_make_response()) is None + + def test_extract_output_type_handles_text_format_mapping(loaded_module): assert ( loaded_module.extract_params( diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_response_wrappers.py b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_response_wrappers.py index b85da731..2e1a445d 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_response_wrappers.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_response_wrappers.py @@ -12,6 +12,14 @@ ResponseStreamWrapper, ) +try: + from openai.types.responses.response import Response + + HAS_RESPONSES_TYPES = True +except ImportError: + Response = None + HAS_RESPONSES_TYPES = False + class _FakeManager: def __init__( @@ -389,15 +397,13 @@ async def test_async_stream_wrapper_exit_closes_without_exception(): @pytest.mark.asyncio async def test_async_stream_wrapper_exit_fails_and_closes_on_exception(): stream = _FakeAsyncStream() - wrapper = _make_async_stream_wrapper(stream) stopped = [] failures = [] - - def record_failure(message, error_type): - failures.append((message, error_type)) - + invocation = SimpleNamespace( + request_model=None, stop=_noop_stop, fail=failures.append + ) + wrapper = _make_async_stream_wrapper(stream, invocation=invocation) wrapper._stop = stopped.append - wrapper._fail = record_failure error = ValueError("boom") result = await wrapper.__aexit__(ValueError, error, None) @@ -405,7 +411,7 @@ def record_failure(message, error_type): assert result is False assert stream.close_calls == 1 assert not stopped - assert failures == [("boom", ValueError)] + assert failures == [error] @pytest.mark.asyncio @@ -469,18 +475,16 @@ async def test_async_stream_wrapper_until_done_consumes_stream(): async def test_async_stream_wrapper_fails_and_reraises_stream_errors(): error = ValueError("boom") stream = _FakeAsyncStream(error=error) - wrapper = _make_async_stream_wrapper(stream) failures = [] - - def record_failure(message, error_type): - failures.append((message, error_type)) - - wrapper._fail = record_failure + invocation = SimpleNamespace( + request_model=None, stop=_noop_stop, fail=failures.append + ) + wrapper = _make_async_stream_wrapper(stream, invocation=invocation) with pytest.raises(ValueError, match="boom"): await anext(wrapper) - assert failures == [("boom", ValueError)] + assert failures == [error] @pytest.mark.asyncio @@ -518,3 +522,90 @@ async def test_async_stream_get_final_response_waits_for_completion(): assert result is final_response assert stream.get_final_response_calls == 1 + + +_requires_responses_types = pytest.mark.skipif( + not HAS_RESPONSES_TYPES, + reason="Responses SDK types require a newer openai SDK", +) + + +def _make_response(**overrides): + payload = { + "id": "resp_1", + "created_at": 0.0, + "model": "gpt-4.1", + "object": "response", + "output": [], + "parallel_tool_calls": False, + "tool_choice": "auto", + "tools": [], + } + payload.update(overrides) + return Response.model_validate(payload) + + +def _capturing_invocation(): + calls = {"stop": 0, "fail": []} + invocation = SimpleNamespace(request_model=None, attributes={}) + invocation.stop = lambda: calls.__setitem__("stop", calls["stop"] + 1) + invocation.fail = lambda error: calls["fail"].append(error) + return invocation, calls + + +@_requires_responses_types +def test_process_event_failed_records_error_from_response_error(): + invocation, calls = _capturing_invocation() + wrapper = _make_stream_wrapper(_FakeSyncStream(), invocation=invocation) + event = SimpleNamespace( + type="response.failed", + response=_make_response( + status="failed", + error={"code": "server_error", "message": "boom"}, + ), + ) + + wrapper.process_event(event) + + assert calls["stop"] == 0 + (error,) = calls["fail"] + assert isinstance(error.type, str) + assert error.type == "server_error" + assert error.message == "boom" + + +@_requires_responses_types +def test_process_event_incomplete_is_not_an_error(): + invocation, calls = _capturing_invocation() + wrapper = _make_stream_wrapper(_FakeSyncStream(), invocation=invocation) + event = SimpleNamespace( + type="response.incomplete", + response=_make_response( + status="incomplete", + incomplete_details={"reason": "max_output_tokens"}, + ), + ) + + wrapper.process_event(event) + + assert calls["fail"] == [] + assert calls["stop"] == 1 + + +@_requires_responses_types +def test_process_event_error_event_records_error(): + invocation, calls = _capturing_invocation() + wrapper = _make_stream_wrapper(_FakeSyncStream(), invocation=invocation) + event = SimpleNamespace( + type="error", + code="rate_limit_exceeded", + message="slow down", + response=None, + ) + + wrapper.process_event(event) + + assert calls["stop"] == 0 + (error,) = calls["fail"] + assert error.type == "rate_limit_exceeded" + assert error.message == "slow down" diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_responses.py b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_responses.py index d28ed973..366b6772 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_responses.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_responses.py @@ -359,7 +359,10 @@ def test_responses_create_connection_error( ) assert span.attributes[ServerAttributes.SERVER_ADDRESS] == "localhost" assert span.attributes[ServerAttributes.SERVER_PORT] == 4242 - assert span.attributes[ErrorAttributes.ERROR_TYPE] == "APIConnectionError" + assert ( + span.attributes[ErrorAttributes.ERROR_TYPE] + == "openai.APIConnectionError" + ) @pytest.mark.vcr() @@ -380,7 +383,7 @@ def test_responses_create_api_error( ) assert ( span.attributes[ErrorAttributes.ERROR_TYPE] - == type(exc_info.value).__name__ + == f"openai.{type(exc_info.value).__name__}" ) @@ -446,7 +449,10 @@ def test_responses_stream_connection_error( assert ( span.attributes[GenAIAttributes.GEN_AI_REQUEST_MODEL] == DEFAULT_MODEL ) - assert span.attributes[ErrorAttributes.ERROR_TYPE] == "APIConnectionError" + assert ( + span.attributes[ErrorAttributes.ERROR_TYPE] + == "openai.APIConnectionError" + ) @pytest.mark.vcr() @@ -646,7 +652,10 @@ def test_responses_create_streaming_connection_error( assert ( span.attributes[GenAIAttributes.GEN_AI_REQUEST_MODEL] == DEFAULT_MODEL ) - assert span.attributes[ErrorAttributes.ERROR_TYPE] == "APIConnectionError" + assert ( + span.attributes[ErrorAttributes.ERROR_TYPE] + == "openai.APIConnectionError" + ) @pytest.mark.vcr() diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_structured_outputs.py b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_structured_outputs.py index d5298b47..85abc911 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_structured_outputs.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_structured_outputs.py @@ -176,4 +176,7 @@ def test_structured_output_404( assert_all_attributes( spans[0], llm_model_value, latest_experimental_enabled ) - assert "NotFoundError" == spans[0].attributes[ErrorAttributes.ERROR_TYPE] + assert ( + "openai.NotFoundError" + == spans[0].attributes[ErrorAttributes.ERROR_TYPE] + ) diff --git a/util/opentelemetry-util-genai/.changelog/282.changed b/util/opentelemetry-util-genai/.changelog/282.changed new file mode 100644 index 00000000..cbc782d5 --- /dev/null +++ b/util/opentelemetry-util-genai/.changelog/282.changed @@ -0,0 +1 @@ +Set `error.type` to the canonical, fully qualified exception name. diff --git a/util/opentelemetry-util-genai/pyproject.toml b/util/opentelemetry-util-genai/pyproject.toml index 2dd40308..6e12b691 100644 --- a/util/opentelemetry-util-genai/pyproject.toml +++ b/util/opentelemetry-util-genai/pyproject.toml @@ -28,7 +28,7 @@ dependencies = [ "opentelemetry-instrumentation >= 0.64b0, <1", "opentelemetry-semantic-conventions >= 0.64b0, <1", "opentelemetry-api ~= 1.43", - "wrapt >= 1.0.0, < 3.0.0", + "wrapt >= 1.17.0, < 3.0.0", ] [project.entry-points.opentelemetry_genai_completion_hook] diff --git a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_invocation.py b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_invocation.py index 5fd921fc..21f5d846 100644 --- a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_invocation.py +++ b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_invocation.py @@ -112,10 +112,9 @@ def _get_metric_token_counts(self) -> dict[str, int]: # pylint: disable=no-self def _apply_error_attributes(self, error: Error) -> None: """Apply error status and error.type attribute to the span, events, and metrics.""" - error_type = error.type.__qualname__ self.span.set_status(Status(StatusCode.ERROR, error.message)) - self.attributes[error_attributes.ERROR_TYPE] = error_type - self.metric_attributes[error_attributes.ERROR_TYPE] = error_type + self.attributes[error_attributes.ERROR_TYPE] = error.type + self.metric_attributes[error_attributes.ERROR_TYPE] = error.type def _call_completion_hook( self, @@ -164,7 +163,7 @@ def stop(self) -> None: def fail(self, error: Error | BaseException) -> None: """Fail the invocation and end its span with error status.""" if isinstance(error, BaseException): - error = Error(type=type(error), message=str(error)) + error = Error.from_exception(error) self._finish(error) def __enter__(self) -> GenAIInvocation: diff --git a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/types.py b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/types.py index 6ec3266c..dd25b8a1 100644 --- a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/types.py +++ b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/types.py @@ -5,7 +5,7 @@ from dataclasses import dataclass from enum import Enum -from typing import TYPE_CHECKING, Any, Literal, Type, Union +from typing import TYPE_CHECKING, Any, Literal, Union if TYPE_CHECKING: from opentelemetry.util.genai._inference_invocation import ( # pylint: disable=useless-import-alias @@ -226,8 +226,27 @@ class OutputMessage: @dataclass class Error: - message: str - type: Type[BaseException] + message: str | None + type: str + """The ``error.type`` attribute value. A short, low-cardinality string (an + exception type name, an error code, etc.) — not necessarily a Python + exception class.""" + exception: BaseException | None = None + """The originating exception, when the error was produced from one; ``None`` + when built from an explicit ``type``/``message``.""" + + @classmethod + def from_exception(cls, exception: BaseException) -> Error: + """Build an ``Error`` from an exception, deriving ``type`` and ``message``.""" + from opentelemetry.util.genai.utils import ( # noqa: PLC0415 # pylint: disable=import-outside-toplevel + fq_exception_type, + ) + + return cls( + message=str(exception), + type=fq_exception_type(exception), + exception=exception, + ) def __getattr__(name: str) -> object: diff --git a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/utils.py b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/utils.py index 7df35b28..a78b46b7 100644 --- a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/utils.py +++ b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/utils.py @@ -85,6 +85,24 @@ def should_capture_content_on_spans() -> bool: ) +def fq_exception_type(exception: BaseException) -> str: + """Return the fully qualified name of an exception's type. + + Matches the ``exception.type`` value the exception event records, so + ``error.type`` and ``exception.type`` stay consistent. Builtins are returned + unqualified (e.g. ``ValueError``, not ``builtins.ValueError``). + """ + # Mirrors the SDK's Span.record_exception so error.type matches the + # exception event's exception.type: + # https://github.com/open-telemetry/opentelemetry-python/blob/main/opentelemetry-sdk/src/opentelemetry/sdk/trace/__init__.py + exc_type = type(exception) + module = exc_type.__module__ + qualname = exc_type.__qualname__ + if module and module != "builtins": + return f"{module}.{qualname}" + return qualname + + class _GenAiJsonEncoder(json.JSONEncoder): def default(self, o: Any) -> Any: if isinstance(o, bytes): diff --git a/util/opentelemetry-util-genai/tests/test_handler_agent.py b/util/opentelemetry-util-genai/tests/test_handler_agent.py index 338786d1..7eacf1e2 100644 --- a/util/opentelemetry-util-genai/tests/test_handler_agent.py +++ b/util/opentelemetry-util-genai/tests/test_handler_agent.py @@ -615,7 +615,7 @@ def test_fail_agent_records_error_metric(self) -> None: invocation = handler.invoke_local_agent(request_model="err-model") invocation.input_tokens = 11 - error = Error(message="boom", type=ValueError) + error = Error(message="boom", type="ValueError") with patch("timeit.default_timer", return_value=2001.0): invocation.fail(error) diff --git a/util/opentelemetry-util-genai/tests/test_handler_metrics.py b/util/opentelemetry-util-genai/tests/test_handler_metrics.py index d5eaa9b0..0eb49a31 100644 --- a/util/opentelemetry-util-genai/tests/test_handler_metrics.py +++ b/util/opentelemetry-util-genai/tests/test_handler_metrics.py @@ -124,7 +124,7 @@ def test_fail_llm_records_error_and_available_tokens(self) -> None: invocation = handler.inference("", request_model="err-model") invocation.input_tokens = 11 - error = Error(message="boom", type=ValueError) + error = Error(message="boom", type="ValueError") with patch( "timeit.default_timer", return_value=2001.0, @@ -283,7 +283,7 @@ def test_fail_embedding_records_error_and_duration(self) -> None: "embed-prov", request_model="embed-err-model" ) - error = Error(message="embedding failed", type=RuntimeError) + error = Error(message="embedding failed", type="RuntimeError") with patch("timeit.default_timer", return_value=3002.5): invocation.fail(error) @@ -373,7 +373,7 @@ def test_fail_tool_records_duration_with_error(self) -> None: with patch("timeit.default_timer", return_value=500.0): invocation = handler.tool("failing_tool") - error = Error(message="Tool execution failed", type=RuntimeError) + error = Error(message="Tool execution failed", type="RuntimeError") with patch("timeit.default_timer", return_value=501.5): invocation.fail(error) @@ -500,7 +500,7 @@ def test_fail_retrieval_records_duration_with_error(self) -> None: with patch("timeit.default_timer", return_value=2000.0): invocation = handler.retrieval(provider="pinecone") - error = Error(message="retrieval failed", type=ConnectionError) + error = Error(message="retrieval failed", type="ConnectionError") with patch("timeit.default_timer", return_value=2003.0): invocation.fail(error) diff --git a/util/opentelemetry-util-genai/tests/test_handler_retrieval.py b/util/opentelemetry-util-genai/tests/test_handler_retrieval.py index 06835b35..eed3b050 100644 --- a/util/opentelemetry-util-genai/tests/test_handler_retrieval.py +++ b/util/opentelemetry-util-genai/tests/test_handler_retrieval.py @@ -232,7 +232,7 @@ def test_stop_omits_none_attributes(self) -> None: def test_fail_sets_error_status(self) -> None: invocation = self.handler.retrieval() - invocation.fail(Error(message="timeout", type=TimeoutError)) + invocation.fail(Error(message="timeout", type="TimeoutError")) spans = self._get_finished_spans() self.assertEqual(spans[0].status.status_code, StatusCode.ERROR) @@ -240,14 +240,14 @@ def test_fail_sets_error_status(self) -> None: def test_fail_sets_error_type_attribute(self) -> None: invocation = self.handler.retrieval() - invocation.fail(Error(message="bad", type=ConnectionError)) + invocation.fail(Error(message="bad", type="ConnectionError")) spans = self._get_finished_spans() self.assertEqual(spans[0].attributes["error.type"], "ConnectionError") def test_fail_sets_operation_name(self) -> None: invocation = self.handler.retrieval() - invocation.fail(Error(message="err", type=RuntimeError)) + invocation.fail(Error(message="err", type="RuntimeError")) spans = self._get_finished_spans() self.assertEqual( diff --git a/util/opentelemetry-util-genai/tests/test_handler_workflow.py b/util/opentelemetry-util-genai/tests/test_handler_workflow.py index a304f5b8..e4993163 100644 --- a/util/opentelemetry-util-genai/tests/test_handler_workflow.py +++ b/util/opentelemetry-util-genai/tests/test_handler_workflow.py @@ -127,7 +127,7 @@ def test_stop_workflow_returns_invocation(self) -> None: def test_fail_workflow_sets_error_status(self) -> None: invocation = self.handler.workflow(name="wf") - error = Error(message="something broke", type=RuntimeError) + error = Error(message="something broke", type="RuntimeError") invocation.fail(error) spans = self._get_finished_spans() @@ -137,7 +137,7 @@ def test_fail_workflow_sets_error_status(self) -> None: def test_fail_workflow_sets_error_type_attribute(self) -> None: invocation = self.handler.workflow(name="wf") - error = Error(message="bad", type=ValueError) + error = Error(message="bad", type="ValueError") invocation.fail(error) spans = self._get_finished_spans() @@ -145,7 +145,7 @@ def test_fail_workflow_sets_error_type_attribute(self) -> None: def test_fail_workflow_sets_operation_name_attribute(self) -> None: invocation = self.handler.workflow(name="wf") - error = Error(message="fail", type=TypeError) + error = Error(message="fail", type="TypeError") invocation.fail(error) spans = self._get_finished_spans() @@ -156,7 +156,7 @@ def test_fail_workflow_sets_operation_name_attribute(self) -> None: def test_fail_workflow_ends_span(self) -> None: invocation = self.handler.workflow(name="wf") - invocation.fail(Error(message="err", type=RuntimeError)) + invocation.fail(Error(message="err", type="RuntimeError")) spans = self._get_finished_spans() self.assertEqual(len(spans), 1) self.assertEqual(spans[0].status.status_code, StatusCode.ERROR) diff --git a/util/opentelemetry-util-genai/tests/test_utils.py b/util/opentelemetry-util-genai/tests/test_utils.py index 0477ee92..43bf19aa 100644 --- a/util/opentelemetry-util-genai/tests/test_utils.py +++ b/util/opentelemetry-util-genai/tests/test_utils.py @@ -809,7 +809,7 @@ class BoomError(RuntimeError): GenAI.GEN_AI_RESPONSE_ID: "error-response", GenAI.GEN_AI_USAGE_INPUT_TOKENS: 11, GenAI.GEN_AI_USAGE_OUTPUT_TOKENS: 22, - error_attributes.ERROR_TYPE: BoomError.__qualname__, + error_attributes.ERROR_TYPE: f"{BoomError.__module__}.{BoomError.__qualname__}", }, ) @@ -848,7 +848,7 @@ class BoomError(RuntimeError): server_attributes.SERVER_ADDRESS: "embed.example.com", server_attributes.SERVER_PORT: 443, "custom_embed_attr": "value", - error_attributes.ERROR_TYPE: BoomError.__qualname__, + error_attributes.ERROR_TYPE: f"{BoomError.__module__}.{BoomError.__qualname__}", }, ) @@ -911,7 +911,7 @@ class BoomError(RuntimeError): assert span.status.description == "boom" assert ( _get_span_attributes(span)[error_attributes.ERROR_TYPE] - == BoomError.__qualname__ + == f"{BoomError.__module__}.{BoomError.__qualname__}" ) diff --git a/util/opentelemetry-util-genai/tests/test_utils_events.py b/util/opentelemetry-util-genai/tests/test_utils_events.py index 11aff49d..dc601d38 100644 --- a/util/opentelemetry-util-genai/tests/test_utils_events.py +++ b/util/opentelemetry-util-genai/tests/test_utils_events.py @@ -211,7 +211,7 @@ class TestError(RuntimeError): "test-provider", request_model="error-model" ) invocation.input_messages = [message] - error = Error(message="Test error occurred", type=TestError) + error = Error(message="Test error occurred", type="TestError") invocation.fail(error) # Check event was emitted @@ -221,9 +221,7 @@ class TestError(RuntimeError): attrs = log_record.attributes # Verify error attribute is present - self.assertEqual( - attrs[error_attributes.ERROR_TYPE], TestError.__qualname__ - ) + self.assertEqual(attrs[error_attributes.ERROR_TYPE], "TestError") self.assertEqual(attrs[GenAI.GEN_AI_OPERATION_NAME], "chat") self.assertEqual(attrs[GenAI.GEN_AI_REQUEST_MODEL], "error-model") # Verify event context matches span context diff --git a/uv.lock b/uv.lock index 94f9b8f1..60d40006 100644 --- a/uv.lock +++ b/uv.lock @@ -1465,7 +1465,7 @@ requires-dist = [ { name = "opentelemetry-instrumentation", specifier = ">=0.64b0,<1" }, { name = "opentelemetry-semantic-conventions", specifier = ">=0.64b0,<1" }, { name = "pytest", marker = "extra == 'test'", specifier = ">=7.0.0" }, - { name = "wrapt", specifier = ">=1.0.0,<3.0.0" }, + { name = "wrapt", specifier = ">=1.17.0,<3.0.0" }, ] provides-extras = ["test", "upload"]