diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/.changelog/188.added b/instrumentation/opentelemetry-instrumentation-genai-langchain/.changelog/188.added new file mode 100644 index 00000000..5e455cb3 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/.changelog/188.added @@ -0,0 +1,2 @@ +Add ChatAnthropic tool-calling test coverage and fix finish_reason extraction for +Anthropic responses in LangChain instrumentation. \ No newline at end of file diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py index 1ef79105..646b6faa 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py @@ -236,12 +236,19 @@ def on_chat_model_start( frequency_penalty = params.get("frequency_penalty") presence_penalty = params.get("presence_penalty") stop_sequences = params.get("stop") + if stop_sequences is None: + stop_sequences = params.get("stop_sequences") + if stop_sequences is None: + serialized_kwargs: dict[str, Any] = ( + serialized.get("kwargs") or {} + ) + stop_sequences = serialized_kwargs.get("stop_sequences") seed = params.get("seed") temperature = params.get("temperature") - # ``max_completion_tokens`` is OpenAI-specific; fall back to the - # generic ``max_tokens`` used by Anthropic, Mistral, Cohere, etc. - max_tokens = params.get("max_completion_tokens") or params.get( - "max_tokens" + max_tokens = ( + params.get("max_completion_tokens") + if params.get("max_completion_tokens") is not None + else params.get("max_tokens") ) provider = normalize_provider(metadata) or "unknown" @@ -327,7 +334,10 @@ def on_llm_end( ): finish_reason = ( chat_generation.message.response_metadata.get( - "stopReason", "unknown" + "stopReason" + ) + or chat_generation.message.response_metadata.get( + "stop_reason", "unknown" ) ) diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/cassettes/test_chat_anthropic_claude_sonnet_llm_call.yaml b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/cassettes/test_chat_anthropic_claude_sonnet_llm_call.yaml new file mode 100644 index 00000000..68f62393 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/cassettes/test_chat_anthropic_claude_sonnet_llm_call.yaml @@ -0,0 +1,46 @@ +interactions: +- request: + body: |- + {"model": "claude-sonnet-4-5", "max_tokens": 1024, "temperature": 0.1, "system": "You are a helpful assistant!", "messages": [{"role": "user", "content": "What is the capital of France?"}]} + headers: + Content-Type: + - application/json + User-Agent: + - !!binary | + QW50aHJvcGljL1B5dGhvbiAxLjAuMA== + x-api-key: + - test_key + anthropic-version: + - '2023-06-01' + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: |- + { + "id": "msg_01XFDUDYJgAACzvnptvVoYEL", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-5", + "content": [ + { + "type": "text", + "text": "The capital of France is Paris." + } + ], + "stop_reason": "end_turn", + "stop_sequence": null, + "usage": { + "input_tokens": 13, + "output_tokens": 10 + } + } + headers: + Content-Type: + - application/json + Date: + - Thu, 04 Sep 2025 20:00:58 GMT + status: + code: 200 + message: OK +version: 1 \ No newline at end of file diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/cassettes/test_chat_anthropic_claude_sonnet_stop_sequences_constructor_fallback.yaml b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/cassettes/test_chat_anthropic_claude_sonnet_stop_sequences_constructor_fallback.yaml new file mode 100644 index 00000000..97cfe6b8 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/cassettes/test_chat_anthropic_claude_sonnet_stop_sequences_constructor_fallback.yaml @@ -0,0 +1,44 @@ +# TODO: this is generated by AI, re-record +interactions: +- request: + body: |- + {"max_tokens":1024,"messages":[{"role":"user","content":"Say hi"}],"model":"claude-sonnet-4-5","stop_sequences":["STOP"]} + headers: + Content-Type: + - application/json + x-api-key: + - test_key + anthropic-version: + - '2023-06-01' + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: |- + { + "id": "msg_anthropic_constructor_stop_sequences_001", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-5", + "content": [ + { + "type": "text", + "text": "Hi there!" + } + ], + "stop_reason": "stop_sequence", + "stop_sequence": "STOP", + "usage": { + "input_tokens": 10, + "output_tokens": 5 + } + } + headers: + Content-Type: + - application/json + Date: + - Thu, 04 Sep 2025 20:00:58 GMT + status: + code: 200 + message: OK +version: 1 \ No newline at end of file diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/cassettes/test_chat_anthropic_claude_sonnet_stop_sequences_fallback.yaml b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/cassettes/test_chat_anthropic_claude_sonnet_stop_sequences_fallback.yaml new file mode 100644 index 00000000..62cf46c6 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/cassettes/test_chat_anthropic_claude_sonnet_stop_sequences_fallback.yaml @@ -0,0 +1,55 @@ +# TODO: this is generated by AI, re-record +interactions: +- request: + body: |- + { + "model": "claude-sonnet-4-5", + "max_tokens": 1024, + "temperature": 0.1, + "stop_sequences": ["STOP"], + "messages": [ + { + "role": "user", + "content": "Say hi" + } + ] + } + headers: + Content-Type: + - application/json + x-api-key: + - test_key + anthropic-version: + - '2023-06-01' + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: |- + { + "id": "msg_anthropic_stop_sequences_001", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-5", + "content": [ + { + "type": "text", + "text": "Hi there!" + } + ], + "stop_reason": "stop_sequence", + "stop_sequence": "STOP", + "usage": { + "input_tokens": 10, + "output_tokens": 5 + } + } + headers: + Content-Type: + - application/json + Date: + - Thu, 04 Sep 2025 20:00:58 GMT + status: + code: 200 + message: OK +version: 1 \ No newline at end of file diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/cassettes/test_chat_anthropic_claude_sonnet_tool_call.yaml b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/cassettes/test_chat_anthropic_claude_sonnet_tool_call.yaml new file mode 100644 index 00000000..bffbe782 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/cassettes/test_chat_anthropic_claude_sonnet_tool_call.yaml @@ -0,0 +1,48 @@ +# TODO: this is generated by AI, re-record +interactions: +- request: + body: |- + {"model":"claude-sonnet-4-5","max_tokens":1024,"messages":[{"role":"user","content":"What's the weather in Seattle?"}],"temperature":0.1,"tools":[{"name":"get_current_weather","input_schema":{"properties":{"location":{"type":"string"}},"required":["location"],"type":"object"},"description":"Get the current weather in a given location."}]} + headers: + Content-Type: + - application/json + x-api-key: + - test_key + anthropic-version: + - '2023-06-01' + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: |- + { + "id": "msg_anthropic_tool_call_001", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-5", + "content": [ + { + "type": "tool_use", + "id": "toolu_01", + "name": "get_current_weather", + "input": { + "location": "Seattle, WA" + } + } + ], + "stop_reason": "tool_use", + "stop_sequence": null, + "usage": { + "input_tokens": 20, + "output_tokens": 15 + } + } + headers: + Content-Type: + - application/json + Date: + - Thu, 04 Sep 2025 20:00:58 GMT + status: + code: 200 + message: OK +version: 1 \ No newline at end of file diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/conftest.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/conftest.py index 93bf04a4..d2c4ce04 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/conftest.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/conftest.py @@ -7,6 +7,7 @@ import boto3 import pytest +from langchain_anthropic import ChatAnthropic from langchain_aws import ChatBedrock from langchain_google_genai import ChatGoogleGenerativeAI from langchain_openai import ChatOpenAI @@ -65,6 +66,18 @@ def fixture_us_amazon_nova_lite_v1_0(): yield llm +@pytest.fixture(scope="function", name="chat_anthropic_claude_sonnet") +def fixture_chat_anthropic_claude_sonnet(): + llm_model_value = "claude-sonnet-4-5" + llm = ChatAnthropic( + model=llm_model_value, + api_key="test_key", + temperature=0.1, + max_tokens=1024, + ) + yield llm + + @pytest.fixture(scope="function", name="gemini") def fixture_gemini(): llm_model_value = "gemini-2.5-pro" diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/requirements.latest.txt b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/requirements.latest.txt index c1be3221..72200f6e 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/requirements.latest.txt +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/requirements.latest.txt @@ -18,7 +18,8 @@ langchain==1.3.14 langchain-openai==1.3.5 langchain-aws==1.6.2 langchain-google-genai==4.2.7 +langchain-anthropic==1.4.8 boto3==1.43.53 -e util/opentelemetry-util-genai --e instrumentation/opentelemetry-instrumentation-genai-langchain[instruments] +-e instrumentation/opentelemetry-instrumentation-genai-langchain[instruments] \ No newline at end of file diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/requirements.oldest.txt b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/requirements.oldest.txt index 7b1e5347..5fcdee5b 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/requirements.oldest.txt +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/requirements.oldest.txt @@ -23,4 +23,5 @@ langchain-openai==0.2.0 langchain-aws==0.2.2 langchain-google-genai==2.0.0 -boto3==1.37.0 +langchain-anthropic==0.3.0 +boto3==1.37.0 \ No newline at end of file diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_llm_call.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_llm_call.py index ebd585dc..1519aec8 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_llm_call.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_llm_call.py @@ -5,11 +5,13 @@ from typing import Optional import pytest +from langchain_anthropic import ChatAnthropic from langchain_core.messages import ( FunctionMessage, HumanMessage, SystemMessage, ) +from langchain_core.tools import tool from openai import AuthenticationError from opentelemetry.instrumentation.genai.langchain.utils import ( @@ -214,6 +216,66 @@ def test_us_amazon_nova_lite_v1_0_bedrock_llm_call( assert_bedrock_completion_attributes(spans[0], result) +@pytest.mark.vcr() +def test_chat_anthropic_claude_sonnet_llm_call( + span_exporter, start_instrumentation, chat_anthropic_claude_sonnet +): + messages = [ + SystemMessage(content="You are a helpful assistant!"), + HumanMessage(content="What is the capital of France?"), + ] + + result = chat_anthropic_claude_sonnet.invoke(messages) + + assert result.content.find("The capital of France is Paris") != -1 + + # verify spans + spans = span_exporter.get_finished_spans() + assert len(spans) == 1 + span = spans[0] + assert ( + span.attributes.get(gen_ai_attributes.GEN_AI_REQUEST_MODEL) + == "claude-sonnet-4-5" + ) + + assert ( + span.attributes.get(gen_ai_attributes.GEN_AI_REQUEST_MAX_TOKENS) + == 1024 + ) + + +@pytest.mark.vcr() +def test_chat_anthropic_claude_sonnet_tool_call( + span_exporter, start_instrumentation, chat_anthropic_claude_sonnet +): + @tool + def get_current_weather(location: str) -> str: + """Get the current weather in a given location.""" + return f"The weather in {location} is sunny." + + llm_with_tools = chat_anthropic_claude_sonnet.bind_tools( + [get_current_weather] + ) + + messages = [ + HumanMessage(content="What's the weather in Seattle?"), + ] + + llm_with_tools.invoke(messages) + + # verify spans + spans = span_exporter.get_finished_spans() + assert len(spans) == 1 + span = spans[0] + assert ( + span.attributes.get(gen_ai_attributes.GEN_AI_REQUEST_MODEL) + == "claude-sonnet-4-5" + ) + assert span.attributes.get( + gen_ai_attributes.GEN_AI_RESPONSE_FINISH_REASONS + ) == ("tool_use",) + + # span_exporter, start_instrumentation, gemini are coming from fixtures defined in conftest.py def test_gemini(span_exporter, start_instrumentation, gemini, vcr): messages = [ @@ -705,3 +767,40 @@ def assert_log_parent(log_record, span): assert log_record.trace_id == span.get_span_context().trace_id assert log_record.span_id == span.get_span_context().span_id assert log_record.trace_flags == span.get_span_context().trace_flags + + +@pytest.mark.vcr() +def test_chat_anthropic_claude_sonnet_stop_sequences_fallback( + span_exporter, start_instrumentation, chat_anthropic_claude_sonnet +): + llm = chat_anthropic_claude_sonnet.bind(stop_sequences=["STOP"]) + llm.invoke([HumanMessage(content="Say hi")]) + + spans = span_exporter.get_finished_spans() + assert len(spans) == 1 + span = spans[0] + stop_sequences = span.attributes.get( + gen_ai_attributes.GEN_AI_REQUEST_STOP_SEQUENCES + ) + assert stop_sequences == ("STOP",) + + +@pytest.mark.vcr() +def test_chat_anthropic_claude_sonnet_stop_sequences_constructor_fallback( + span_exporter, start_instrumentation +): + model = ChatAnthropic( + model="claude-sonnet-4-5", + api_key="test_key", + max_tokens=1024, + stop_sequences=["STOP"], + ) + model.invoke([HumanMessage(content="Say hi")]) + + spans = span_exporter.get_finished_spans() + assert len(spans) == 1 + span = spans[0] + stop_sequences = span.attributes.get( + gen_ai_attributes.GEN_AI_REQUEST_STOP_SEQUENCES + ) + assert stop_sequences == ("STOP",) diff --git a/tox.ini b/tox.ini index ac839f6e..53ce2c77 100644 --- a/tox.ini +++ b/tox.ini @@ -146,6 +146,7 @@ deps = langchain-conformance: -e {toxinidir}/util/opentelemetry-util-genai langchain-conformance: -e {toxinidir}/instrumentation/opentelemetry-instrumentation-genai-langchain[instruments] langchain-conformance: langchain-openai + langchain-conformance: langchain-anthropic langchain-conformance: langchain-aws langchain-conformance: langchain-google-genai langchain-conformance: boto3