From 1f32952d0066a9dc1ff1482cef48c3cbe0acb663 Mon Sep 17 00:00:00 2001 From: Fabian Schindler Date: Wed, 17 Dec 2025 10:45:45 +0100 Subject: [PATCH 01/12] fix(ai): redact message parts content of type blob --- sentry_sdk/ai/utils.py | 51 +++++++++++++++++ tests/test_ai_monitoring.py | 106 +++++++++++++++++++++++++++++++++++- 2 files changed, 156 insertions(+), 1 deletion(-) diff --git a/sentry_sdk/ai/utils.py b/sentry_sdk/ai/utils.py index 1d2b4483c9..73155b0305 100644 --- a/sentry_sdk/ai/utils.py +++ b/sentry_sdk/ai/utils.py @@ -5,6 +5,8 @@ from sys import getsizeof from typing import TYPE_CHECKING +from sentry_sdk._types import SENSITIVE_DATA_SUBSTITUTE + if TYPE_CHECKING: from typing import Any, Callable, Dict, List, Optional, Tuple @@ -141,6 +143,53 @@ def _find_truncation_index(messages: "List[Dict[str, Any]]", max_bytes: int) -> return 0 +def redact_blob_message_parts(messages): + # type: (List[Dict[str, Any]]) -> Tuple[List[Dict[str, Any]], int] + """ + Redact blob message parts from the messages, by removing the "content" key. + e.g: + { + "role": "user", + "content": [ + { + "text": "How many ponies do you see in the image?", + "type": "text" + }, + { + "type": "blob", + "modality": "image", + "mime_type": "image/jpeg", + "content": "data:image/jpeg;base64,..." + } + ] + } + becomes: + { + "role": "user", + "content": [ + { + "text": "How many ponies do you see in the image?", + "type": "text" + }, + { + "type": "blob", + "modality": "image", + "mime_type": "image/jpeg", + "content": "[Filtered]" + } + ] + } + """ + + for message in messages: + content = message.get("content") + if isinstance(content, list): + for item in content: + if item.get("type") == "blob": + item["content"] = SENSITIVE_DATA_SUBSTITUTE + return messages + + def truncate_messages_by_size( messages: "List[Dict[str, Any]]", max_bytes: int = MAX_GEN_AI_MESSAGE_BYTES, @@ -186,6 +235,8 @@ def truncate_and_annotate_messages( if not messages: return None + messages = redact_blob_message_parts(messages) + truncated_messages, removed_count = truncate_messages_by_size(messages, max_bytes) if removed_count > 0: scope._gen_ai_original_message_count[span.span_id] = len(messages) diff --git a/tests/test_ai_monitoring.py b/tests/test_ai_monitoring.py index 8d3d4ba204..e9f3712cd3 100644 --- a/tests/test_ai_monitoring.py +++ b/tests/test_ai_monitoring.py @@ -4,7 +4,7 @@ import pytest import sentry_sdk -from sentry_sdk._types import AnnotatedValue +from sentry_sdk._types import AnnotatedValue, SENSITIVE_DATA_SUBSTITUTE from sentry_sdk.ai.monitoring import ai_track from sentry_sdk.ai.utils import ( MAX_GEN_AI_MESSAGE_BYTES, @@ -13,6 +13,7 @@ truncate_and_annotate_messages, truncate_messages_by_size, _find_truncation_index, + redact_blob_message_parts, ) from sentry_sdk.serializer import serialize from sentry_sdk.utils import safe_serialize @@ -542,3 +543,106 @@ def __init__(self): assert isinstance(messages_value, AnnotatedValue) assert messages_value.metadata["len"] == stored_original_length assert len(messages_value.value) == len(truncated_messages) + + +class TestRedactBlobMessageParts: + def test_redacts_single_blob_content(self): + """Test that blob content is redacted in a message with single blob part""" + messages = [ + { + "role": "user", + "content": [ + { + "text": "How many ponies do you see in the image?", + "type": "text", + }, + { + "type": "blob", + "modality": "image", + "mime_type": "image/jpeg", + "content": "data:image/jpeg;base64,/9j/4AAQSkZJRg==", + }, + ], + } + ] + + result = redact_blob_message_parts(messages) + + assert result == messages # Returns the same list + assert ( + messages[0]["content"][0]["text"] + == "How many ponies do you see in the image?" + ) + assert messages[0]["content"][0]["type"] == "text" + assert messages[0]["content"][1]["type"] == "blob" + assert messages[0]["content"][1]["modality"] == "image" + assert messages[0]["content"][1]["mime_type"] == "image/jpeg" + assert messages[0]["content"][1]["content"] == SENSITIVE_DATA_SUBSTITUTE + + def test_redacts_multiple_blob_parts(self): + """Test that multiple blob parts in a single message are all redacted""" + messages = [ + { + "role": "user", + "content": [ + {"text": "Compare these images", "type": "text"}, + { + "type": "blob", + "modality": "image", + "mime_type": "image/jpeg", + "content": "data:image/jpeg;base64,first_image", + }, + { + "type": "blob", + "modality": "image", + "mime_type": "image/png", + "content": "data:image/png;base64,second_image", + }, + ], + } + ] + + result = redact_blob_message_parts(messages) + + assert result == messages + assert messages[0]["content"][0]["text"] == "Compare these images" + assert messages[0]["content"][1]["content"] == SENSITIVE_DATA_SUBSTITUTE + assert messages[0]["content"][2]["content"] == SENSITIVE_DATA_SUBSTITUTE + + def test_redacts_blobs_in_multiple_messages(self): + """Test that blob parts are redacted across multiple messages""" + messages = [ + { + "role": "user", + "content": [ + {"text": "First message", "type": "text"}, + { + "type": "blob", + "modality": "image", + "content": "data:image/jpeg;base64,first", + }, + ], + }, + { + "role": "assistant", + "content": "I see the image.", + }, + { + "role": "user", + "content": [ + {"text": "Second message", "type": "text"}, + { + "type": "blob", + "modality": "image", + "content": "data:image/jpeg;base64,second", + }, + ], + }, + ] + + result = redact_blob_message_parts(messages) + + assert result == messages + assert messages[0]["content"][1]["content"] == SENSITIVE_DATA_SUBSTITUTE + assert messages[1]["content"] == "I see the image." # Unchanged + assert messages[2]["content"][1]["content"] == SENSITIVE_DATA_SUBSTITUTE From 795bcea241f7777e646a4da14c870a3049bdbe90 Mon Sep 17 00:00:00 2001 From: Fabian Schindler Date: Wed, 17 Dec 2025 11:05:04 +0100 Subject: [PATCH 02/12] fix(ai): skip non dict messages --- sentry_sdk/ai/utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sentry_sdk/ai/utils.py b/sentry_sdk/ai/utils.py index 73155b0305..ae507e898b 100644 --- a/sentry_sdk/ai/utils.py +++ b/sentry_sdk/ai/utils.py @@ -182,6 +182,9 @@ def redact_blob_message_parts(messages): """ for message in messages: + if not isinstance(message, dict): + continue + content = message.get("content") if isinstance(content, list): for item in content: From a623e137d26e982c0d85258256c0ba013f9ecb24 Mon Sep 17 00:00:00 2001 From: Fabian Schindler Date: Wed, 17 Dec 2025 11:21:43 +0100 Subject: [PATCH 03/12] fix(ai): typing --- sentry_sdk/ai/utils.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sentry_sdk/ai/utils.py b/sentry_sdk/ai/utils.py index ae507e898b..1b61c7a113 100644 --- a/sentry_sdk/ai/utils.py +++ b/sentry_sdk/ai/utils.py @@ -143,8 +143,9 @@ def _find_truncation_index(messages: "List[Dict[str, Any]]", max_bytes: int) -> return 0 -def redact_blob_message_parts(messages): - # type: (List[Dict[str, Any]]) -> Tuple[List[Dict[str, Any]], int] +def redact_blob_message_parts( + messages: "List[Dict[str, Any]]", +) -> "List[Dict[str, Any]]": """ Redact blob message parts from the messages, by removing the "content" key. e.g: From 3d3ce5bbdca43f14194edbbbee11d3b6dcd6d8a3 Mon Sep 17 00:00:00 2001 From: Fabian Schindler Date: Wed, 17 Dec 2025 11:37:12 +0100 Subject: [PATCH 04/12] fix(ai): content items may not be dicts --- sentry_sdk/ai/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sentry_sdk/ai/utils.py b/sentry_sdk/ai/utils.py index 1b61c7a113..78a64ab737 100644 --- a/sentry_sdk/ai/utils.py +++ b/sentry_sdk/ai/utils.py @@ -189,7 +189,7 @@ def redact_blob_message_parts( content = message.get("content") if isinstance(content, list): for item in content: - if item.get("type") == "blob": + if isinstance(item, dict) and item.get("type") == "blob": item["content"] = SENSITIVE_DATA_SUBSTITUTE return messages From c606b66f1dbe62f3235f0b501c9250ba2b54632a Mon Sep 17 00:00:00 2001 From: Fabian Schindler Date: Mon, 5 Jan 2026 20:15:27 +0100 Subject: [PATCH 05/12] fix(integrations): langchain add multimodal content transformation functions for images, audio, and files --- sentry_sdk/integrations/langchain.py | 122 ++++++++- .../integrations/langchain/test_langchain.py | 242 ++++++++++++++++++ 2 files changed, 363 insertions(+), 1 deletion(-) diff --git a/sentry_sdk/integrations/langchain.py b/sentry_sdk/integrations/langchain.py index 950f437d4c..51cce8942d 100644 --- a/sentry_sdk/integrations/langchain.py +++ b/sentry_sdk/integrations/langchain.py @@ -116,6 +116,124 @@ "top_p": SPANDATA.GEN_AI_REQUEST_TOP_P, } +# Map LangChain content types to Sentry modalities +LANGCHAIN_TYPE_TO_MODALITY = { + "image": "image", + "image_url": "image", + "audio": "audio", + "video": "video", + "file": "document", +} + + +def _transform_langchain_content_block( + content_block: "Dict[str, Any]", +) -> "Dict[str, Any]": + """ + Transform a LangChain content block to Sentry-compatible format. + + Handles multimodal content (images, audio, video, documents) by converting them + to the standardized format: + - base64 encoded data -> type: "blob" + - URL references -> type: "uri" + - file_id references -> type: "file" + """ + if not isinstance(content_block, dict): + return content_block + + block_type = content_block.get("type") + + # Handle standard multimodal content types (image, audio, video, file) + if block_type in ("image", "audio", "video", "file"): + modality = LANGCHAIN_TYPE_TO_MODALITY.get(block_type, block_type) + mime_type = content_block.get("mime_type", "") + + # Check for base64 encoded content + if "base64" in content_block: + return { + "type": "blob", + "modality": modality, + "mime_type": mime_type, + "content": content_block.get("base64", ""), + } + # Check for URL reference + elif "url" in content_block: + return { + "type": "uri", + "modality": modality, + "mime_type": mime_type, + "uri": content_block.get("url", ""), + } + # Check for file_id reference + elif "file_id" in content_block: + return { + "type": "file", + "modality": modality, + "mime_type": mime_type, + "file_id": content_block.get("file_id", ""), + } + + # Handle legacy image_url format (OpenAI style) + elif block_type == "image_url": + image_url_data = content_block.get("image_url", {}) + if isinstance(image_url_data, dict): + url = image_url_data.get("url", "") + else: + url = str(image_url_data) + + # Check if it's a data URI (base64 encoded) + if url.startswith("data:"): + # Parse data URI: data:mime_type;base64,content + try: + # Format: data:image/jpeg;base64,/9j/4AAQ... + header, content = url.split(",", 1) + mime_type = header.split(":")[1].split(";")[0] if ":" in header else "" + return { + "type": "blob", + "modality": "image", + "mime_type": mime_type, + "content": content, + } + except (ValueError, IndexError): + # If parsing fails, return as URI + return { + "type": "uri", + "modality": "image", + "mime_type": "", + "uri": url, + } + else: + # Regular URL + return { + "type": "uri", + "modality": "image", + "mime_type": "", + "uri": url, + } + + # For text blocks and other types, return as-is + return content_block + + +def _transform_langchain_message_content(content: "Any") -> "Any": + """ + Transform LangChain message content, handling both string content and + list of content blocks. + """ + if isinstance(content, str): + return content + + if isinstance(content, (list, tuple)): + transformed = [] + for block in content: + if isinstance(block, dict): + transformed.append(_transform_langchain_content_block(block)) + else: + transformed.append(block) + return transformed + + return content + # Contextvar to track agent names in a stack for re-entrant agent support _agent_stack: "contextvars.ContextVar[Optional[List[Optional[str]]]]" = ( @@ -234,7 +352,9 @@ def _handle_error(self, run_id: "UUID", error: "Any") -> None: del self.span_map[run_id] def _normalize_langchain_message(self, message: "BaseMessage") -> "Any": - parsed = {"role": message.type, "content": message.content} + # Transform content to handle multimodal data (images, audio, video, files) + transformed_content = _transform_langchain_message_content(message.content) + parsed = {"role": message.type, "content": transformed_content} parsed.update(message.additional_kwargs) return parsed diff --git a/tests/integrations/langchain/test_langchain.py b/tests/integrations/langchain/test_langchain.py index 114e819bfb..07a37f2382 100644 --- a/tests/integrations/langchain/test_langchain.py +++ b/tests/integrations/langchain/test_langchain.py @@ -25,6 +25,8 @@ from sentry_sdk.integrations.langchain import ( LangchainIntegration, SentryLangchainCallback, + _transform_langchain_content_block, + _transform_langchain_message_content, ) try: @@ -1747,3 +1749,243 @@ def test_langchain_response_model_extraction( assert llm_span["data"][SPANDATA.GEN_AI_RESPONSE_MODEL] == expected_model else: assert SPANDATA.GEN_AI_RESPONSE_MODEL not in llm_span.get("data", {}) + + +# Tests for multimodal content transformation functions + + +class TestTransformLangchainContentBlock: + """Tests for _transform_langchain_content_block function.""" + + def test_transform_image_base64(self): + """Test transformation of base64-encoded image content.""" + content_block = { + "type": "image", + "base64": "/9j/4AAQSkZJRgABAQAAAQABAAD...", + "mime_type": "image/jpeg", + } + result = _transform_langchain_content_block(content_block) + assert result == { + "type": "blob", + "modality": "image", + "mime_type": "image/jpeg", + "content": "/9j/4AAQSkZJRgABAQAAAQABAAD...", + } + + def test_transform_image_url(self): + """Test transformation of URL-referenced image content.""" + content_block = { + "type": "image", + "url": "https://example.com/image.jpg", + "mime_type": "image/jpeg", + } + result = _transform_langchain_content_block(content_block) + assert result == { + "type": "uri", + "modality": "image", + "mime_type": "image/jpeg", + "uri": "https://example.com/image.jpg", + } + + def test_transform_image_file_id(self): + """Test transformation of file_id-referenced image content.""" + content_block = { + "type": "image", + "file_id": "file-abc123", + "mime_type": "image/png", + } + result = _transform_langchain_content_block(content_block) + assert result == { + "type": "file", + "modality": "image", + "mime_type": "image/png", + "file_id": "file-abc123", + } + + def test_transform_image_url_legacy_with_data_uri(self): + """Test transformation of legacy image_url format with data: URI (base64).""" + content_block = { + "type": "image_url", + "image_url": {"url": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD"}, + } + result = _transform_langchain_content_block(content_block) + assert result == { + "type": "blob", + "modality": "image", + "mime_type": "image/jpeg", + "content": "/9j/4AAQSkZJRgABAQAAAQABAAD", + } + + def test_transform_image_url_legacy_with_http_url(self): + """Test transformation of legacy image_url format with HTTP URL.""" + content_block = { + "type": "image_url", + "image_url": {"url": "https://example.com/image.png"}, + } + result = _transform_langchain_content_block(content_block) + assert result == { + "type": "uri", + "modality": "image", + "mime_type": "", + "uri": "https://example.com/image.png", + } + + def test_transform_image_url_legacy_string_url(self): + """Test transformation of legacy image_url format with string URL.""" + content_block = { + "type": "image_url", + "image_url": "https://example.com/image.gif", + } + result = _transform_langchain_content_block(content_block) + assert result == { + "type": "uri", + "modality": "image", + "mime_type": "", + "uri": "https://example.com/image.gif", + } + + def test_transform_image_url_legacy_data_uri_png(self): + """Test transformation of legacy image_url format with PNG data URI.""" + content_block = { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==" + }, + } + result = _transform_langchain_content_block(content_block) + assert result == { + "type": "blob", + "modality": "image", + "mime_type": "image/png", + "content": "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==", + } + + def test_transform_missing_mime_type(self): + """Test transformation when mime_type is not provided.""" + content_block = { + "type": "image", + "base64": "/9j/4AAQSkZJRgABAQAAAQABAAD...", + } + result = _transform_langchain_content_block(content_block) + assert result == { + "type": "blob", + "modality": "image", + "mime_type": "", + "content": "/9j/4AAQSkZJRgABAQAAAQABAAD...", + } + + +class TestTransformLangchainMessageContent: + """Tests for _transform_langchain_message_content function.""" + + def test_transform_string_content(self): + """Test that string content is returned unchanged.""" + result = _transform_langchain_message_content("Hello, world!") + assert result == "Hello, world!" + + def test_transform_list_with_text_blocks(self): + """Test transformation of list with text blocks (unchanged).""" + content = [ + {"type": "text", "text": "First message"}, + {"type": "text", "text": "Second message"}, + ] + result = _transform_langchain_message_content(content) + assert result == content + + def test_transform_list_with_image_blocks(self): + """Test transformation of list containing image blocks.""" + content = [ + {"type": "text", "text": "Check out this image:"}, + { + "type": "image", + "base64": "/9j/4AAQSkZJRgABAQAAAQABAAD...", + "mime_type": "image/jpeg", + }, + ] + result = _transform_langchain_message_content(content) + assert len(result) == 2 + assert result[0] == {"type": "text", "text": "Check out this image:"} + assert result[1] == { + "type": "blob", + "modality": "image", + "mime_type": "image/jpeg", + "content": "/9j/4AAQSkZJRgABAQAAAQABAAD...", + } + + def test_transform_list_with_mixed_content(self): + """Test transformation of list with mixed content types.""" + content = [ + {"type": "text", "text": "Here are some files:"}, + { + "type": "image", + "url": "https://example.com/image.jpg", + "mime_type": "image/jpeg", + }, + { + "type": "file", + "file_id": "doc-123", + "mime_type": "application/pdf", + }, + {"type": "audio", "base64": "audio_data...", "mime_type": "audio/mp3"}, + ] + result = _transform_langchain_message_content(content) + assert len(result) == 4 + assert result[0] == {"type": "text", "text": "Here are some files:"} + assert result[1] == { + "type": "uri", + "modality": "image", + "mime_type": "image/jpeg", + "uri": "https://example.com/image.jpg", + } + assert result[2] == { + "type": "file", + "modality": "document", + "mime_type": "application/pdf", + "file_id": "doc-123", + } + assert result[3] == { + "type": "blob", + "modality": "audio", + "mime_type": "audio/mp3", + "content": "audio_data...", + } + + def test_transform_list_with_non_dict_items(self): + """Test transformation handles non-dict items in list.""" + content = ["plain string", {"type": "text", "text": "dict text"}] + result = _transform_langchain_message_content(content) + assert result == ["plain string", {"type": "text", "text": "dict text"}] + + def test_transform_tuple_content(self): + """Test transformation of tuple content.""" + content = ( + {"type": "text", "text": "Message"}, + {"type": "image", "base64": "data...", "mime_type": "image/png"}, + ) + result = _transform_langchain_message_content(content) + assert len(result) == 2 + assert result[1] == { + "type": "blob", + "modality": "image", + "mime_type": "image/png", + "content": "data...", + } + + def test_transform_list_with_legacy_image_url(self): + """Test transformation of list containing legacy image_url blocks.""" + content = [ + {"type": "text", "text": "Check this:"}, + { + "type": "image_url", + "image_url": {"url": "data:image/jpeg;base64,/9j/4AAQ..."}, + }, + ] + result = _transform_langchain_message_content(content) + assert len(result) == 2 + assert result[0] == {"type": "text", "text": "Check this:"} + assert result[1] == { + "type": "blob", + "modality": "image", + "mime_type": "image/jpeg", + "content": "/9j/4AAQ...", + } From c650799f0b7de741cd77811732644aaa2d722686 Mon Sep 17 00:00:00 2001 From: Fabian Schindler Date: Thu, 8 Jan 2026 14:22:59 +0100 Subject: [PATCH 06/12] fix(integrations): ensure URL check for data URIs handles empty strings --- sentry_sdk/integrations/langchain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sentry_sdk/integrations/langchain.py b/sentry_sdk/integrations/langchain.py index 51cce8942d..1b9389c23a 100644 --- a/sentry_sdk/integrations/langchain.py +++ b/sentry_sdk/integrations/langchain.py @@ -182,7 +182,7 @@ def _transform_langchain_content_block( url = str(image_url_data) # Check if it's a data URI (base64 encoded) - if url.startswith("data:"): + if url and url.startswith("data:"): # Parse data URI: data:mime_type;base64,content try: # Format: data:image/jpeg;base64,/9j/4AAQ... From 510e2ed206be5a01667bdd03719fa2ee7be45876 Mon Sep 17 00:00:00 2001 From: Fabian Schindler Date: Wed, 14 Jan 2026 14:22:00 +0100 Subject: [PATCH 07/12] fix(integrations): Langchain: Handle Anthropic and Google provider-native content formats --- sentry_sdk/integrations/langchain.py | 49 +++++++++++ .../integrations/langchain/test_langchain.py | 86 +++++++++++++++++++ 2 files changed, 135 insertions(+) diff --git a/sentry_sdk/integrations/langchain.py b/sentry_sdk/integrations/langchain.py index 1b9389c23a..68f5d0ad95 100644 --- a/sentry_sdk/integrations/langchain.py +++ b/sentry_sdk/integrations/langchain.py @@ -137,6 +137,12 @@ def _transform_langchain_content_block( - base64 encoded data -> type: "blob" - URL references -> type: "uri" - file_id references -> type: "file" + + Supports multiple content block formats: + - LangChain standard: type + base64/url/file_id fields + - OpenAI legacy: image_url with nested url field + - Anthropic: type + source dict with type/media_type/data or url + - Google: inline_data or file_data dicts """ if not isinstance(content_block, dict): return content_block @@ -172,6 +178,27 @@ def _transform_langchain_content_block( "mime_type": mime_type, "file_id": content_block.get("file_id", ""), } + # Handle Anthropic-style format with nested "source" dict + elif "source" in content_block: + source = content_block.get("source", {}) + if isinstance(source, dict): + source_type = source.get("type") + media_type = source.get("media_type", "") or mime_type + + if source_type == "base64": + return { + "type": "blob", + "modality": modality, + "mime_type": media_type, + "content": source.get("data", ""), + } + elif source_type == "url": + return { + "type": "uri", + "modality": modality, + "mime_type": media_type, + "uri": source.get("url", ""), + } # Handle legacy image_url format (OpenAI style) elif block_type == "image_url": @@ -211,6 +238,28 @@ def _transform_langchain_content_block( "uri": url, } + # Handle Google-style inline_data format + if "inline_data" in content_block: + inline_data = content_block.get("inline_data", {}) + if isinstance(inline_data, dict): + return { + "type": "blob", + "modality": "image", + "mime_type": inline_data.get("mime_type", ""), + "content": inline_data.get("data", ""), + } + + # Handle Google-style file_data format + if "file_data" in content_block: + file_data = content_block.get("file_data", {}) + if isinstance(file_data, dict): + return { + "type": "uri", + "modality": "image", + "mime_type": file_data.get("mime_type", ""), + "uri": file_data.get("file_uri", ""), + } + # For text blocks and other types, return as-is return content_block diff --git a/tests/integrations/langchain/test_langchain.py b/tests/integrations/langchain/test_langchain.py index 07a37f2382..de5f5841ca 100644 --- a/tests/integrations/langchain/test_langchain.py +++ b/tests/integrations/langchain/test_langchain.py @@ -1874,6 +1874,92 @@ def test_transform_missing_mime_type(self): "content": "/9j/4AAQSkZJRgABAQAAAQABAAD...", } + def test_transform_anthropic_source_base64(self): + """Test transformation of Anthropic-style image with base64 source.""" + content_block = { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": "iVBORw0KGgoAAAANSUhEUgAAAAE...", + }, + } + result = _transform_langchain_content_block(content_block) + assert result == { + "type": "blob", + "modality": "image", + "mime_type": "image/png", + "content": "iVBORw0KGgoAAAANSUhEUgAAAAE...", + } + + def test_transform_anthropic_source_url(self): + """Test transformation of Anthropic-style image with URL source.""" + content_block = { + "type": "image", + "source": { + "type": "url", + "media_type": "image/jpeg", + "url": "https://example.com/image.jpg", + }, + } + result = _transform_langchain_content_block(content_block) + assert result == { + "type": "uri", + "modality": "image", + "mime_type": "image/jpeg", + "uri": "https://example.com/image.jpg", + } + + def test_transform_anthropic_source_without_media_type(self): + """Test transformation of Anthropic-style image without media_type falls back to mime_type.""" + content_block = { + "type": "image", + "mime_type": "image/webp", + "source": { + "type": "base64", + "data": "UklGRh4AAABXRUJQVlA4IBIAAAAwAQCdASoBAAEAAQAcJYgCdAEO", + }, + } + result = _transform_langchain_content_block(content_block) + assert result == { + "type": "blob", + "modality": "image", + "mime_type": "image/webp", + "content": "UklGRh4AAABXRUJQVlA4IBIAAAAwAQCdASoBAAEAAQAcJYgCdAEO", + } + + def test_transform_google_inline_data(self): + """Test transformation of Google-style inline_data format.""" + content_block = { + "inline_data": { + "mime_type": "image/jpeg", + "data": "/9j/4AAQSkZJRgABAQAAAQABAAD...", + } + } + result = _transform_langchain_content_block(content_block) + assert result == { + "type": "blob", + "modality": "image", + "mime_type": "image/jpeg", + "content": "/9j/4AAQSkZJRgABAQAAAQABAAD...", + } + + def test_transform_google_file_data(self): + """Test transformation of Google-style file_data format.""" + content_block = { + "file_data": { + "mime_type": "image/png", + "file_uri": "gs://bucket/path/to/image.png", + } + } + result = _transform_langchain_content_block(content_block) + assert result == { + "type": "uri", + "modality": "image", + "mime_type": "image/png", + "uri": "gs://bucket/path/to/image.png", + } + class TestTransformLangchainMessageContent: """Tests for _transform_langchain_message_content function.""" From 1764e571247a12963148b7bbecec37a6b23bfb4e Mon Sep 17 00:00:00 2001 From: Fabian Schindler Date: Wed, 14 Jan 2026 16:51:10 +0100 Subject: [PATCH 08/12] fix(integrations): Use correct modality for Google-style content formats and use common function for data URI parsing --- sentry_sdk/integrations/langchain.py | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/sentry_sdk/integrations/langchain.py b/sentry_sdk/integrations/langchain.py index 68f5d0ad95..f29dfbe870 100644 --- a/sentry_sdk/integrations/langchain.py +++ b/sentry_sdk/integrations/langchain.py @@ -12,6 +12,7 @@ GEN_AI_ALLOWED_MESSAGE_ROLES, get_start_span_function, normalize_message_roles, + parse_data_uri, set_data_normalized, truncate_and_annotate_messages, ) @@ -199,6 +200,26 @@ def _transform_langchain_content_block( "mime_type": media_type, "uri": source.get("url", ""), } + # Handle Google-style inline_data format with standard type + elif "inline_data" in content_block: + inline_data = content_block.get("inline_data", {}) + if isinstance(inline_data, dict): + return { + "type": "blob", + "modality": modality, + "mime_type": inline_data.get("mime_type", "") or mime_type, + "content": inline_data.get("data", ""), + } + # Handle Google-style file_data format with standard type + elif "file_data" in content_block: + file_data = content_block.get("file_data", {}) + if isinstance(file_data, dict): + return { + "type": "uri", + "modality": modality, + "mime_type": file_data.get("mime_type", "") or mime_type, + "uri": file_data.get("file_uri", ""), + } # Handle legacy image_url format (OpenAI style) elif block_type == "image_url": @@ -210,18 +231,15 @@ def _transform_langchain_content_block( # Check if it's a data URI (base64 encoded) if url and url.startswith("data:"): - # Parse data URI: data:mime_type;base64,content try: - # Format: data:image/jpeg;base64,/9j/4AAQ... - header, content = url.split(",", 1) - mime_type = header.split(":")[1].split(";")[0] if ":" in header else "" + mime_type, content = parse_data_uri(url) return { "type": "blob", "modality": "image", "mime_type": mime_type, "content": content, } - except (ValueError, IndexError): + except ValueError: # If parsing fails, return as URI return { "type": "uri", From 5ca2770bc74b190dc3918723b16bf4a141989515 Mon Sep 17 00:00:00 2001 From: Fabian Schindler Date: Thu, 15 Jan 2026 10:11:57 +0100 Subject: [PATCH 09/12] fix: helper function to get modality for mime-type --- sentry_sdk/integrations/langchain.py | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/sentry_sdk/integrations/langchain.py b/sentry_sdk/integrations/langchain.py index f29dfbe870..a2aa604218 100644 --- a/sentry_sdk/integrations/langchain.py +++ b/sentry_sdk/integrations/langchain.py @@ -127,6 +127,24 @@ } +def _get_modality_from_mime_type(mime_type: str) -> str: + """Infer the content modality from a MIME type string.""" + if not mime_type: + return "image" # Default fallback + + mime_lower = mime_type.lower() + if mime_lower.startswith("image/"): + return "image" + elif mime_lower.startswith("audio/"): + return "audio" + elif mime_lower.startswith("video/"): + return "video" + elif mime_lower.startswith("application/") or mime_lower.startswith("text/"): + return "document" + else: + return "image" # Default fallback for unknown types + + def _transform_langchain_content_block( content_block: "Dict[str, Any]", ) -> "Dict[str, Any]": @@ -260,10 +278,11 @@ def _transform_langchain_content_block( if "inline_data" in content_block: inline_data = content_block.get("inline_data", {}) if isinstance(inline_data, dict): + mime_type = inline_data.get("mime_type", "") return { "type": "blob", - "modality": "image", - "mime_type": inline_data.get("mime_type", ""), + "modality": _get_modality_from_mime_type(mime_type), + "mime_type": mime_type, "content": inline_data.get("data", ""), } @@ -271,10 +290,11 @@ def _transform_langchain_content_block( if "file_data" in content_block: file_data = content_block.get("file_data", {}) if isinstance(file_data, dict): + mime_type = file_data.get("mime_type", "") return { "type": "uri", - "modality": "image", - "mime_type": file_data.get("mime_type", ""), + "modality": _get_modality_from_mime_type(mime_type), + "mime_type": mime_type, "uri": file_data.get("file_uri", ""), } From bd781654c11ef4f1892ad8891296da92e250bb60 Mon Sep 17 00:00:00 2001 From: Fabian Schindler Date: Thu, 15 Jan 2026 14:01:42 +0100 Subject: [PATCH 10/12] feat(ai): Add shared content transformation functions for multimodal AI messages Add transform_content_part() and transform_message_content() functions to standardize content part handling across all AI integrations. These functions transform various SDK-specific formats (OpenAI, Anthropic, Google, LangChain) into a unified format: - blob: base64-encoded binary data - uri: URL references (including file URIs) - file: file ID references Also adds get_modality_from_mime_type() helper to infer content modality (image/audio/video/document) from MIME types. --- sentry_sdk/ai/utils.py | 237 ++++++++++++++++++ tests/test_ai_monitoring.py | 484 ++++++++++++++++++++++++++++++++++++ 2 files changed, 721 insertions(+) diff --git a/sentry_sdk/ai/utils.py b/sentry_sdk/ai/utils.py index 71f7544a1c..b7b3b790d2 100644 --- a/sentry_sdk/ai/utils.py +++ b/sentry_sdk/ai/utils.py @@ -72,6 +72,243 @@ def parse_data_uri(url: str) -> "Tuple[str, str]": return mime_type, content +def get_modality_from_mime_type(mime_type: str) -> str: + """ + Infer the content modality from a MIME type string. + + Args: + mime_type: A MIME type string (e.g., "image/jpeg", "audio/mp3") + + Returns: + One of: "image", "audio", "video", or "document" + Defaults to "image" for unknown or empty MIME types. + + Examples: + "image/jpeg" -> "image" + "audio/mp3" -> "audio" + "video/mp4" -> "video" + "application/pdf" -> "document" + "text/plain" -> "document" + """ + if not mime_type: + return "image" # Default fallback + + mime_lower = mime_type.lower() + if mime_lower.startswith("image/"): + return "image" + elif mime_lower.startswith("audio/"): + return "audio" + elif mime_lower.startswith("video/"): + return "video" + elif mime_lower.startswith("application/") or mime_lower.startswith("text/"): + return "document" + else: + return "image" # Default fallback for unknown types + + +def transform_content_part( + content_part: "Dict[str, Any]", +) -> "Optional[Dict[str, Any]]": + """ + Transform a content part from various AI SDK formats to Sentry's standardized format. + + Supported input formats: + - OpenAI/LiteLLM: {"type": "image_url", "image_url": {"url": "..."}} + - Anthropic: {"type": "image|document", "source": {"type": "base64|url|file", ...}} + - Google: {"inline_data": {...}} or {"file_data": {...}} + - Generic: {"type": "image|audio|video|file", "base64|url|file_id": "...", "mime_type": "..."} + + Output format (one of): + - {"type": "blob", "modality": "...", "mime_type": "...", "content": "..."} + - {"type": "uri", "modality": "...", "mime_type": "...", "uri": "..."} + - {"type": "file", "modality": "...", "mime_type": "...", "file_id": "..."} + + Args: + content_part: A dictionary representing a content part from an AI SDK + + Returns: + A transformed dictionary in standardized format, or None if the format + is unrecognized or transformation fails. + """ + if not isinstance(content_part, dict): + return None + + block_type = content_part.get("type") + + # Handle OpenAI/LiteLLM image_url format + # {"type": "image_url", "image_url": {"url": "..."}} or {"type": "image_url", "image_url": "..."} + if block_type == "image_url": + image_url_data = content_part.get("image_url") + if isinstance(image_url_data, str): + url = image_url_data + elif isinstance(image_url_data, dict): + url = image_url_data.get("url", "") + else: + return None + + if not url: + return None + + # Check if it's a data URI (base64 encoded) + if url.startswith("data:"): + try: + mime_type, content = parse_data_uri(url) + return { + "type": "blob", + "modality": get_modality_from_mime_type(mime_type), + "mime_type": mime_type, + "content": content, + } + except ValueError: + # If parsing fails, return as URI + return { + "type": "uri", + "modality": "image", + "mime_type": "", + "uri": url, + } + else: + # Regular URL + return { + "type": "uri", + "modality": "image", + "mime_type": "", + "uri": url, + } + + # Handle Anthropic format with source dict + # {"type": "image|document", "source": {"type": "base64|url|file", "media_type": "...", "data|url|file_id": "..."}} + if block_type in ("image", "document") and "source" in content_part: + source = content_part.get("source") + if not isinstance(source, dict): + return None + + source_type = source.get("type") + media_type = source.get("media_type", "") + modality = ( + "document" + if block_type == "document" + else get_modality_from_mime_type(media_type) + ) + + if source_type == "base64": + return { + "type": "blob", + "modality": modality, + "mime_type": media_type, + "content": source.get("data", ""), + } + elif source_type == "url": + return { + "type": "uri", + "modality": modality, + "mime_type": media_type, + "uri": source.get("url", ""), + } + elif source_type == "file": + return { + "type": "file", + "modality": modality, + "mime_type": media_type, + "file_id": source.get("file_id", ""), + } + return None + + # Handle Google inline_data format + # {"inline_data": {"mime_type": "...", "data": "..."}} + if "inline_data" in content_part: + inline_data = content_part.get("inline_data") + if isinstance(inline_data, dict): + mime_type = inline_data.get("mime_type", "") + return { + "type": "blob", + "modality": get_modality_from_mime_type(mime_type), + "mime_type": mime_type, + "content": inline_data.get("data", ""), + } + return None + + # Handle Google file_data format + # {"file_data": {"mime_type": "...", "file_uri": "..."}} + if "file_data" in content_part: + file_data = content_part.get("file_data") + if isinstance(file_data, dict): + mime_type = file_data.get("mime_type", "") + return { + "type": "uri", + "modality": get_modality_from_mime_type(mime_type), + "mime_type": mime_type, + "uri": file_data.get("file_uri", ""), + } + return None + + # Handle generic format with direct fields (LangChain style) + # {"type": "image|audio|video|file", "base64|url|file_id": "...", "mime_type": "..."} + if block_type in ("image", "audio", "video", "file"): + mime_type = content_part.get("mime_type", "") + modality = block_type if block_type != "file" else "document" + + # Check for base64 encoded content + if "base64" in content_part: + return { + "type": "blob", + "modality": modality, + "mime_type": mime_type, + "content": content_part.get("base64", ""), + } + # Check for URL reference + elif "url" in content_part: + return { + "type": "uri", + "modality": modality, + "mime_type": mime_type, + "uri": content_part.get("url", ""), + } + # Check for file_id reference + elif "file_id" in content_part: + return { + "type": "file", + "modality": modality, + "mime_type": mime_type, + "file_id": content_part.get("file_id", ""), + } + + # Unrecognized format + return None + + +def transform_message_content(content: "Any") -> "Any": + """ + Transform message content, handling both string content and list of content blocks. + + For list content, each item is transformed using transform_content_part(). + Items that cannot be transformed (return None) are kept as-is. + + Args: + content: Message content - can be a string, list of content blocks, or other + + Returns: + - String content: returned as-is + - List content: list with each transformable item converted to standardized format + - Other: returned as-is + """ + if isinstance(content, str): + return content + + if isinstance(content, (list, tuple)): + transformed = [] + for item in content: + if isinstance(item, dict): + result = transform_content_part(item) + # If transformation succeeded, use the result; otherwise keep original + transformed.append(result if result is not None else item) + else: + transformed.append(item) + return transformed + + return content + + def _normalize_data(data: "Any", unpack: bool = True) -> "Any": # convert pydantic data (e.g. OpenAI v1+) to json compatible format if hasattr(data, "model_dump"): diff --git a/tests/test_ai_monitoring.py b/tests/test_ai_monitoring.py index 1ff354f473..209d24e502 100644 --- a/tests/test_ai_monitoring.py +++ b/tests/test_ai_monitoring.py @@ -19,6 +19,9 @@ _find_truncation_index, parse_data_uri, redact_blob_message_parts, + get_modality_from_mime_type, + transform_content_part, + transform_message_content, ) from sentry_sdk.serializer import serialize from sentry_sdk.utils import safe_serialize @@ -842,3 +845,484 @@ def test_handles_uri_without_data_prefix(self): assert mime_type == "image/jpeg" assert content == "/9j/4AAQ" + + +class TestGetModalityFromMimeType: + def test_image_mime_types(self): + """Test that image MIME types return 'image' modality""" + assert get_modality_from_mime_type("image/jpeg") == "image" + assert get_modality_from_mime_type("image/png") == "image" + assert get_modality_from_mime_type("image/gif") == "image" + assert get_modality_from_mime_type("image/webp") == "image" + assert get_modality_from_mime_type("IMAGE/JPEG") == "image" # case insensitive + + def test_audio_mime_types(self): + """Test that audio MIME types return 'audio' modality""" + assert get_modality_from_mime_type("audio/mp3") == "audio" + assert get_modality_from_mime_type("audio/wav") == "audio" + assert get_modality_from_mime_type("audio/ogg") == "audio" + assert get_modality_from_mime_type("AUDIO/MP3") == "audio" # case insensitive + + def test_video_mime_types(self): + """Test that video MIME types return 'video' modality""" + assert get_modality_from_mime_type("video/mp4") == "video" + assert get_modality_from_mime_type("video/webm") == "video" + assert get_modality_from_mime_type("video/quicktime") == "video" + assert get_modality_from_mime_type("VIDEO/MP4") == "video" # case insensitive + + def test_document_mime_types(self): + """Test that application and text MIME types return 'document' modality""" + assert get_modality_from_mime_type("application/pdf") == "document" + assert get_modality_from_mime_type("application/json") == "document" + assert get_modality_from_mime_type("text/plain") == "document" + assert get_modality_from_mime_type("text/html") == "document" + + def test_empty_mime_type_returns_image(self): + """Test that empty MIME type defaults to 'image'""" + assert get_modality_from_mime_type("") == "image" + + def test_none_mime_type_returns_image(self): + """Test that None-like values default to 'image'""" + assert get_modality_from_mime_type(None) == "image" + + def test_unknown_mime_type_returns_image(self): + """Test that unknown MIME types default to 'image'""" + assert get_modality_from_mime_type("unknown/type") == "image" + assert get_modality_from_mime_type("custom/format") == "image" + + +class TestTransformContentPart: + # OpenAI/LiteLLM format tests + def test_openai_image_url_with_data_uri(self): + """Test transforming OpenAI image_url with base64 data URI""" + content_part = { + "type": "image_url", + "image_url": {"url": "data:image/jpeg;base64,/9j/4AAQSkZJRg=="}, + } + result = transform_content_part(content_part) + + assert result == { + "type": "blob", + "modality": "image", + "mime_type": "image/jpeg", + "content": "/9j/4AAQSkZJRg==", + } + + def test_openai_image_url_with_regular_url(self): + """Test transforming OpenAI image_url with regular URL""" + content_part = { + "type": "image_url", + "image_url": {"url": "https://example.com/image.jpg"}, + } + result = transform_content_part(content_part) + + assert result == { + "type": "uri", + "modality": "image", + "mime_type": "", + "uri": "https://example.com/image.jpg", + } + + def test_openai_image_url_string_format(self): + """Test transforming OpenAI image_url where image_url is a string""" + content_part = { + "type": "image_url", + "image_url": "https://example.com/image.jpg", + } + result = transform_content_part(content_part) + + assert result == { + "type": "uri", + "modality": "image", + "mime_type": "", + "uri": "https://example.com/image.jpg", + } + + def test_openai_image_url_invalid_data_uri(self): + """Test transforming OpenAI image_url with invalid data URI falls back to URI""" + content_part = { + "type": "image_url", + "image_url": {"url": "data:image/jpeg;base64"}, # Missing comma + } + result = transform_content_part(content_part) + + assert result == { + "type": "uri", + "modality": "image", + "mime_type": "", + "uri": "data:image/jpeg;base64", + } + + # Anthropic format tests + def test_anthropic_image_base64(self): + """Test transforming Anthropic image with base64 source""" + content_part = { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": "iVBORw0KGgo=", + }, + } + result = transform_content_part(content_part) + + assert result == { + "type": "blob", + "modality": "image", + "mime_type": "image/png", + "content": "iVBORw0KGgo=", + } + + def test_anthropic_image_url(self): + """Test transforming Anthropic image with URL source""" + content_part = { + "type": "image", + "source": { + "type": "url", + "media_type": "image/jpeg", + "url": "https://example.com/image.jpg", + }, + } + result = transform_content_part(content_part) + + assert result == { + "type": "uri", + "modality": "image", + "mime_type": "image/jpeg", + "uri": "https://example.com/image.jpg", + } + + def test_anthropic_image_file(self): + """Test transforming Anthropic image with file source""" + content_part = { + "type": "image", + "source": { + "type": "file", + "media_type": "image/jpeg", + "file_id": "file_123", + }, + } + result = transform_content_part(content_part) + + assert result == { + "type": "file", + "modality": "image", + "mime_type": "image/jpeg", + "file_id": "file_123", + } + + def test_anthropic_document_base64(self): + """Test transforming Anthropic document with base64 source""" + content_part = { + "type": "document", + "source": { + "type": "base64", + "media_type": "application/pdf", + "data": "JVBERi0xLjQ=", + }, + } + result = transform_content_part(content_part) + + assert result == { + "type": "blob", + "modality": "document", + "mime_type": "application/pdf", + "content": "JVBERi0xLjQ=", + } + + def test_anthropic_document_url(self): + """Test transforming Anthropic document with URL source""" + content_part = { + "type": "document", + "source": { + "type": "url", + "media_type": "application/pdf", + "url": "https://example.com/doc.pdf", + }, + } + result = transform_content_part(content_part) + + assert result == { + "type": "uri", + "modality": "document", + "mime_type": "application/pdf", + "uri": "https://example.com/doc.pdf", + } + + # Google format tests + def test_google_inline_data(self): + """Test transforming Google inline_data format""" + content_part = { + "inline_data": { + "mime_type": "image/jpeg", + "data": "/9j/4AAQSkZJRg==", + } + } + result = transform_content_part(content_part) + + assert result == { + "type": "blob", + "modality": "image", + "mime_type": "image/jpeg", + "content": "/9j/4AAQSkZJRg==", + } + + def test_google_file_data(self): + """Test transforming Google file_data format""" + content_part = { + "file_data": { + "mime_type": "video/mp4", + "file_uri": "gs://bucket/video.mp4", + } + } + result = transform_content_part(content_part) + + assert result == { + "type": "uri", + "modality": "video", + "mime_type": "video/mp4", + "uri": "gs://bucket/video.mp4", + } + + def test_google_inline_data_audio(self): + """Test transforming Google inline_data with audio""" + content_part = { + "inline_data": { + "mime_type": "audio/wav", + "data": "UklGRiQA", + } + } + result = transform_content_part(content_part) + + assert result == { + "type": "blob", + "modality": "audio", + "mime_type": "audio/wav", + "content": "UklGRiQA", + } + + # Generic format tests (LangChain style) + def test_generic_image_base64(self): + """Test transforming generic format with base64""" + content_part = { + "type": "image", + "base64": "/9j/4AAQSkZJRg==", + "mime_type": "image/jpeg", + } + result = transform_content_part(content_part) + + assert result == { + "type": "blob", + "modality": "image", + "mime_type": "image/jpeg", + "content": "/9j/4AAQSkZJRg==", + } + + def test_generic_audio_url(self): + """Test transforming generic format with URL""" + content_part = { + "type": "audio", + "url": "https://example.com/audio.mp3", + "mime_type": "audio/mp3", + } + result = transform_content_part(content_part) + + assert result == { + "type": "uri", + "modality": "audio", + "mime_type": "audio/mp3", + "uri": "https://example.com/audio.mp3", + } + + def test_generic_file_with_file_id(self): + """Test transforming generic format with file_id""" + content_part = { + "type": "file", + "file_id": "file_456", + "mime_type": "application/pdf", + } + result = transform_content_part(content_part) + + assert result == { + "type": "file", + "modality": "document", + "mime_type": "application/pdf", + "file_id": "file_456", + } + + def test_generic_video_base64(self): + """Test transforming generic video format""" + content_part = { + "type": "video", + "base64": "AAAA", + "mime_type": "video/mp4", + } + result = transform_content_part(content_part) + + assert result == { + "type": "blob", + "modality": "video", + "mime_type": "video/mp4", + "content": "AAAA", + } + + # Edge cases and error handling + def test_text_block_returns_none(self): + """Test that text blocks return None (not transformed)""" + content_part = {"type": "text", "text": "Hello world"} + result = transform_content_part(content_part) + + assert result is None + + def test_non_dict_returns_none(self): + """Test that non-dict input returns None""" + assert transform_content_part("string") is None + assert transform_content_part(123) is None + assert transform_content_part(None) is None + assert transform_content_part([1, 2, 3]) is None + + def test_empty_dict_returns_none(self): + """Test that empty dict returns None""" + assert transform_content_part({}) is None + + def test_unknown_type_returns_none(self): + """Test that unknown type returns None""" + content_part = {"type": "unknown", "data": "something"} + assert transform_content_part(content_part) is None + + def test_openai_image_url_empty_url_returns_none(self): + """Test that image_url with empty URL returns None""" + content_part = {"type": "image_url", "image_url": {"url": ""}} + assert transform_content_part(content_part) is None + + def test_anthropic_invalid_source_returns_none(self): + """Test that Anthropic format with invalid source returns None""" + content_part = {"type": "image", "source": "not_a_dict"} + assert transform_content_part(content_part) is None + + def test_anthropic_unknown_source_type_returns_none(self): + """Test that Anthropic format with unknown source type returns None""" + content_part = { + "type": "image", + "source": {"type": "unknown", "data": "something"}, + } + assert transform_content_part(content_part) is None + + def test_google_inline_data_not_dict_returns_none(self): + """Test that Google inline_data with non-dict value returns None""" + content_part = {"inline_data": "not_a_dict"} + assert transform_content_part(content_part) is None + + def test_google_file_data_not_dict_returns_none(self): + """Test that Google file_data with non-dict value returns None""" + content_part = {"file_data": "not_a_dict"} + assert transform_content_part(content_part) is None + + +class TestTransformMessageContent: + def test_string_content_returned_as_is(self): + """Test that string content is returned unchanged""" + content = "Hello, world!" + result = transform_message_content(content) + + assert result == "Hello, world!" + + def test_list_with_transformable_items(self): + """Test transforming a list with transformable content parts""" + content = [ + {"type": "text", "text": "What's in this image?"}, + { + "type": "image_url", + "image_url": {"url": "data:image/jpeg;base64,/9j/4AAQ"}, + }, + ] + result = transform_message_content(content) + + assert len(result) == 2 + # Text block should be unchanged (transform returns None, so original kept) + assert result[0] == {"type": "text", "text": "What's in this image?"} + # Image should be transformed + assert result[1] == { + "type": "blob", + "modality": "image", + "mime_type": "image/jpeg", + "content": "/9j/4AAQ", + } + + def test_list_with_non_dict_items(self): + """Test that non-dict items in list are kept as-is""" + content = ["text string", 123, {"type": "text", "text": "hi"}] + result = transform_message_content(content) + + assert result == ["text string", 123, {"type": "text", "text": "hi"}] + + def test_tuple_content(self): + """Test that tuple content is also handled""" + content = ( + {"type": "text", "text": "Hello"}, + { + "type": "image_url", + "image_url": {"url": "https://example.com/img.jpg"}, + }, + ) + result = transform_message_content(content) + + assert len(result) == 2 + assert result[0] == {"type": "text", "text": "Hello"} + assert result[1] == { + "type": "uri", + "modality": "image", + "mime_type": "", + "uri": "https://example.com/img.jpg", + } + + def test_other_types_returned_as_is(self): + """Test that other types are returned unchanged""" + assert transform_message_content(123) == 123 + assert transform_message_content(None) is None + assert transform_message_content({"key": "value"}) == {"key": "value"} + + def test_mixed_content_types(self): + """Test transforming mixed content with multiple formats""" + content = [ + {"type": "text", "text": "Look at these:"}, + { + "type": "image_url", + "image_url": {"url": "data:image/png;base64,iVBORw0"}, + }, + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/jpeg", + "data": "/9j/4AAQ", + }, + }, + {"inline_data": {"mime_type": "audio/wav", "data": "UklGRiQA"}}, + ] + result = transform_message_content(content) + + assert len(result) == 4 + assert result[0] == {"type": "text", "text": "Look at these:"} + assert result[1] == { + "type": "blob", + "modality": "image", + "mime_type": "image/png", + "content": "iVBORw0", + } + assert result[2] == { + "type": "blob", + "modality": "image", + "mime_type": "image/jpeg", + "content": "/9j/4AAQ", + } + assert result[3] == { + "type": "blob", + "modality": "audio", + "mime_type": "audio/wav", + "content": "UklGRiQA", + } + + def test_empty_list(self): + """Test that empty list is returned as empty list""" + assert transform_message_content([]) == [] From 20367ba0f83a65bbae58d81ca0aa0cbd29bd7966 Mon Sep 17 00:00:00 2001 From: Fabian Schindler Date: Thu, 15 Jan 2026 14:09:45 +0100 Subject: [PATCH 11/12] refactor(langchain): Use shared transform_content_part from ai/utils Replace local _transform_langchain_content_block and _get_modality_from_mime_type functions with the shared transform_content_part function. This removes ~170 lines of duplicated code. --- sentry_sdk/integrations/langchain.py | 185 +----------------- .../integrations/langchain/test_langchain.py | 7 +- 2 files changed, 11 insertions(+), 181 deletions(-) diff --git a/sentry_sdk/integrations/langchain.py b/sentry_sdk/integrations/langchain.py index a2aa604218..8a9bc5167c 100644 --- a/sentry_sdk/integrations/langchain.py +++ b/sentry_sdk/integrations/langchain.py @@ -12,9 +12,9 @@ GEN_AI_ALLOWED_MESSAGE_ROLES, get_start_span_function, normalize_message_roles, - parse_data_uri, set_data_normalized, truncate_and_annotate_messages, + transform_content_part, ) from sentry_sdk.consts import OP, SPANDATA from sentry_sdk.integrations import DidNotEnable, Integration @@ -117,189 +117,18 @@ "top_p": SPANDATA.GEN_AI_REQUEST_TOP_P, } -# Map LangChain content types to Sentry modalities -LANGCHAIN_TYPE_TO_MODALITY = { - "image": "image", - "image_url": "image", - "audio": "audio", - "video": "video", - "file": "document", -} - - -def _get_modality_from_mime_type(mime_type: str) -> str: - """Infer the content modality from a MIME type string.""" - if not mime_type: - return "image" # Default fallback - - mime_lower = mime_type.lower() - if mime_lower.startswith("image/"): - return "image" - elif mime_lower.startswith("audio/"): - return "audio" - elif mime_lower.startswith("video/"): - return "video" - elif mime_lower.startswith("application/") or mime_lower.startswith("text/"): - return "document" - else: - return "image" # Default fallback for unknown types - def _transform_langchain_content_block( content_block: "Dict[str, Any]", ) -> "Dict[str, Any]": """ - Transform a LangChain content block to Sentry-compatible format. - - Handles multimodal content (images, audio, video, documents) by converting them - to the standardized format: - - base64 encoded data -> type: "blob" - - URL references -> type: "uri" - - file_id references -> type: "file" - - Supports multiple content block formats: - - LangChain standard: type + base64/url/file_id fields - - OpenAI legacy: image_url with nested url field - - Anthropic: type + source dict with type/media_type/data or url - - Google: inline_data or file_data dicts - """ - if not isinstance(content_block, dict): - return content_block - - block_type = content_block.get("type") - - # Handle standard multimodal content types (image, audio, video, file) - if block_type in ("image", "audio", "video", "file"): - modality = LANGCHAIN_TYPE_TO_MODALITY.get(block_type, block_type) - mime_type = content_block.get("mime_type", "") - - # Check for base64 encoded content - if "base64" in content_block: - return { - "type": "blob", - "modality": modality, - "mime_type": mime_type, - "content": content_block.get("base64", ""), - } - # Check for URL reference - elif "url" in content_block: - return { - "type": "uri", - "modality": modality, - "mime_type": mime_type, - "uri": content_block.get("url", ""), - } - # Check for file_id reference - elif "file_id" in content_block: - return { - "type": "file", - "modality": modality, - "mime_type": mime_type, - "file_id": content_block.get("file_id", ""), - } - # Handle Anthropic-style format with nested "source" dict - elif "source" in content_block: - source = content_block.get("source", {}) - if isinstance(source, dict): - source_type = source.get("type") - media_type = source.get("media_type", "") or mime_type - - if source_type == "base64": - return { - "type": "blob", - "modality": modality, - "mime_type": media_type, - "content": source.get("data", ""), - } - elif source_type == "url": - return { - "type": "uri", - "modality": modality, - "mime_type": media_type, - "uri": source.get("url", ""), - } - # Handle Google-style inline_data format with standard type - elif "inline_data" in content_block: - inline_data = content_block.get("inline_data", {}) - if isinstance(inline_data, dict): - return { - "type": "blob", - "modality": modality, - "mime_type": inline_data.get("mime_type", "") or mime_type, - "content": inline_data.get("data", ""), - } - # Handle Google-style file_data format with standard type - elif "file_data" in content_block: - file_data = content_block.get("file_data", {}) - if isinstance(file_data, dict): - return { - "type": "uri", - "modality": modality, - "mime_type": file_data.get("mime_type", "") or mime_type, - "uri": file_data.get("file_uri", ""), - } - - # Handle legacy image_url format (OpenAI style) - elif block_type == "image_url": - image_url_data = content_block.get("image_url", {}) - if isinstance(image_url_data, dict): - url = image_url_data.get("url", "") - else: - url = str(image_url_data) + Transform a LangChain content block using the shared transform_content_part function. - # Check if it's a data URI (base64 encoded) - if url and url.startswith("data:"): - try: - mime_type, content = parse_data_uri(url) - return { - "type": "blob", - "modality": "image", - "mime_type": mime_type, - "content": content, - } - except ValueError: - # If parsing fails, return as URI - return { - "type": "uri", - "modality": "image", - "mime_type": "", - "uri": url, - } - else: - # Regular URL - return { - "type": "uri", - "modality": "image", - "mime_type": "", - "uri": url, - } - - # Handle Google-style inline_data format - if "inline_data" in content_block: - inline_data = content_block.get("inline_data", {}) - if isinstance(inline_data, dict): - mime_type = inline_data.get("mime_type", "") - return { - "type": "blob", - "modality": _get_modality_from_mime_type(mime_type), - "mime_type": mime_type, - "content": inline_data.get("data", ""), - } - - # Handle Google-style file_data format - if "file_data" in content_block: - file_data = content_block.get("file_data", {}) - if isinstance(file_data, dict): - mime_type = file_data.get("mime_type", "") - return { - "type": "uri", - "modality": _get_modality_from_mime_type(mime_type), - "mime_type": mime_type, - "uri": file_data.get("file_uri", ""), - } - - # For text blocks and other types, return as-is - return content_block + Returns the original content block if transformation is not applicable + (e.g., for text blocks or unrecognized formats). + """ + result = transform_content_part(content_block) + return result if result is not None else content_block def _transform_langchain_message_content(content: "Any") -> "Any": diff --git a/tests/integrations/langchain/test_langchain.py b/tests/integrations/langchain/test_langchain.py index de5f5841ca..6f5f9f14a1 100644 --- a/tests/integrations/langchain/test_langchain.py +++ b/tests/integrations/langchain/test_langchain.py @@ -1911,20 +1911,21 @@ def test_transform_anthropic_source_url(self): } def test_transform_anthropic_source_without_media_type(self): - """Test transformation of Anthropic-style image without media_type falls back to mime_type.""" + """Test transformation of Anthropic-style image without media_type uses empty mime_type.""" content_block = { "type": "image", - "mime_type": "image/webp", + "mime_type": "image/webp", # Top-level mime_type is ignored by standard Anthropic format "source": { "type": "base64", "data": "UklGRh4AAABXRUJQVlA4IBIAAAAwAQCdASoBAAEAAQAcJYgCdAEO", }, } result = _transform_langchain_content_block(content_block) + # Note: The shared transform_content_part uses media_type from source, not top-level mime_type assert result == { "type": "blob", "modality": "image", - "mime_type": "image/webp", + "mime_type": "", "content": "UklGRh4AAABXRUJQVlA4IBIAAAAwAQCdASoBAAEAAQAcJYgCdAEO", } From 412b93e19d68699b7c9b379a7e1cb28e0f5be43d Mon Sep 17 00:00:00 2001 From: Fabian Schindler Date: Thu, 15 Jan 2026 15:39:37 +0100 Subject: [PATCH 12/12] refactor(ai): split transform_content_part into SDK-specific functions Add dedicated transform functions for each AI SDK: - transform_openai_content_part() for OpenAI/LiteLLM image_url format - transform_anthropic_content_part() for Anthropic image/document format - transform_google_content_part() for Google GenAI inline_data/file_data - transform_generic_content_part() for LangChain-style generic format Refactor transform_content_part() to be a heuristic dispatcher that detects the format and delegates to the appropriate specific function. This allows integrations to use the specific function directly for better performance and clarity, while maintaining backward compatibility through the dispatcher for frameworks that can receive any format. Added 38 new unit tests for the SDK-specific functions. --- sentry_sdk/ai/utils.py | 378 ++++++++++++++++++++++---------- tests/test_ai_monitoring.py | 426 ++++++++++++++++++++++++++++++++++++ 2 files changed, 692 insertions(+), 112 deletions(-) diff --git a/sentry_sdk/ai/utils.py b/sentry_sdk/ai/utils.py index b7b3b790d2..a4ebe96d99 100644 --- a/sentry_sdk/ai/utils.py +++ b/sentry_sdk/ai/utils.py @@ -106,116 +106,174 @@ def get_modality_from_mime_type(mime_type: str) -> str: return "image" # Default fallback for unknown types -def transform_content_part( +def transform_openai_content_part( content_part: "Dict[str, Any]", ) -> "Optional[Dict[str, Any]]": """ - Transform a content part from various AI SDK formats to Sentry's standardized format. + Transform an OpenAI/LiteLLM content part to Sentry's standardized format. + + This handles the OpenAI image_url format used by OpenAI and LiteLLM SDKs. - Supported input formats: - - OpenAI/LiteLLM: {"type": "image_url", "image_url": {"url": "..."}} - - Anthropic: {"type": "image|document", "source": {"type": "base64|url|file", ...}} - - Google: {"inline_data": {...}} or {"file_data": {...}} - - Generic: {"type": "image|audio|video|file", "base64|url|file_id": "...", "mime_type": "..."} + Input format: + - {"type": "image_url", "image_url": {"url": "..."}} + - {"type": "image_url", "image_url": "..."} (string shorthand) Output format (one of): - - {"type": "blob", "modality": "...", "mime_type": "...", "content": "..."} - - {"type": "uri", "modality": "...", "mime_type": "...", "uri": "..."} - - {"type": "file", "modality": "...", "mime_type": "...", "file_id": "..."} + - {"type": "blob", "modality": "image", "mime_type": "...", "content": "..."} + - {"type": "uri", "modality": "image", "mime_type": "", "uri": "..."} Args: - content_part: A dictionary representing a content part from an AI SDK + content_part: A dictionary representing a content part from OpenAI/LiteLLM Returns: A transformed dictionary in standardized format, or None if the format - is unrecognized or transformation fails. + is not OpenAI image_url format or transformation fails. """ if not isinstance(content_part, dict): return None block_type = content_part.get("type") - # Handle OpenAI/LiteLLM image_url format - # {"type": "image_url", "image_url": {"url": "..."}} or {"type": "image_url", "image_url": "..."} - if block_type == "image_url": - image_url_data = content_part.get("image_url") - if isinstance(image_url_data, str): - url = image_url_data - elif isinstance(image_url_data, dict): - url = image_url_data.get("url", "") - else: - return None - - if not url: - return None - - # Check if it's a data URI (base64 encoded) - if url.startswith("data:"): - try: - mime_type, content = parse_data_uri(url) - return { - "type": "blob", - "modality": get_modality_from_mime_type(mime_type), - "mime_type": mime_type, - "content": content, - } - except ValueError: - # If parsing fails, return as URI - return { - "type": "uri", - "modality": "image", - "mime_type": "", - "uri": url, - } - else: - # Regular URL - return { - "type": "uri", - "modality": "image", - "mime_type": "", - "uri": url, - } + if block_type != "image_url": + return None - # Handle Anthropic format with source dict - # {"type": "image|document", "source": {"type": "base64|url|file", "media_type": "...", "data|url|file_id": "..."}} - if block_type in ("image", "document") and "source" in content_part: - source = content_part.get("source") - if not isinstance(source, dict): - return None - - source_type = source.get("type") - media_type = source.get("media_type", "") - modality = ( - "document" - if block_type == "document" - else get_modality_from_mime_type(media_type) - ) + image_url_data = content_part.get("image_url") + if isinstance(image_url_data, str): + url = image_url_data + elif isinstance(image_url_data, dict): + url = image_url_data.get("url", "") + else: + return None - if source_type == "base64": + if not url: + return None + + # Check if it's a data URI (base64 encoded) + if url.startswith("data:"): + try: + mime_type, content = parse_data_uri(url) return { "type": "blob", - "modality": modality, - "mime_type": media_type, - "content": source.get("data", ""), + "modality": get_modality_from_mime_type(mime_type), + "mime_type": mime_type, + "content": content, } - elif source_type == "url": + except ValueError: + # If parsing fails, return as URI return { "type": "uri", - "modality": modality, - "mime_type": media_type, - "uri": source.get("url", ""), - } - elif source_type == "file": - return { - "type": "file", - "modality": modality, - "mime_type": media_type, - "file_id": source.get("file_id", ""), + "modality": "image", + "mime_type": "", + "uri": url, } + else: + # Regular URL + return { + "type": "uri", + "modality": "image", + "mime_type": "", + "uri": url, + } + + +def transform_anthropic_content_part( + content_part: "Dict[str, Any]", +) -> "Optional[Dict[str, Any]]": + """ + Transform an Anthropic content part to Sentry's standardized format. + + This handles the Anthropic image and document formats with source dictionaries. + + Input format: + - {"type": "image", "source": {"type": "base64", "media_type": "...", "data": "..."}} + - {"type": "image", "source": {"type": "url", "media_type": "...", "url": "..."}} + - {"type": "image", "source": {"type": "file", "media_type": "...", "file_id": "..."}} + - {"type": "document", "source": {...}} (same source formats) + + Output format (one of): + - {"type": "blob", "modality": "...", "mime_type": "...", "content": "..."} + - {"type": "uri", "modality": "...", "mime_type": "...", "uri": "..."} + - {"type": "file", "modality": "...", "mime_type": "...", "file_id": "..."} + + Args: + content_part: A dictionary representing a content part from Anthropic + + Returns: + A transformed dictionary in standardized format, or None if the format + is not Anthropic format or transformation fails. + """ + if not isinstance(content_part, dict): + return None + + block_type = content_part.get("type") + + if block_type not in ("image", "document") or "source" not in content_part: + return None + + source = content_part.get("source") + if not isinstance(source, dict): + return None + + source_type = source.get("type") + media_type = source.get("media_type", "") + modality = ( + "document" + if block_type == "document" + else get_modality_from_mime_type(media_type) + ) + + if source_type == "base64": + return { + "type": "blob", + "modality": modality, + "mime_type": media_type, + "content": source.get("data", ""), + } + elif source_type == "url": + return { + "type": "uri", + "modality": modality, + "mime_type": media_type, + "uri": source.get("url", ""), + } + elif source_type == "file": + return { + "type": "file", + "modality": modality, + "mime_type": media_type, + "file_id": source.get("file_id", ""), + } + + return None + + +def transform_google_content_part( + content_part: "Dict[str, Any]", +) -> "Optional[Dict[str, Any]]": + """ + Transform a Google GenAI content part to Sentry's standardized format. + + This handles the Google GenAI inline_data and file_data formats. + + Input format: + - {"inline_data": {"mime_type": "...", "data": "..."}} + - {"file_data": {"mime_type": "...", "file_uri": "..."}} + + Output format (one of): + - {"type": "blob", "modality": "...", "mime_type": "...", "content": "..."} + - {"type": "uri", "modality": "...", "mime_type": "...", "uri": "..."} + + Args: + content_part: A dictionary representing a content part from Google GenAI + + Returns: + A transformed dictionary in standardized format, or None if the format + is not Google format or transformation fails. + """ + if not isinstance(content_part, dict): return None # Handle Google inline_data format - # {"inline_data": {"mime_type": "...", "data": "..."}} if "inline_data" in content_part: inline_data = content_part.get("inline_data") if isinstance(inline_data, dict): @@ -229,7 +287,6 @@ def transform_content_part( return None # Handle Google file_data format - # {"file_data": {"mime_type": "...", "file_uri": "..."}} if "file_data" in content_part: file_data = content_part.get("file_data") if isinstance(file_data, dict): @@ -242,36 +299,133 @@ def transform_content_part( } return None - # Handle generic format with direct fields (LangChain style) - # {"type": "image|audio|video|file", "base64|url|file_id": "...", "mime_type": "..."} - if block_type in ("image", "audio", "video", "file"): - mime_type = content_part.get("mime_type", "") - modality = block_type if block_type != "file" else "document" + return None - # Check for base64 encoded content - if "base64" in content_part: - return { - "type": "blob", - "modality": modality, - "mime_type": mime_type, - "content": content_part.get("base64", ""), - } - # Check for URL reference - elif "url" in content_part: - return { - "type": "uri", - "modality": modality, - "mime_type": mime_type, - "uri": content_part.get("url", ""), - } - # Check for file_id reference - elif "file_id" in content_part: - return { - "type": "file", - "modality": modality, - "mime_type": mime_type, - "file_id": content_part.get("file_id", ""), - } + +def transform_generic_content_part( + content_part: "Dict[str, Any]", +) -> "Optional[Dict[str, Any]]": + """ + Transform a generic/LangChain-style content part to Sentry's standardized format. + + This handles generic formats where the type indicates the modality and + the data is provided via direct base64, url, or file_id fields. + + Input format: + - {"type": "image", "base64": "...", "mime_type": "..."} + - {"type": "audio", "url": "...", "mime_type": "..."} + - {"type": "video", "base64": "...", "mime_type": "..."} + - {"type": "file", "file_id": "...", "mime_type": "..."} + + Output format (one of): + - {"type": "blob", "modality": "...", "mime_type": "...", "content": "..."} + - {"type": "uri", "modality": "...", "mime_type": "...", "uri": "..."} + - {"type": "file", "modality": "...", "mime_type": "...", "file_id": "..."} + + Args: + content_part: A dictionary representing a content part in generic format + + Returns: + A transformed dictionary in standardized format, or None if the format + is not generic format or transformation fails. + """ + if not isinstance(content_part, dict): + return None + + block_type = content_part.get("type") + + if block_type not in ("image", "audio", "video", "file"): + return None + + # Ensure it's not Anthropic format (which also uses type: "image") + if "source" in content_part: + return None + + mime_type = content_part.get("mime_type", "") + modality = block_type if block_type != "file" else "document" + + # Check for base64 encoded content + if "base64" in content_part: + return { + "type": "blob", + "modality": modality, + "mime_type": mime_type, + "content": content_part.get("base64", ""), + } + # Check for URL reference + elif "url" in content_part: + return { + "type": "uri", + "modality": modality, + "mime_type": mime_type, + "uri": content_part.get("url", ""), + } + # Check for file_id reference + elif "file_id" in content_part: + return { + "type": "file", + "modality": modality, + "mime_type": mime_type, + "file_id": content_part.get("file_id", ""), + } + + return None + + +def transform_content_part( + content_part: "Dict[str, Any]", +) -> "Optional[Dict[str, Any]]": + """ + Transform a content part from various AI SDK formats to Sentry's standardized format. + + This is a heuristic dispatcher that detects the format and delegates to the + appropriate SDK-specific transformer. For direct SDK integration, prefer using + the specific transformers directly: + - transform_openai_content_part() for OpenAI/LiteLLM + - transform_anthropic_content_part() for Anthropic + - transform_google_content_part() for Google GenAI + - transform_generic_content_part() for LangChain and other generic formats + + Detection order: + 1. OpenAI: type == "image_url" + 2. Google: "inline_data" or "file_data" keys present + 3. Anthropic: type in ("image", "document") with "source" key + 4. Generic: type in ("image", "audio", "video", "file") with base64/url/file_id + + Output format (one of): + - {"type": "blob", "modality": "...", "mime_type": "...", "content": "..."} + - {"type": "uri", "modality": "...", "mime_type": "...", "uri": "..."} + - {"type": "file", "modality": "...", "mime_type": "...", "file_id": "..."} + + Args: + content_part: A dictionary representing a content part from an AI SDK + + Returns: + A transformed dictionary in standardized format, or None if the format + is unrecognized or transformation fails. + """ + if not isinstance(content_part, dict): + return None + + # Try OpenAI format first (most common, clear indicator) + result = transform_openai_content_part(content_part) + if result is not None: + return result + + # Try Google format (unique keys make it easy to detect) + result = transform_google_content_part(content_part) + if result is not None: + return result + + # Try Anthropic format (has "source" key) + result = transform_anthropic_content_part(content_part) + if result is not None: + return result + + # Try generic format as fallback + result = transform_generic_content_part(content_part) + if result is not None: + return result # Unrecognized format return None diff --git a/tests/test_ai_monitoring.py b/tests/test_ai_monitoring.py index 209d24e502..f6852d54bb 100644 --- a/tests/test_ai_monitoring.py +++ b/tests/test_ai_monitoring.py @@ -20,6 +20,10 @@ parse_data_uri, redact_blob_message_parts, get_modality_from_mime_type, + transform_openai_content_part, + transform_anthropic_content_part, + transform_google_content_part, + transform_generic_content_part, transform_content_part, transform_message_content, ) @@ -891,6 +895,428 @@ def test_unknown_mime_type_returns_image(self): assert get_modality_from_mime_type("custom/format") == "image" +class TestTransformOpenAIContentPart: + """Tests for the OpenAI-specific transform function.""" + + def test_image_url_with_data_uri(self): + """Test transforming OpenAI image_url with base64 data URI""" + content_part = { + "type": "image_url", + "image_url": {"url": "data:image/jpeg;base64,/9j/4AAQSkZJRg=="}, + } + result = transform_openai_content_part(content_part) + + assert result == { + "type": "blob", + "modality": "image", + "mime_type": "image/jpeg", + "content": "/9j/4AAQSkZJRg==", + } + + def test_image_url_with_regular_url(self): + """Test transforming OpenAI image_url with regular URL""" + content_part = { + "type": "image_url", + "image_url": {"url": "https://example.com/image.jpg"}, + } + result = transform_openai_content_part(content_part) + + assert result == { + "type": "uri", + "modality": "image", + "mime_type": "", + "uri": "https://example.com/image.jpg", + } + + def test_image_url_string_format(self): + """Test transforming OpenAI image_url where image_url is a string""" + content_part = { + "type": "image_url", + "image_url": "https://example.com/image.jpg", + } + result = transform_openai_content_part(content_part) + + assert result == { + "type": "uri", + "modality": "image", + "mime_type": "", + "uri": "https://example.com/image.jpg", + } + + def test_image_url_invalid_data_uri(self): + """Test transforming OpenAI image_url with invalid data URI falls back to URI""" + content_part = { + "type": "image_url", + "image_url": {"url": "data:image/jpeg;base64"}, # Missing comma + } + result = transform_openai_content_part(content_part) + + assert result == { + "type": "uri", + "modality": "image", + "mime_type": "", + "uri": "data:image/jpeg;base64", + } + + def test_empty_url_returns_none(self): + """Test that image_url with empty URL returns None""" + content_part = {"type": "image_url", "image_url": {"url": ""}} + assert transform_openai_content_part(content_part) is None + + def test_non_image_url_type_returns_none(self): + """Test that non-image_url types return None""" + content_part = {"type": "text", "text": "Hello"} + assert transform_openai_content_part(content_part) is None + + def test_anthropic_format_returns_none(self): + """Test that Anthropic format returns None (not handled)""" + content_part = { + "type": "image", + "source": {"type": "base64", "media_type": "image/png", "data": "abc"}, + } + assert transform_openai_content_part(content_part) is None + + def test_google_format_returns_none(self): + """Test that Google format returns None (not handled)""" + content_part = {"inline_data": {"mime_type": "image/jpeg", "data": "abc"}} + assert transform_openai_content_part(content_part) is None + + def test_non_dict_returns_none(self): + """Test that non-dict input returns None""" + assert transform_openai_content_part("string") is None + assert transform_openai_content_part(123) is None + assert transform_openai_content_part(None) is None + + +class TestTransformAnthropicContentPart: + """Tests for the Anthropic-specific transform function.""" + + def test_image_base64(self): + """Test transforming Anthropic image with base64 source""" + content_part = { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": "iVBORw0KGgo=", + }, + } + result = transform_anthropic_content_part(content_part) + + assert result == { + "type": "blob", + "modality": "image", + "mime_type": "image/png", + "content": "iVBORw0KGgo=", + } + + def test_image_url(self): + """Test transforming Anthropic image with URL source""" + content_part = { + "type": "image", + "source": { + "type": "url", + "media_type": "image/jpeg", + "url": "https://example.com/image.jpg", + }, + } + result = transform_anthropic_content_part(content_part) + + assert result == { + "type": "uri", + "modality": "image", + "mime_type": "image/jpeg", + "uri": "https://example.com/image.jpg", + } + + def test_image_file(self): + """Test transforming Anthropic image with file source""" + content_part = { + "type": "image", + "source": { + "type": "file", + "media_type": "image/jpeg", + "file_id": "file_123", + }, + } + result = transform_anthropic_content_part(content_part) + + assert result == { + "type": "file", + "modality": "image", + "mime_type": "image/jpeg", + "file_id": "file_123", + } + + def test_document_base64(self): + """Test transforming Anthropic document with base64 source""" + content_part = { + "type": "document", + "source": { + "type": "base64", + "media_type": "application/pdf", + "data": "JVBERi0xLjQ=", + }, + } + result = transform_anthropic_content_part(content_part) + + assert result == { + "type": "blob", + "modality": "document", + "mime_type": "application/pdf", + "content": "JVBERi0xLjQ=", + } + + def test_document_url(self): + """Test transforming Anthropic document with URL source""" + content_part = { + "type": "document", + "source": { + "type": "url", + "media_type": "application/pdf", + "url": "https://example.com/doc.pdf", + }, + } + result = transform_anthropic_content_part(content_part) + + assert result == { + "type": "uri", + "modality": "document", + "mime_type": "application/pdf", + "uri": "https://example.com/doc.pdf", + } + + def test_invalid_source_returns_none(self): + """Test that Anthropic format with invalid source returns None""" + content_part = {"type": "image", "source": "not_a_dict"} + assert transform_anthropic_content_part(content_part) is None + + def test_unknown_source_type_returns_none(self): + """Test that Anthropic format with unknown source type returns None""" + content_part = { + "type": "image", + "source": {"type": "unknown", "data": "something"}, + } + assert transform_anthropic_content_part(content_part) is None + + def test_missing_source_returns_none(self): + """Test that Anthropic format without source returns None""" + content_part = {"type": "image", "data": "something"} + assert transform_anthropic_content_part(content_part) is None + + def test_openai_format_returns_none(self): + """Test that OpenAI format returns None (not handled)""" + content_part = { + "type": "image_url", + "image_url": {"url": "https://example.com"}, + } + assert transform_anthropic_content_part(content_part) is None + + def test_google_format_returns_none(self): + """Test that Google format returns None (not handled)""" + content_part = {"inline_data": {"mime_type": "image/jpeg", "data": "abc"}} + assert transform_anthropic_content_part(content_part) is None + + def test_non_dict_returns_none(self): + """Test that non-dict input returns None""" + assert transform_anthropic_content_part("string") is None + assert transform_anthropic_content_part(123) is None + assert transform_anthropic_content_part(None) is None + + +class TestTransformGoogleContentPart: + """Tests for the Google GenAI-specific transform function.""" + + def test_inline_data(self): + """Test transforming Google inline_data format""" + content_part = { + "inline_data": { + "mime_type": "image/jpeg", + "data": "/9j/4AAQSkZJRg==", + } + } + result = transform_google_content_part(content_part) + + assert result == { + "type": "blob", + "modality": "image", + "mime_type": "image/jpeg", + "content": "/9j/4AAQSkZJRg==", + } + + def test_file_data(self): + """Test transforming Google file_data format""" + content_part = { + "file_data": { + "mime_type": "video/mp4", + "file_uri": "gs://bucket/video.mp4", + } + } + result = transform_google_content_part(content_part) + + assert result == { + "type": "uri", + "modality": "video", + "mime_type": "video/mp4", + "uri": "gs://bucket/video.mp4", + } + + def test_inline_data_audio(self): + """Test transforming Google inline_data with audio""" + content_part = { + "inline_data": { + "mime_type": "audio/wav", + "data": "UklGRiQA", + } + } + result = transform_google_content_part(content_part) + + assert result == { + "type": "blob", + "modality": "audio", + "mime_type": "audio/wav", + "content": "UklGRiQA", + } + + def test_inline_data_not_dict_returns_none(self): + """Test that Google inline_data with non-dict value returns None""" + content_part = {"inline_data": "not_a_dict"} + assert transform_google_content_part(content_part) is None + + def test_file_data_not_dict_returns_none(self): + """Test that Google file_data with non-dict value returns None""" + content_part = {"file_data": "not_a_dict"} + assert transform_google_content_part(content_part) is None + + def test_openai_format_returns_none(self): + """Test that OpenAI format returns None (not handled)""" + content_part = { + "type": "image_url", + "image_url": {"url": "https://example.com"}, + } + assert transform_google_content_part(content_part) is None + + def test_anthropic_format_returns_none(self): + """Test that Anthropic format returns None (not handled)""" + content_part = { + "type": "image", + "source": {"type": "base64", "media_type": "image/png", "data": "abc"}, + } + assert transform_google_content_part(content_part) is None + + def test_non_dict_returns_none(self): + """Test that non-dict input returns None""" + assert transform_google_content_part("string") is None + assert transform_google_content_part(123) is None + assert transform_google_content_part(None) is None + + +class TestTransformGenericContentPart: + """Tests for the generic/LangChain-style transform function.""" + + def test_image_base64(self): + """Test transforming generic format with base64""" + content_part = { + "type": "image", + "base64": "/9j/4AAQSkZJRg==", + "mime_type": "image/jpeg", + } + result = transform_generic_content_part(content_part) + + assert result == { + "type": "blob", + "modality": "image", + "mime_type": "image/jpeg", + "content": "/9j/4AAQSkZJRg==", + } + + def test_audio_url(self): + """Test transforming generic format with URL""" + content_part = { + "type": "audio", + "url": "https://example.com/audio.mp3", + "mime_type": "audio/mp3", + } + result = transform_generic_content_part(content_part) + + assert result == { + "type": "uri", + "modality": "audio", + "mime_type": "audio/mp3", + "uri": "https://example.com/audio.mp3", + } + + def test_file_with_file_id(self): + """Test transforming generic format with file_id""" + content_part = { + "type": "file", + "file_id": "file_456", + "mime_type": "application/pdf", + } + result = transform_generic_content_part(content_part) + + assert result == { + "type": "file", + "modality": "document", + "mime_type": "application/pdf", + "file_id": "file_456", + } + + def test_video_base64(self): + """Test transforming generic video format""" + content_part = { + "type": "video", + "base64": "AAAA", + "mime_type": "video/mp4", + } + result = transform_generic_content_part(content_part) + + assert result == { + "type": "blob", + "modality": "video", + "mime_type": "video/mp4", + "content": "AAAA", + } + + def test_image_with_source_returns_none(self): + """Test that image with source key (Anthropic style) returns None""" + # This is Anthropic format, should NOT be handled by generic + content_part = { + "type": "image", + "source": {"type": "base64", "data": "abc"}, + } + assert transform_generic_content_part(content_part) is None + + def test_text_type_returns_none(self): + """Test that text type returns None""" + content_part = {"type": "text", "text": "Hello"} + assert transform_generic_content_part(content_part) is None + + def test_openai_format_returns_none(self): + """Test that OpenAI format returns None (not handled)""" + content_part = { + "type": "image_url", + "image_url": {"url": "https://example.com"}, + } + assert transform_generic_content_part(content_part) is None + + def test_google_format_returns_none(self): + """Test that Google format returns None (not handled)""" + content_part = {"inline_data": {"mime_type": "image/jpeg", "data": "abc"}} + assert transform_generic_content_part(content_part) is None + + def test_non_dict_returns_none(self): + """Test that non-dict input returns None""" + assert transform_generic_content_part("string") is None + assert transform_generic_content_part(123) is None + assert transform_generic_content_part(None) is None + + def test_missing_data_key_returns_none(self): + """Test that missing data key (base64/url/file_id) returns None""" + content_part = {"type": "image", "mime_type": "image/jpeg"} + assert transform_generic_content_part(content_part) is None + + class TestTransformContentPart: # OpenAI/LiteLLM format tests def test_openai_image_url_with_data_uri(self):