diff --git a/sentry_sdk/integrations/openai.py b/sentry_sdk/integrations/openai.py index 53d464c3c4..79724f389d 100644 --- a/sentry_sdk/integrations/openai.py +++ b/sentry_sdk/integrations/openai.py @@ -18,7 +18,7 @@ safe_serialize, ) -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Dict if TYPE_CHECKING: from typing import Any, Iterable, List, Optional, Callable, AsyncIterator, Iterator @@ -177,6 +177,68 @@ def _calculate_token_usage( ) +def _convert_message_parts(messages: "List[Dict[str, Any]]") -> "List[Dict[str, Any]]": + """ + Convert the message parts from OpenAI format to the `gen_ai.request.messages` format. + e.g: + { + "role": "user", + "content": [ + { + "text": "How many ponies do you see in the image?", + "type": "text" + }, + { + "type": "image_url", + "image_url": { + "url": "data:image/jpeg;base64,...", + "detail": "high" + } + } + ] + } + becomes: + { + "role": "user", + "content": [ + { + "text": "How many ponies do you see in the image?", + "type": "text" + }, + { + "type": "blob", + "modality": "image", + "mime_type": "image/jpeg", + "content": "data:image/jpeg;base64,..." + } + ] + } + """ + + def _map_item(item: "Dict[str, Any]") -> "Dict[str, Any]": + if item.get("type") == "image_url": + image_url = item.get("image_url") or {} + if image_url.get("url", "").startswith("data:"): + return { + "type": "blob", + "modality": "image", + "mime_type": item["image_url"]["url"].split(";base64,")[0], + "content": item["image_url"]["url"].split(";base64,")[1], + } + else: + return { + "type": "uri", + "uri": item["image_url"]["url"], + } + return item + + for message in messages: + content = message.get("content") + if isinstance(content, list): + message["content"] = [_map_item(item) for item in content] + return messages + + def _set_input_data( span: "Span", kwargs: "dict[str, Any]", @@ -198,6 +260,8 @@ def _set_input_data( and integration.include_prompts ): normalized_messages = normalize_message_roles(messages) + normalized_messages = _convert_message_parts(normalized_messages) + scope = sentry_sdk.get_current_scope() messages_data = truncate_and_annotate_messages(normalized_messages, span, scope) if messages_data is not None: diff --git a/tests/integrations/openai/test_openai.py b/tests/integrations/openai/test_openai.py index 814289c887..f4f616fad3 100644 --- a/tests/integrations/openai/test_openai.py +++ b/tests/integrations/openai/test_openai.py @@ -43,6 +43,7 @@ from sentry_sdk.integrations.openai import ( OpenAIIntegration, _calculate_token_usage, + _convert_message_parts, ) from sentry_sdk.ai.utils import MAX_GEN_AI_MESSAGE_BYTES from sentry_sdk._types import AnnotatedValue @@ -1509,6 +1510,77 @@ def test_openai_message_role_mapping(sentry_init, capture_events): assert "ai" not in roles +def test_convert_message_parts_image_url_to_blob(): + """Test that OpenAI image_url message parts are correctly converted to blob format""" + messages = [ + { + "role": "user", + "content": [ + { + "text": "How many ponies do you see in the image?", + "type": "text", + }, + { + "type": "image_url", + "image_url": { + "url": "data:image/jpeg;base64,/9j/4AAQSkZJRg==", + "detail": "high", + }, + }, + ], + } + ] + + converted = _convert_message_parts(messages) + + assert len(converted) == 1 + assert converted[0]["role"] == "user" + assert isinstance(converted[0]["content"], list) + assert len(converted[0]["content"]) == 2 + + # First item (text) should remain unchanged + assert converted[0]["content"][0] == { + "text": "How many ponies do you see in the image?", + "type": "text", + } + + # Second item (image_url) should be converted to blob format + blob_item = converted[0]["content"][1] + assert blob_item["type"] == "blob" + assert blob_item["modality"] == "image" + assert blob_item["mime_type"] == "data:image/jpeg" + assert blob_item["content"] == "/9j/4AAQSkZJRg==" + # Verify the original image_url structure is replaced + assert "image_url" not in blob_item + + +def test_convert_message_parts_image_url_to_uri(): + """Test that OpenAI image_url with non-data URLs are converted to uri format""" + messages = [ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": { + "url": "https://example.com/image.jpg", + "detail": "low", + }, + }, + ], + } + ] + + converted = _convert_message_parts(messages) + + assert len(converted) == 1 + uri_item = converted[0]["content"][0] + assert uri_item["type"] == "uri" + assert uri_item["uri"] == "https://example.com/image.jpg" + # Verify the original image_url structure is replaced + assert "image_url" not in uri_item + + def test_openai_message_truncation(sentry_init, capture_events): """Test that large messages are truncated properly in OpenAI integration.""" sentry_init(