diff --git a/sentry_sdk/integrations/openai.py b/sentry_sdk/integrations/openai.py index a400393237..c080c0db8d 100644 --- a/sentry_sdk/integrations/openai.py +++ b/sentry_sdk/integrations/openai.py @@ -114,7 +114,7 @@ class OpenAIIntegration(Integration): def __init__( self: "OpenAIIntegration", - include_prompts: bool = True, + include_prompts: "Optional[bool]" = None, tiktoken_encoding_name: "Optional[str]" = None, ) -> None: self.include_prompts = include_prompts @@ -325,6 +325,18 @@ def _calculate_responses_token_usage( ) +def _set_tool_definitions_responses_api( + set_on_span: "Callable[[str, Any], None]", + kwargs: "dict[str, Any]", +) -> None: + tools = kwargs.get("tools") + if tools is not None and _is_given(tools): + set_on_span( + SPANDATA.GEN_AI_TOOL_DEFINITIONS, + json.dumps(_transform_tool_definitions_responses(tools)), + ) + + def _set_responses_api_input_data( span: "Union[Span, StreamedSpan]", kwargs: "dict[str, Any]", @@ -371,33 +383,24 @@ def _set_responses_api_input_data( client_options = sentry_sdk.get_client().options if has_data_collection_enabled(client_options): - if ( - integration.include_prompts - and client_options["data_collection"]["gen_ai"]["inputs"] - ): - tools = kwargs.get("tools") - if tools is not None and _is_given(tools): - set_on_span( - SPANDATA.GEN_AI_TOOL_DEFINITIONS, - json.dumps(_transform_tool_definitions_responses(tools)), - ) + if integration.include_prompts is not None: + if integration.include_prompts: + _set_tool_definitions_responses_api(set_on_span, kwargs) + elif client_options["data_collection"]["gen_ai"]["inputs"]: + _set_tool_definitions_responses_api(set_on_span, kwargs) else: # Pre-data collection this was always set, so this needs to be left here for now until # we deprecate `send_default_pii`. Once we do, this 'else' branch should be removed, # and the above branch placed below the "if not should_send_default_pii() or not integration.include_prompts" # line below - tools = kwargs.get("tools") - if tools is not None and _is_given(tools): - set_on_span( - SPANDATA.GEN_AI_TOOL_DEFINITIONS, - json.dumps(_transform_tool_definitions_responses(tools)), - ) + _set_tool_definitions_responses_api(set_on_span, kwargs) if has_data_collection_enabled(client_options): # This takes precedence over the global data collection settings - if not integration.include_prompts: - return - if not client_options["data_collection"]["gen_ai"]["inputs"]: + if integration.include_prompts is not None: + if not integration.include_prompts: + return + elif not client_options["data_collection"]["gen_ai"]["inputs"]: return elif not should_send_default_pii() or not integration.include_prompts: return @@ -478,6 +481,18 @@ def _set_responses_api_input_data( ) +def _set_tool_definitions_completions_api( + set_on_span: "Callable[[str, Any], None]", + kwargs: "dict[str, Any]", +) -> None: + tools = kwargs.get("tools") + if tools is not None and _is_given(tools): + set_on_span( + SPANDATA.GEN_AI_TOOL_DEFINITIONS, + json.dumps(_transform_tool_definitions_completions(tools)), + ) + + def _set_completions_api_input_data( span: "Union[Span, StreamedSpan]", kwargs: "dict[str, Any]", @@ -519,27 +534,17 @@ def _set_completions_api_input_data( client = sentry_sdk.get_client() if has_data_collection_enabled(client.options): - if ( - integration.include_prompts - and client.options["data_collection"]["gen_ai"]["inputs"] - ): - tools = kwargs.get("tools") - if tools is not None and _is_given(tools): - set_on_span( - SPANDATA.GEN_AI_TOOL_DEFINITIONS, - json.dumps(_transform_tool_definitions_completions(tools)), - ) + if integration.include_prompts is not None: + if integration.include_prompts: + _set_tool_definitions_completions_api(set_on_span, kwargs) + elif client.options["data_collection"]["gen_ai"]["inputs"]: + _set_tool_definitions_completions_api(set_on_span, kwargs) else: # Pre-data collection this was always set, so this needs to be left here for now until # we deprecate `send_default_pii`. Once we do, this 'else' branch should be removed, # and the above branch placed below the "if not should_send_default_pii() or not integration.include_prompts" # line below - tools = kwargs.get("tools") - if tools is not None and _is_given(tools): - set_on_span( - SPANDATA.GEN_AI_TOOL_DEFINITIONS, - json.dumps(_transform_tool_definitions_completions(tools)), - ) + _set_tool_definitions_completions_api(set_on_span, kwargs) messages: "Optional[Union[str, Iterable[ChatCompletionMessageParam]]]" = kwargs.get( "messages" @@ -547,9 +552,10 @@ def _set_completions_api_input_data( if has_data_collection_enabled(client.options): # This takes precedence over the global data collection settings - if not integration.include_prompts: - return - if not client.options["data_collection"]["gen_ai"]["inputs"]: + if integration.include_prompts is not None: + if not integration.include_prompts: + return + elif not client.options["data_collection"]["gen_ai"]["inputs"]: return elif not should_send_default_pii() or not integration.include_prompts: return @@ -610,31 +616,37 @@ def _set_embeddings_input_data( kwargs: "dict[str, Any]", integration: "OpenAIIntegration", ) -> None: - messages: "Union[str, SequenceNotStr[str], Iterable[int], Iterable[Iterable[int]]]" = kwargs.get( - "input" - ) + + set_data_normalized(span, SPANDATA.GEN_AI_OPERATION_NAME, "embeddings") set_on_span = ( span.set_attribute if isinstance(span, StreamedSpan) else span.set_data ) + model = kwargs.get("model") if model is not None: set_on_span(SPANDATA.GEN_AI_REQUEST_MODEL, model) - if ( - not should_send_default_pii() - or not integration.include_prompts - or messages is None - ): - set_data_normalized(span, SPANDATA.GEN_AI_OPERATION_NAME, "embeddings") + messages: "Union[str, SequenceNotStr[str], Iterable[int], Iterable[Iterable[int]]]" = kwargs.get( + "input" + ) + client = sentry_sdk.get_client() + if has_data_collection_enabled(client.options): + # This takes precedence over the global data collection settings + if integration.include_prompts is not None: + if not integration.include_prompts: + return + elif not client.options["data_collection"]["gen_ai"]["inputs"]: + return + elif not should_send_default_pii() or not integration.include_prompts: return - if isinstance(messages, str): - set_data_normalized(span, SPANDATA.GEN_AI_OPERATION_NAME, "embeddings") + if messages is None: + return + if isinstance(messages, str): normalized_messages = normalize_message_roles([messages]) # type: ignore - client = sentry_sdk.get_client() scope = sentry_sdk.get_current_scope() messages_data = ( truncate_and_annotate_embedding_inputs(normalized_messages, span, scope) @@ -650,7 +662,6 @@ def _set_embeddings_input_data( # dict special case following https://github.com/openai/openai-python/blob/3e0c05b84a2056870abf3bd6a5e7849020209cc3/src/openai/_utils/_transform.py#L194-L197 if not isinstance(messages, Iterable) or isinstance(messages, dict): - set_data_normalized(span, SPANDATA.GEN_AI_OPERATION_NAME, "embeddings") return messages = list(messages) @@ -658,7 +669,6 @@ def _set_embeddings_input_data( if len(messages) > 0: normalized_messages = normalize_message_roles(messages) - client = sentry_sdk.get_client() scope = sentry_sdk.get_current_scope() messages_data = ( truncate_and_annotate_embedding_inputs(normalized_messages, span, scope) @@ -670,8 +680,6 @@ def _set_embeddings_input_data( span, SPANDATA.GEN_AI_EMBEDDINGS_INPUT, messages_data, unpack=False ) - set_data_normalized(span, SPANDATA.GEN_AI_OPERATION_NAME, "embeddings") - def _set_common_output_data( span: "Union[Span, StreamedSpan]", @@ -762,12 +770,25 @@ def _set_common_output_data( span.__exit__(None, None, None) +def _resolve_include_prompts( + integration: "OpenAIIntegration", options: "Optional[dict[str, Any]]" +) -> None: + if integration.include_prompts is None and not has_data_collection_enabled(options): + # Pre-data-collection behavior: prompts were included by default (subject + # to send_default_pii). Resolve the unset default here, at call time, + # rather than in __init__ so that integrations constructed before + # sentry_sdk.init() still see the active client's options. + integration.include_prompts = True + + def _new_sync_chat_completion(f: "Any", *args: "Any", **kwargs: "Any") -> "Any": client = sentry_sdk.get_client() integration = client.get_integration(OpenAIIntegration) if integration is None: return f(*args, **kwargs) + _resolve_include_prompts(integration, client.options) + if "messages" not in kwargs: # invalid call (in all versions of openai), let it return error return f(*args, **kwargs) @@ -849,6 +870,8 @@ async def _new_async_chat_completion(f: "Any", *args: "Any", **kwargs: "Any") -> if integration is None: return await f(*args, **kwargs) + _resolve_include_prompts(integration, client.options) + if "messages" not in kwargs: # invalid call (in all versions of openai), let it return error return await f(*args, **kwargs) @@ -1279,6 +1302,8 @@ def _new_sync_embeddings_create(f: "Any", *args: "Any", **kwargs: "Any") -> "Any if integration is None: return f(*args, **kwargs) + _resolve_include_prompts(integration, client.options) + model = kwargs.get("model") if has_span_streaming_enabled(client.options): @@ -1337,6 +1362,8 @@ async def _new_async_embeddings_create( if integration is None: return await f(*args, **kwargs) + _resolve_include_prompts(integration, client.options) + model = kwargs.get("model") if has_span_streaming_enabled(client.options): @@ -1417,6 +1444,8 @@ def _new_sync_responses_create(f: "Any", *args: "Any", **kwargs: "Any") -> "Any" if integration is None: return f(*args, **kwargs) + _resolve_include_prompts(integration, client.options) + model = kwargs.get("model") # Same bool handling as in https://github.com/openai/openai-python/blob/acd0c54d8a68efeedde0e5b4e6c310eef1ce7867/src/openai/resources/responses/responses.py#L940 @@ -1487,6 +1516,8 @@ async def _new_async_responses_create(f: "Any", *args: "Any", **kwargs: "Any") - if integration is None: return await f(*args, **kwargs) + _resolve_include_prompts(integration, client.options) + model = kwargs.get("model") # Same bool handling as in https://github.com/openai/openai-python/blob/acd0c54d8a68efeedde0e5b4e6c310eef1ce7867/src/openai/resources/responses/responses.py#L940 diff --git a/tests/integrations/openai/test_openai.py b/tests/integrations/openai/test_openai.py index e03ab2bbcf..a929ed9752 100644 --- a/tests/integrations/openai/test_openai.py +++ b/tests/integrations/openai/test_openai.py @@ -158,6 +158,31 @@ async def __call__(self, *args, **kwargs): ] +@pytest.mark.parametrize( + "data_collection_enabled", + [True, False], + ids=["data-collection-enabled", "data-collection-disabled"], +) +@pytest.mark.parametrize("include_prompts", [None, True, False]) +def test_include_prompts_init_stores_passed_value( + sentry_init, data_collection_enabled, include_prompts +): + init_kwargs = { + "traces_sample_rate": 1.0, + "disabled_integrations": [StdlibIntegration], + } + if data_collection_enabled: + init_kwargs["_experiments"] = {"data_collection": {"gen_ai": {"inputs": True}}} + sentry_init(**init_kwargs) + + integration = OpenAIIntegration(include_prompts=include_prompts) + + # The constructor stores the value as passed; the None default is resolved + # lazily at call time based on the active client's options (covered by the + # data_collection parametrizations below). + assert integration.include_prompts is include_prompts + + @pytest.mark.skipif( OPENAI_VERSION <= (2, 10, 0), reason="ChatCompletionCustomToolParam is unavailable before.", @@ -686,13 +711,17 @@ def test_nonstreaming_chat_completion( pytest.param( {"gen_ai": {"inputs": False}}, True, - {}, - [ - SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS, - SPANDATA.GEN_AI_REQUEST_MESSAGES, - SPANDATA.GEN_AI_TOOL_DEFINITIONS, - ], - id="inputs-disabled", + { + SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS: json.dumps( + [{"type": "text", "content": "You are a helpful assistant."}] + ), + SPANDATA.GEN_AI_REQUEST_MESSAGES: safe_serialize( + [{"role": "user", "content": "hello"}] + ), + SPANDATA.GEN_AI_TOOL_DEFINITIONS: safe_serialize(EXAMPLE_TOOLS), + }, + [], + id="include-prompts-enabled-overrides-inputs-disabled", ), pytest.param( {}, @@ -720,6 +749,47 @@ def test_nonstreaming_chat_completion( ], id="include-prompts-disabled-overrides-inputs-enabled", ), + pytest.param( + {"gen_ai": {"inputs": True}}, + None, + { + SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS: json.dumps( + [{"type": "text", "content": "You are a helpful assistant."}] + ), + SPANDATA.GEN_AI_REQUEST_MESSAGES: safe_serialize( + [{"role": "user", "content": "hello"}] + ), + SPANDATA.GEN_AI_TOOL_DEFINITIONS: safe_serialize(EXAMPLE_TOOLS), + }, + [], + id="include-prompts-default-follows-inputs-enabled", + ), + pytest.param( + {"gen_ai": {"inputs": False}}, + None, + {}, + [ + SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS, + SPANDATA.GEN_AI_REQUEST_MESSAGES, + SPANDATA.GEN_AI_TOOL_DEFINITIONS, + ], + id="include-prompts-default-follows-inputs-disabled", + ), + pytest.param( + {}, + None, + { + SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS: json.dumps( + [{"type": "text", "content": "You are a helpful assistant."}] + ), + SPANDATA.GEN_AI_REQUEST_MESSAGES: safe_serialize( + [{"role": "user", "content": "hello"}] + ), + SPANDATA.GEN_AI_TOOL_DEFINITIONS: safe_serialize(EXAMPLE_TOOLS), + }, + [], + id="include-prompts-default-follows-default-enabled", + ), ], ) def test_completions_api_data_collection( @@ -2980,6 +3050,227 @@ def test_embeddings_create( assert span["data"]["gen_ai.usage.total_tokens"] == 30 +def _collect_embeddings_span_data( + capture_events, capture_items, span_streaming, stream_gen_ai_spans, create +): + if span_streaming or stream_gen_ai_spans: + items = capture_items("span") + + with start_transaction(name="openai tx"): + response = create() + + assert len(response.data[0].embedding) == 3 + + sentry_sdk.flush() + span = next(item.payload for item in items) + assert span["attributes"]["sentry.op"] == "gen_ai.embeddings" + return span["attributes"] + + events = capture_events() + + with start_transaction(name="openai tx"): + response = create() + + assert len(response.data[0].embedding) == 3 + + tx = events[0] + assert tx["type"] == "transaction" + span = tx["spans"][0] + assert span["op"] == "gen_ai.embeddings" + return span["data"] + + +@pytest.mark.parametrize("span_streaming", [True, False]) +@pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) +@pytest.mark.parametrize( + "data_collection,send_default_pii,include_prompts,expect_input", + [ + pytest.param( + {"gen_ai": {"inputs": True}}, + False, + True, + True, + id="inputs-enabled-overrides-pii-disabled", + ), + pytest.param( + {"gen_ai": {"inputs": False}}, + True, + True, + True, + id="include-prompts-overrides-inputs-disabled", + ), + pytest.param( + {}, + False, + True, + True, + id="gen-ai-omitted-defaults-to-enabled", + ), + pytest.param( + {"gen_ai": {"inputs": False}}, + False, + True, + True, + id="include-prompts-overrides-inputs-disabled-and-pii-disabled", + ), + pytest.param( + {"gen_ai": {"inputs": True}}, + True, + False, + False, + id="include-prompts-disabled-overrides-inputs-enabled", + ), + pytest.param( + None, + False, + True, + False, + id="no-experiment-falls-back-to-pii", + ), + pytest.param( + {"gen_ai": {"inputs": True}}, + False, + None, + True, + id="include-prompts-default-follows-inputs-enabled", + ), + pytest.param( + {"gen_ai": {"inputs": False}}, + True, + None, + False, + id="include-prompts-default-follows-inputs-disabled", + ), + pytest.param( + {}, + False, + None, + True, + id="include-prompts-default-follows-default-enabled", + ), + ], +) +def test_embeddings_create_data_collection( + sentry_init, + capture_events, + capture_items, + data_collection, + send_default_pii, + include_prompts, + expect_input, + stream_gen_ai_spans, + span_streaming, +): + init_kwargs = { + "integrations": [OpenAIIntegration(include_prompts=include_prompts)], + "disabled_integrations": [StdlibIntegration], + "traces_sample_rate": 1.0, + "send_default_pii": send_default_pii, + "stream_gen_ai_spans": stream_gen_ai_spans, + "trace_lifecycle": "stream" if span_streaming else "static", + } + + sentry_init_kwargs = dict(init_kwargs) + if data_collection is not None: + sentry_init_kwargs["_experiments"] = {"data_collection": data_collection} + + sentry_init(**sentry_init_kwargs) + + client = OpenAI(api_key="z") + + returned_embedding = CreateEmbeddingResponse( + data=[Embedding(object="embedding", index=0, embedding=[1.0, 2.0, 3.0])], + model="some-model", + object="list", + usage=EmbeddingTokenUsage( + prompt_tokens=20, + total_tokens=30, + ), + ) + + client.embeddings._post = mock.Mock(return_value=returned_embedding) + + span_data = _collect_embeddings_span_data( + capture_events, + capture_items, + span_streaming, + stream_gen_ai_spans, + lambda: client.embeddings.create(input="hello", model="text-embedding-3-large"), + ) + + assert span_data[SPANDATA.GEN_AI_SYSTEM] == "openai" + assert span_data[SPANDATA.GEN_AI_OPERATION_NAME] == "embeddings" + assert span_data[SPANDATA.GEN_AI_REQUEST_MODEL] == "text-embedding-3-large" + + if expect_input: + assert json.loads(span_data[SPANDATA.GEN_AI_EMBEDDINGS_INPUT]) == ["hello"] + else: + assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT not in span_data + + assert span_data["gen_ai.usage.input_tokens"] == 20 + assert span_data["gen_ai.usage.total_tokens"] == 30 + + +@pytest.mark.parametrize("span_streaming", [True, False]) +@pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) +@pytest.mark.parametrize( + "get_input", + [ + lambda: "hello", + lambda: ["First text", "Second text"], + lambda: iter(["First text", "Second text"]), + lambda: [5, 8, 13, 21, 34], + lambda: [[5, 8, 13], [8, 13, 21]], + lambda: {"text": "hello"}, + ], +) +def test_embeddings_create_data_collection_inputs_disabled_input_shapes( + sentry_init, + capture_events, + capture_items, + get_input, + stream_gen_ai_spans, + span_streaming, +): + sentry_init( + integrations=[OpenAIIntegration(include_prompts=False)], + disabled_integrations=[StdlibIntegration], + traces_sample_rate=1.0, + send_default_pii=True, + stream_gen_ai_spans=stream_gen_ai_spans, + trace_lifecycle="stream" if span_streaming else "static", + _experiments={"data_collection": {"gen_ai": {"inputs": False}}}, + ) + + client = OpenAI(api_key="z") + + returned_embedding = CreateEmbeddingResponse( + data=[Embedding(object="embedding", index=0, embedding=[1.0, 2.0, 3.0])], + model="some-model", + object="list", + usage=EmbeddingTokenUsage( + prompt_tokens=20, + total_tokens=30, + ), + ) + + client.embeddings._post = mock.Mock(return_value=returned_embedding) + + span_data = _collect_embeddings_span_data( + capture_events, + capture_items, + span_streaming, + stream_gen_ai_spans, + lambda: client.embeddings.create( + input=get_input(), model="text-embedding-3-large" + ), + ) + + assert span_data[SPANDATA.GEN_AI_OPERATION_NAME] == "embeddings" + assert span_data[SPANDATA.GEN_AI_REQUEST_MODEL] == "text-embedding-3-large" + assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT not in span_data + + @pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) @pytest.mark.asyncio @@ -3225,6 +3516,153 @@ async def test_embeddings_create_async( assert span["data"]["gen_ai.usage.total_tokens"] == 30 +@pytest.mark.parametrize("span_streaming", [True, False]) +@pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) +@pytest.mark.asyncio +@pytest.mark.parametrize( + "data_collection,send_default_pii,include_prompts,expect_input", + [ + pytest.param( + {"gen_ai": {"inputs": True}}, + False, + True, + True, + id="inputs-enabled-overrides-pii-disabled", + ), + pytest.param( + {"gen_ai": {"inputs": False}}, + True, + True, + True, + id="include-prompts-overrides-inputs-disabled", + ), + pytest.param( + {}, + False, + True, + True, + id="gen-ai-omitted-defaults-to-enabled", + ), + pytest.param( + {"gen_ai": {"inputs": True}}, + True, + False, + False, + id="include-prompts-disabled-overrides-inputs-enabled", + ), + pytest.param( + None, + False, + True, + False, + id="no-experiment-falls-back-to-pii", + ), + pytest.param( + {"gen_ai": {"inputs": True}}, + False, + None, + True, + id="include-prompts-default-follows-inputs-enabled", + ), + pytest.param( + {"gen_ai": {"inputs": False}}, + True, + None, + False, + id="include-prompts-default-follows-inputs-disabled", + ), + pytest.param( + {}, + False, + None, + True, + id="include-prompts-default-follows-default-enabled", + ), + ], +) +async def test_embeddings_create_async_data_collection( + sentry_init, + capture_events, + capture_items, + data_collection, + send_default_pii, + include_prompts, + expect_input, + stream_gen_ai_spans, + span_streaming, +): + init_kwargs = { + "integrations": [OpenAIIntegration(include_prompts=include_prompts)], + "disabled_integrations": [StdlibIntegration], + "traces_sample_rate": 1.0, + "send_default_pii": send_default_pii, + "stream_gen_ai_spans": stream_gen_ai_spans, + "trace_lifecycle": "stream" if span_streaming else "static", + } + + sentry_init_kwargs = dict(init_kwargs) + if data_collection is not None: + sentry_init_kwargs["_experiments"] = {"data_collection": data_collection} + + sentry_init(**sentry_init_kwargs) + + client = AsyncOpenAI(api_key="z") + + returned_embedding = CreateEmbeddingResponse( + data=[Embedding(object="embedding", index=0, embedding=[1.0, 2.0, 3.0])], + model="some-model", + object="list", + usage=EmbeddingTokenUsage( + prompt_tokens=20, + total_tokens=30, + ), + ) + + client.embeddings._post = AsyncMock(return_value=returned_embedding) + + if span_streaming or stream_gen_ai_spans: + items = capture_items("span") + + with start_transaction(name="openai tx"): + response = await client.embeddings.create( + input="hello", model="text-embedding-3-large" + ) + + assert len(response.data[0].embedding) == 3 + + sentry_sdk.flush() + span = next(item.payload for item in items) + assert span["attributes"]["sentry.op"] == "gen_ai.embeddings" + span_data = span["attributes"] + else: + events = capture_events() + + with start_transaction(name="openai tx"): + response = await client.embeddings.create( + input="hello", model="text-embedding-3-large" + ) + + assert len(response.data[0].embedding) == 3 + + tx = events[0] + assert tx["type"] == "transaction" + span = tx["spans"][0] + assert span["op"] == "gen_ai.embeddings" + span_data = span["data"] + + assert span_data[SPANDATA.GEN_AI_SYSTEM] == "openai" + assert span_data[SPANDATA.GEN_AI_OPERATION_NAME] == "embeddings" + assert span_data[SPANDATA.GEN_AI_REQUEST_MODEL] == "text-embedding-3-large" + + if expect_input: + assert json.loads(span_data[SPANDATA.GEN_AI_EMBEDDINGS_INPUT]) == ["hello"] + else: + assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT not in span_data + + assert span_data["gen_ai.usage.input_tokens"] == 20 + assert span_data["gen_ai.usage.total_tokens"] == 30 + + @pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) @pytest.mark.parametrize( @@ -4967,14 +5405,23 @@ def test_ai_client_span_responses_api( "input": "How do I check if a Python object is an instance of a class?", "tools": EXAMPLE_TOOLS, }, - {}, - [ - SPANDATA.GEN_AI_REQUEST_MESSAGES, - SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS, - SPANDATA.GEN_AI_TOOL_DEFINITIONS, - ], + { + SPANDATA.GEN_AI_REQUEST_MESSAGES: safe_serialize( + ["How do I check if a Python object is an instance of a class?"] + ), + SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS: safe_serialize( + [ + { + "type": "text", + "content": "You are a coding assistant that talks like a pirate.", + } + ] + ), + SPANDATA.GEN_AI_TOOL_DEFINITIONS: safe_serialize(EXAMPLE_TOOLS), + }, + [], True, - id="inputs-disabled", + id="include-prompts-enabled-overrides-inputs-disabled", ), pytest.param( {}, @@ -5020,6 +5467,63 @@ def test_ai_client_span_responses_api( False, id="include-prompts-disabled-overrides-inputs-enabled", ), + pytest.param( + {"gen_ai": {"inputs": True}}, + { + "instructions": "You are a coding assistant that talks like a pirate.", + "input": "How do I check if a Python object is an instance of a class?", + "tools": EXAMPLE_TOOLS, + }, + { + SPANDATA.GEN_AI_REQUEST_MESSAGES: safe_serialize( + ["How do I check if a Python object is an instance of a class?"] + ), + SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS: safe_serialize( + [ + { + "type": "text", + "content": "You are a coding assistant that talks like a pirate.", + } + ] + ), + SPANDATA.GEN_AI_TOOL_DEFINITIONS: safe_serialize(EXAMPLE_TOOLS), + }, + [], + None, + id="include-prompts-default-follows-inputs-enabled", + ), + pytest.param( + {"gen_ai": {"inputs": False}}, + { + "instructions": "You are a coding assistant that talks like a pirate.", + "input": "How do I check if a Python object is an instance of a class?", + "tools": EXAMPLE_TOOLS, + }, + {}, + [ + SPANDATA.GEN_AI_REQUEST_MESSAGES, + SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS, + SPANDATA.GEN_AI_TOOL_DEFINITIONS, + ], + None, + id="include-prompts-default-follows-inputs-disabled", + ), + pytest.param( + {}, + { + "input": "How do I check if a Python object is an instance of a class?", + "tools": EXAMPLE_TOOLS, + }, + { + SPANDATA.GEN_AI_REQUEST_MESSAGES: safe_serialize( + ["How do I check if a Python object is an instance of a class?"] + ), + SPANDATA.GEN_AI_TOOL_DEFINITIONS: safe_serialize(EXAMPLE_TOOLS), + }, + [SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS], + None, + id="include-prompts-default-follows-default-enabled", + ), ], ) @pytest.mark.skipif(SKIP_RESPONSES_TESTS, reason="Responses API not available")