Skip to content

Commit 463fd1e

Browse files
codefromthecryptcodefromthecrypt
committed
feat(openai): standardize completions to indexed attribute format
Signed-off-by: Adrian Cole <[email protected]>
1 parent b827f3d commit 463fd1e

File tree

9 files changed

+177
-10
lines changed

9 files changed

+177
-10
lines changed

‎python/instrumentation/openinference-instrumentation-openai/src/openinference/instrumentation/openai/_request_attributes_extractor.py‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,8 @@
3232

3333
# TODO: Update to use SpanAttributes.EMBEDDING_INVOCATION_PARAMETERS when released in semconv
3434
_EMBEDDING_INVOCATION_PARAMETERS = "embedding.invocation_parameters"
35+
# TODO: Update to use SpanAttributes.LLM_INPUT_PROMPTS when released in semconv
36+
_LLM_INPUT_PROMPTS = "llm.input_prompts"
3537

3638
if TYPE_CHECKING:
3739
from openai.types import Completion, CreateEmbeddingResponse
@@ -226,13 +228,14 @@ def _get_attributes_from_completion_create_param(
226228

227229
model_prompt = params.get("prompt")
228230
if isinstance(model_prompt, str):
229-
yield SpanAttributes.LLM_PROMPTS, [model_prompt]
231+
yield f"{_LLM_INPUT_PROMPTS}.0", model_prompt
230232
elif (
231233
isinstance(model_prompt, list)
232234
and model_prompt
233235
and all(isinstance(item, str) for item in model_prompt)
234236
):
235-
yield SpanAttributes.LLM_PROMPTS, model_prompt
237+
for index, prompt in enumerate(model_prompt):
238+
yield f"{_LLM_INPUT_PROMPTS}.{index}", prompt
236239

237240

238241
def _get_attributes_from_embedding_create_param(

‎python/instrumentation/openinference-instrumentation-openai/src/openinference/instrumentation/openai/_response_attributes_extractor.py‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,11 @@
2525
ToolCallAttributes,
2626
)
2727

28+
# TODO: Update to use SpanAttributes.LLM_OUTPUT_CHOICES when released in semconv
29+
_LLM_OUTPUT_CHOICES = "llm.output_choices"
30+
# TODO: Update to use ChoiceAttributes.CHOICE_TEXT when released in semconv
31+
_CHOICE_TEXT = "choice.text"
32+
2833
if TYPE_CHECKING:
2934
from openai.types import Completion, CreateEmbeddingResponse
3035
from openai.types.chat import ChatCompletion
@@ -114,6 +119,13 @@ def _get_attributes_from_completion(
114119
if usage := getattr(completion, "usage", None):
115120
yield from self._get_attributes_from_completion_usage(usage)
116121

122+
if (choices := getattr(completion, "choices", None)) and isinstance(choices, Iterable):
123+
for choice in choices:
124+
if (index := getattr(choice, "index", None)) is None:
125+
continue
126+
if text := getattr(choice, "text", None):
127+
yield f"{_LLM_OUTPUT_CHOICES}.{index}.{_CHOICE_TEXT}", text
128+
117129
def _get_attributes_from_create_embedding_response(
118130
self,
119131
response: "CreateEmbeddingResponse",

‎python/instrumentation/openinference-instrumentation-openai/tests/openinference/instrumentation/openai/test_instrumentor.py‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,6 @@
1616
List,
1717
Mapping,
1818
Optional,
19-
Sequence,
2019
Tuple,
2120
Union,
2221
cast,
@@ -300,8 +299,8 @@ def test_completions(
300299
prompt_template_version: str,
301300
prompt_template_variables: Dict[str, Any],
302301
) -> None:
303-
# SpanAttributes.LLM_PROMPTS is always a list, so coerce the input accordingly.
304-
prompt = prompt_input if isinstance(prompt_input, list) else [prompt_input]
302+
# Normalize prompt_input (string or list) to a list for iteration
303+
prompts = prompt_input if isinstance(prompt_input, list) else [prompt_input]
305304
output_texts: List[str] = completion_mock_stream[1] if is_stream else get_texts()
306305
invocation_parameters = {
307306
"stream": is_stream,
@@ -399,10 +398,14 @@ async def task() -> None:
399398
assert isinstance(attributes.pop(INPUT_VALUE, None), str)
400399
assert isinstance(attributes.pop(INPUT_MIME_TYPE, None), str)
401400
# Prompts are recorded in request phase, so present regardless of status
402-
assert list(cast(Sequence[str], attributes.pop(LLM_PROMPTS, None))) == prompt
401+
for i, prompt_text in enumerate(prompts):
402+
assert attributes.pop(f"llm.input_prompts.{i}", None) == prompt_text
403403
if status_code == 200:
404404
assert isinstance(attributes.pop(OUTPUT_VALUE, None), str)
405405
assert isinstance(attributes.pop(OUTPUT_MIME_TYPE, None), str)
406+
# Check output choices
407+
for i, text in enumerate(output_texts):
408+
assert attributes.pop(f"llm.output_choices.{i}.choice.text", None) == text
406409
if not is_stream:
407410
# Usage is not available for streaming in general.
408411
assert attributes.pop(LLM_TOKEN_COUNT_TOTAL, None) == completion_usage["total_tokens"]

‎python/openinference-instrumentation/src/openinference/instrumentation/config.py‎

Lines changed: 47 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -84,12 +84,22 @@ def __aexit__(
8484
OPENINFERENCE_BASE64_IMAGE_MAX_LENGTH = "OPENINFERENCE_BASE64_IMAGE_MAX_LENGTH"
8585
# Limits characters of a base64 encoding of an image
8686
OPENINFERENCE_HIDE_PROMPTS = "OPENINFERENCE_HIDE_PROMPTS"
87-
# Hides LLM prompts
87+
# DEPRECATED: Use OPENINFERENCE_HIDE_INPUT_PROMPTS instead
88+
OPENINFERENCE_HIDE_INPUT_PROMPTS = "OPENINFERENCE_HIDE_INPUT_PROMPTS"
89+
# Hides input prompts
90+
OPENINFERENCE_HIDE_OUTPUT_CHOICES = "OPENINFERENCE_HIDE_OUTPUT_CHOICES"
91+
# Hides output choices
8892
REDACTED_VALUE = "__REDACTED__"
8993
# When a value is hidden, it will be replaced by this redacted value
9094

95+
# TODO: Update to use SpanAttributes constants when released in semconv
96+
_LLM_INPUT_PROMPTS = "llm.input_prompts"
97+
_LLM_OUTPUT_CHOICES = "llm.output_choices"
98+
9199
DEFAULT_HIDE_LLM_INVOCATION_PARAMETERS = False
92100
DEFAULT_HIDE_PROMPTS = False
101+
DEFAULT_HIDE_INPUT_PROMPTS = False
102+
DEFAULT_HIDE_OUTPUT_CHOICES = False
93103
DEFAULT_HIDE_INPUTS = False
94104
DEFAULT_HIDE_OUTPUTS = False
95105

@@ -195,7 +205,23 @@ class TraceConfig:
195205
"default_value": DEFAULT_HIDE_PROMPTS,
196206
},
197207
)
198-
"""Hides LLM prompts"""
208+
"""Hides LLM prompts (DEPRECATED: use hide_input_prompts)"""
209+
hide_input_prompts: Optional[bool] = field(
210+
default=None,
211+
metadata={
212+
"env_var": OPENINFERENCE_HIDE_INPUT_PROMPTS,
213+
"default_value": DEFAULT_HIDE_INPUT_PROMPTS,
214+
},
215+
)
216+
"""Hides input prompts"""
217+
hide_output_choices: Optional[bool] = field(
218+
default=None,
219+
metadata={
220+
"env_var": OPENINFERENCE_HIDE_OUTPUT_CHOICES,
221+
"default_value": DEFAULT_HIDE_OUTPUT_CHOICES,
222+
},
223+
)
224+
"""Hides output choices"""
199225
base64_image_max_length: Optional[int] = field(
200226
default=None,
201227
metadata={
@@ -206,6 +232,12 @@ class TraceConfig:
206232
"""Limits characters of a base64 encoding of an image"""
207233

208234
def __post_init__(self) -> None:
235+
# Track if hide_input_prompts was explicitly set (not None)
236+
# to avoid overriding explicit user configuration
237+
hide_input_prompts_was_explicit = self.hide_input_prompts is not None
238+
# Also check if it's set via environment variable
239+
hide_input_prompts_env_was_set = os.getenv(OPENINFERENCE_HIDE_INPUT_PROMPTS) is not None
240+
209241
for f in fields(self):
210242
expected_type = get_args(f.type)[0]
211243
# Optional is Union[T,NoneType]. get_args()returns (T, NoneType).
@@ -216,6 +248,15 @@ def __post_init__(self) -> None:
216248
f.metadata["env_var"],
217249
f.metadata["default_value"],
218250
)
251+
# TODO: Remove this backward compatibility after deprecation period
252+
# Only apply backward compat if hide_input_prompts wasn't explicitly set
253+
# via constructor argument or environment variable
254+
if (
255+
not hide_input_prompts_was_explicit
256+
and not hide_input_prompts_env_was_set
257+
and self.hide_prompts is True
258+
):
259+
object.__setattr__(self, "hide_input_prompts", True)
219260

220261
def mask(
221262
self,
@@ -226,6 +267,8 @@ def mask(
226267
return None
227268
elif self.hide_prompts and key == SpanAttributes.LLM_PROMPTS:
228269
value = REDACTED_VALUE
270+
elif (self.hide_inputs or self.hide_input_prompts) and _LLM_INPUT_PROMPTS in key:
271+
value = REDACTED_VALUE
229272
elif self.hide_inputs and key == SpanAttributes.INPUT_VALUE:
230273
value = REDACTED_VALUE
231274
elif self.hide_inputs and key == SpanAttributes.INPUT_MIME_TYPE:
@@ -242,6 +285,8 @@ def mask(
242285
self.hide_outputs or self.hide_output_messages
243286
) and SpanAttributes.LLM_OUTPUT_MESSAGES in key:
244287
return None
288+
elif (self.hide_outputs or self.hide_output_choices) and _LLM_OUTPUT_CHOICES in key:
289+
value = REDACTED_VALUE
245290
elif (
246291
self.hide_input_text
247292
and SpanAttributes.LLM_INPUT_MESSAGES in key

‎python/openinference-instrumentation/tests/test_config.py‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,18 +17,22 @@
1717
DEFAULT_BASE64_IMAGE_MAX_LENGTH,
1818
DEFAULT_HIDE_INPUT_IMAGES,
1919
DEFAULT_HIDE_INPUT_MESSAGES,
20+
DEFAULT_HIDE_INPUT_PROMPTS,
2021
DEFAULT_HIDE_INPUT_TEXT,
2122
DEFAULT_HIDE_INPUTS,
2223
DEFAULT_HIDE_LLM_INVOCATION_PARAMETERS,
24+
DEFAULT_HIDE_OUTPUT_CHOICES,
2325
DEFAULT_HIDE_OUTPUT_MESSAGES,
2426
DEFAULT_HIDE_OUTPUT_TEXT,
2527
DEFAULT_HIDE_OUTPUTS,
2628
DEFAULT_HIDE_PROMPTS,
2729
OPENINFERENCE_BASE64_IMAGE_MAX_LENGTH,
2830
OPENINFERENCE_HIDE_INPUT_IMAGES,
2931
OPENINFERENCE_HIDE_INPUT_MESSAGES,
32+
OPENINFERENCE_HIDE_INPUT_PROMPTS,
3033
OPENINFERENCE_HIDE_INPUT_TEXT,
3134
OPENINFERENCE_HIDE_INPUTS,
35+
OPENINFERENCE_HIDE_OUTPUT_CHOICES,
3236
OPENINFERENCE_HIDE_OUTPUT_MESSAGES,
3337
OPENINFERENCE_HIDE_OUTPUT_TEXT,
3438
OPENINFERENCE_HIDE_OUTPUTS,
@@ -49,6 +53,8 @@ def test_default_settings() -> None:
4953
assert config.hide_input_text == DEFAULT_HIDE_INPUT_TEXT
5054
assert config.hide_output_text == DEFAULT_HIDE_OUTPUT_TEXT
5155
assert config.hide_prompts == DEFAULT_HIDE_PROMPTS
56+
assert config.hide_input_prompts == DEFAULT_HIDE_INPUT_PROMPTS
57+
assert config.hide_output_choices == DEFAULT_HIDE_OUTPUT_CHOICES
5258
assert config.base64_image_max_length == DEFAULT_BASE64_IMAGE_MAX_LENGTH
5359

5460

@@ -121,6 +127,8 @@ def test_attribute_priority(k: str, in_memory_span_exporter: InMemorySpanExporte
121127
@pytest.mark.parametrize("hide_input_text", [False, True])
122128
@pytest.mark.parametrize("hide_output_text", [False, True])
123129
@pytest.mark.parametrize("hide_prompts", [False, True])
130+
@pytest.mark.parametrize("hide_input_prompts", [False, True])
131+
@pytest.mark.parametrize("hide_output_choices", [False, True])
124132
@pytest.mark.parametrize("base64_image_max_length", [10_000])
125133
def test_settings_from_env_vars_and_code(
126134
hide_inputs: bool,
@@ -131,6 +139,8 @@ def test_settings_from_env_vars_and_code(
131139
hide_input_text: bool,
132140
hide_output_text: bool,
133141
hide_prompts: bool,
142+
hide_input_prompts: bool,
143+
hide_output_choices: bool,
134144
base64_image_max_length: int,
135145
monkeypatch: pytest.MonkeyPatch,
136146
) -> None:
@@ -141,6 +151,8 @@ def test_settings_from_env_vars_and_code(
141151
monkeypatch.setenv(OPENINFERENCE_HIDE_OUTPUT_MESSAGES, str(hide_output_messages))
142152
monkeypatch.setenv(OPENINFERENCE_HIDE_INPUT_IMAGES, str(hide_input_images))
143153
monkeypatch.setenv(OPENINFERENCE_HIDE_PROMPTS, str(hide_prompts))
154+
monkeypatch.setenv(OPENINFERENCE_HIDE_INPUT_PROMPTS, str(hide_input_prompts))
155+
monkeypatch.setenv(OPENINFERENCE_HIDE_OUTPUT_CHOICES, str(hide_output_choices))
144156
monkeypatch.setenv(OPENINFERENCE_HIDE_INPUT_TEXT, str(hide_input_text))
145157
monkeypatch.setenv(OPENINFERENCE_HIDE_OUTPUT_TEXT, str(hide_output_text))
146158
monkeypatch.setenv(OPENINFERENCE_BASE64_IMAGE_MAX_LENGTH, str(base64_image_max_length))
@@ -154,6 +166,8 @@ def test_settings_from_env_vars_and_code(
154166
assert config.hide_input_text is parse_bool_from_env(OPENINFERENCE_HIDE_INPUT_TEXT)
155167
assert config.hide_output_text is parse_bool_from_env(OPENINFERENCE_HIDE_OUTPUT_TEXT)
156168
assert config.hide_prompts is parse_bool_from_env(OPENINFERENCE_HIDE_PROMPTS)
169+
assert config.hide_input_prompts is parse_bool_from_env(OPENINFERENCE_HIDE_INPUT_PROMPTS)
170+
assert config.hide_output_choices is parse_bool_from_env(OPENINFERENCE_HIDE_OUTPUT_CHOICES)
157171
assert config.base64_image_max_length == int(
158172
os.getenv(OPENINFERENCE_BASE64_IMAGE_MAX_LENGTH, default=-1)
159173
)
@@ -169,6 +183,8 @@ def test_settings_from_env_vars_and_code(
169183
new_hide_input_text = not hide_input_text
170184
new_hide_output_text = not hide_output_text
171185
new_hide_prompts = not hide_prompts
186+
new_hide_input_prompts = not hide_input_prompts
187+
new_hide_output_choices = not hide_output_choices
172188
config = TraceConfig(
173189
hide_inputs=new_hide_inputs,
174190
hide_outputs=new_hide_outputs,
@@ -178,6 +194,8 @@ def test_settings_from_env_vars_and_code(
178194
hide_input_text=new_hide_input_text,
179195
hide_output_text=new_hide_output_text,
180196
hide_prompts=new_hide_prompts,
197+
hide_input_prompts=new_hide_input_prompts,
198+
hide_output_choices=new_hide_output_choices,
181199
base64_image_max_length=new_base64_image_max_length,
182200
)
183201
assert config.hide_inputs is new_hide_inputs
@@ -188,6 +206,8 @@ def test_settings_from_env_vars_and_code(
188206
assert config.hide_input_text is new_hide_input_text
189207
assert config.hide_output_text is new_hide_output_text
190208
assert config.hide_prompts is new_hide_prompts
209+
assert config.hide_input_prompts is new_hide_input_prompts
210+
assert config.hide_output_choices is new_hide_output_choices
191211
assert config.base64_image_max_length == new_base64_image_max_length
192212

193213

‎python/openinference-semantic-conventions/src/openinference/semconv/trace/__init__.py‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,17 @@ class SpanAttributes:
5757
LLM_PROMPTS = "llm.prompts"
5858
"""
5959
Prompts provided to a completions API.
60+
DEPRECATED: Use LLM_INPUT_PROMPTS instead for indexed format.
61+
"""
62+
LLM_INPUT_PROMPTS = "llm.input_prompts"
63+
"""
64+
Prompt(s) provided to a completions API. Use indexed format for arrays.
65+
Maps to the 'prompt' field in the request (e.g., request.prompt or request.prompt[0]).
66+
"""
67+
LLM_OUTPUT_CHOICES = "llm.output_choices"
68+
"""
69+
Choice(s) returned from a completions API. Use indexed format for arrays.
70+
Maps to the 'choices' array in the response (e.g., response.choices[0]).
6071
"""
6172
LLM_PROMPT_TEMPLATE = "llm.prompt_template.template"
6273
"""
@@ -293,6 +304,18 @@ class MessageAttributes:
293304
"""
294305

295306

307+
class ChoiceAttributes:
308+
"""
309+
Attributes for a choice returned from a completions API
310+
"""
311+
312+
CHOICE_TEXT = "choice.text"
313+
"""
314+
The text content of a completion choice.
315+
Maps to the 'text' field in the choice (e.g., response.choices[0].text).
316+
"""
317+
318+
296319
class MessageContentAttributes:
297320
"""
298321
Attributes for the contents of user messages sent to an LLM.

‎python/openinference-semantic-conventions/tests/openinference/semconv/test_attributes.py‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@
5454

5555
from openinference.semconv.resource import ResourceAttributes
5656
from openinference.semconv.trace import (
57+
ChoiceAttributes,
5758
DocumentAttributes,
5859
EmbeddingAttributes,
5960
ImageAttributes,
@@ -117,8 +118,10 @@ def test_nesting(self) -> None:
117118
},
118119
"function_call": SpanAttributes.LLM_FUNCTION_CALL,
119120
"input_messages": SpanAttributes.LLM_INPUT_MESSAGES,
121+
"input_prompts": SpanAttributes.LLM_INPUT_PROMPTS,
120122
"invocation_parameters": SpanAttributes.LLM_INVOCATION_PARAMETERS,
121123
"model_name": SpanAttributes.LLM_MODEL_NAME,
124+
"output_choices": SpanAttributes.LLM_OUTPUT_CHOICES,
122125
"output_messages": SpanAttributes.LLM_OUTPUT_MESSAGES,
123126
"prompt_template": {
124127
"template": SpanAttributes.LLM_PROMPT_TEMPLATE,
@@ -330,6 +333,22 @@ def test_nesting(self) -> None:
330333
}
331334

332335

336+
class TestChoiceAttributes:
337+
"""Tests for ChoiceAttributes namespace structure.
338+
339+
Verifies that completion choice attributes from flat spans are properly
340+
organized under the choice namespace.
341+
"""
342+
343+
def test_nesting(self) -> None:
344+
attributes = _get_attributes(ChoiceAttributes)
345+
assert _nested_dict(attributes) == {
346+
"choice": {
347+
"text": ChoiceAttributes.CHOICE_TEXT,
348+
}
349+
}
350+
351+
333352
class TestResourceAttributes:
334353
"""Tests for ResourceAttributes namespace structure.
335354

‎spec/configuration.md‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,10 @@ The possible settings are:
1515
| OPENINFERENCE_HIDE_OUTPUT_MESSAGES | Hides all output messages (independent of HIDE_OUTPUTS) | bool | False |
1616
| OPENINFERENCE_HIDE_INPUT_IMAGES | Hides images from input messages (only applies when input messages are not already hidden) | bool | False |
1717
| OPENINFERENCE_HIDE_INPUT_TEXT | Hides text from input messages (only applies when input messages are not already hidden) | bool | False |
18-
| OPENINFERENCE_HIDE_PROMPTS | Hides LLM prompts | bool | False |
18+
| OPENINFERENCE_HIDE_PROMPTS | DEPRECATED: Use OPENINFERENCE_HIDE_INPUT_PROMPTS instead | bool | False |
19+
| OPENINFERENCE_HIDE_INPUT_PROMPTS | Hides input prompts (for completions API) | bool | False |
1920
| OPENINFERENCE_HIDE_OUTPUT_TEXT | Hides text from output messages (only applies when output messages are not already hidden) | bool | False |
21+
| OPENINFERENCE_HIDE_OUTPUT_CHOICES | Hides output choices (for completions API) | bool | False |
2022
| OPENINFERENCE_HIDE_EMBEDDING_VECTORS | Hides embedding vectors | bool | False |
2123
| OPENINFERENCE_BASE64_IMAGE_MAX_LENGTH | Limits characters of a base64 encoding of an image | int | 32,000 |
2224

@@ -51,7 +53,9 @@ If you are working in Python, and want to set up a configuration different than
5153
hide_output_text=...,
5254
hide_embedding_vectors=...,
5355
base64_image_max_length=...,
54-
hide_prompts=...,
56+
hide_prompts=..., # DEPRECATED: use hide_input_prompts
57+
hide_input_prompts=...,
58+
hide_output_choices=...,
5559
)
5660

5761
from openinference.instrumentation.openai import OpenAIInstrumentor

0 commit comments

Comments
 (0)