Skip to content

Commit b6939d1

Browse files
sabrennersabrenner
authored
fix(openai): fix responses reasoning token metric tracking support for openai>=1.66.0,<1.66.2 (#14072)
I discovered another instance of where we check object attributes when we should be checking dict attributes. This is specifically for older versions of the OpenAI SDK that have support for the Responses API (`openai==1.66.0,<1.66.2`). ## Checklist - [x] PR author has checked that all the criteria below are met - The PR description includes an overview of the change - The PR description articulates the motivation for the change - The change includes tests OR the PR description describes a testing strategy - The PR description notes risks associated with the change, if any - Newly-added code is easy to change - The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) - The change includes or references documentation updates if necessary - Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) ## Reviewer Checklist - [x] Reviewer has checked that all the criteria below are met - Title is accurate - All changes are related to the pull request's stated goal - Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - Testing strategy adequately addresses listed risks - Newly-added code is easy to change - Release note makes sense to a user of the library - If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)
1 parent bb588c7 commit b6939d1

File tree

10 files changed

+312
-37
lines changed

10 files changed

+312
-37
lines changed

‎.riot/requirements/1911f94.txt

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
#
2+
# This file is autogenerated by pip-compile with Python 3.10
3+
# by the following command:
4+
#
5+
# pip-compile --allow-unsafe --no-annotate .riot/requirements/1911f94.in
6+
#
7+
annotated-types==0.7.0
8+
anyio==4.9.0
9+
attrs==25.3.0
10+
certifi==2025.7.14
11+
charset-normalizer==3.4.2
12+
coverage[toml]==7.9.2
13+
distro==1.9.0
14+
exceptiongroup==1.3.0
15+
h11==0.16.0
16+
httpcore==1.0.9
17+
httpx==0.28.1
18+
hypothesis==6.45.0
19+
idna==3.10
20+
iniconfig==2.1.0
21+
jiter==0.10.0
22+
mock==5.2.0
23+
multidict==6.6.3
24+
openai==1.66.0
25+
opentracing==2.4.0
26+
packaging==25.0
27+
pillow==11.3.0
28+
pluggy==1.6.0
29+
propcache==0.3.2
30+
pydantic==2.11.7
31+
pydantic-core==2.33.2
32+
pygments==2.19.2
33+
pytest==8.4.1
34+
pytest-asyncio==0.21.1
35+
pytest-cov==6.2.1
36+
pytest-mock==3.14.1
37+
pytest-randomly==3.16.0
38+
pyyaml==6.0.2
39+
regex==2024.11.6
40+
requests==2.32.4
41+
sniffio==1.3.1
42+
sortedcontainers==2.4.0
43+
tiktoken==0.9.0
44+
tomli==2.2.1
45+
tqdm==4.67.1
46+
typing-extensions==4.14.1
47+
typing-inspection==0.4.1
48+
urllib3==1.26.20
49+
vcrpy==7.0.0
50+
wrapt==1.17.2
51+
yarl==1.20.1

‎.riot/requirements/1a0657d.txt

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
#
2+
# This file is autogenerated by pip-compile with Python 3.13
3+
# by the following command:
4+
#
5+
# pip-compile --allow-unsafe --no-annotate .riot/requirements/1a0657d.in
6+
#
7+
annotated-types==0.7.0
8+
anyio==4.9.0
9+
attrs==25.3.0
10+
certifi==2025.7.14
11+
charset-normalizer==3.4.2
12+
coverage[toml]==7.9.2
13+
distro==1.9.0
14+
h11==0.16.0
15+
httpcore==1.0.9
16+
httpx==0.28.1
17+
hypothesis==6.45.0
18+
idna==3.10
19+
iniconfig==2.1.0
20+
jiter==0.10.0
21+
mock==5.2.0
22+
multidict==6.6.3
23+
openai==1.66.0
24+
opentracing==2.4.0
25+
packaging==25.0
26+
pillow==11.3.0
27+
pluggy==1.6.0
28+
propcache==0.3.2
29+
pydantic==2.11.7
30+
pydantic-core==2.33.2
31+
pygments==2.19.2
32+
pytest==8.4.1
33+
pytest-asyncio==0.21.1
34+
pytest-cov==6.2.1
35+
pytest-mock==3.14.1
36+
pytest-randomly==3.16.0
37+
pyyaml==6.0.2
38+
regex==2024.11.6
39+
requests==2.32.4
40+
sniffio==1.3.1
41+
sortedcontainers==2.4.0
42+
tiktoken==0.9.0
43+
tqdm==4.67.1
44+
typing-extensions==4.14.1
45+
typing-inspection==0.4.1
46+
urllib3==1.26.20
47+
vcrpy==7.0.0
48+
wrapt==1.17.2
49+
yarl==1.20.1

‎.riot/requirements/85ff44d.txt

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
#
2+
# This file is autogenerated by pip-compile with Python 3.12
3+
# by the following command:
4+
#
5+
# pip-compile --allow-unsafe --no-annotate .riot/requirements/85ff44d.in
6+
#
7+
annotated-types==0.7.0
8+
anyio==4.9.0
9+
attrs==25.3.0
10+
certifi==2025.7.14
11+
charset-normalizer==3.4.2
12+
coverage[toml]==7.9.2
13+
distro==1.9.0
14+
h11==0.16.0
15+
httpcore==1.0.9
16+
httpx==0.28.1
17+
hypothesis==6.45.0
18+
idna==3.10
19+
iniconfig==2.1.0
20+
jiter==0.10.0
21+
mock==5.2.0
22+
multidict==6.6.3
23+
openai==1.66.0
24+
opentracing==2.4.0
25+
packaging==25.0
26+
pillow==11.3.0
27+
pluggy==1.6.0
28+
propcache==0.3.2
29+
pydantic==2.11.7
30+
pydantic-core==2.33.2
31+
pygments==2.19.2
32+
pytest==8.4.1
33+
pytest-asyncio==0.21.1
34+
pytest-cov==6.2.1
35+
pytest-mock==3.14.1
36+
pytest-randomly==3.16.0
37+
pyyaml==6.0.2
38+
regex==2024.11.6
39+
requests==2.32.4
40+
sniffio==1.3.1
41+
sortedcontainers==2.4.0
42+
tiktoken==0.9.0
43+
tqdm==4.67.1
44+
typing-extensions==4.14.1
45+
typing-inspection==0.4.1
46+
urllib3==1.26.20
47+
vcrpy==7.0.0
48+
wrapt==1.17.2
49+
yarl==1.20.1

‎.riot/requirements/b68c552.txt

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
#
2+
# This file is autogenerated by pip-compile with Python 3.8
3+
# by the following command:
4+
#
5+
# pip-compile --allow-unsafe --no-annotate .riot/requirements/b68c552.in
6+
#
7+
annotated-types==0.7.0
8+
anyio==4.5.2
9+
attrs==25.3.0
10+
certifi==2025.7.14
11+
charset-normalizer==3.4.2
12+
coverage[toml]==7.6.1
13+
distro==1.9.0
14+
exceptiongroup==1.3.0
15+
h11==0.16.0
16+
httpcore==1.0.9
17+
httpx==0.28.1
18+
hypothesis==6.45.0
19+
idna==3.10
20+
importlib-metadata==8.5.0
21+
iniconfig==2.1.0
22+
jiter==0.9.1
23+
mock==5.2.0
24+
multidict==6.1.0
25+
openai==1.66.0
26+
opentracing==2.4.0
27+
packaging==25.0
28+
pillow==10.4.0
29+
pluggy==1.5.0
30+
propcache==0.2.0
31+
pydantic==2.10.6
32+
pydantic-core==2.27.2
33+
pytest==8.3.5
34+
pytest-asyncio==0.21.1
35+
pytest-cov==5.0.0
36+
pytest-mock==3.14.1
37+
pytest-randomly==3.15.0
38+
pyyaml==6.0.2
39+
regex==2024.11.6
40+
requests==2.32.4
41+
sniffio==1.3.1
42+
sortedcontainers==2.4.0
43+
tiktoken==0.7.0
44+
tomli==2.2.1
45+
tqdm==4.67.1
46+
typing-extensions==4.13.2
47+
urllib3==1.26.20
48+
vcrpy==6.0.2
49+
wrapt==1.17.2
50+
yarl==1.15.2
51+
zipp==3.20.2

‎.riot/requirements/c5399d2.txt

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
#
2+
# This file is autogenerated by pip-compile with Python 3.11
3+
# by the following command:
4+
#
5+
# pip-compile --allow-unsafe --no-annotate .riot/requirements/c5399d2.in
6+
#
7+
annotated-types==0.7.0
8+
anyio==4.9.0
9+
attrs==25.3.0
10+
certifi==2025.7.14
11+
charset-normalizer==3.4.2
12+
coverage[toml]==7.9.2
13+
distro==1.9.0
14+
h11==0.16.0
15+
httpcore==1.0.9
16+
httpx==0.28.1
17+
hypothesis==6.45.0
18+
idna==3.10
19+
iniconfig==2.1.0
20+
jiter==0.10.0
21+
mock==5.2.0
22+
multidict==6.6.3
23+
openai==1.66.0
24+
opentracing==2.4.0
25+
packaging==25.0
26+
pillow==11.3.0
27+
pluggy==1.6.0
28+
propcache==0.3.2
29+
pydantic==2.11.7
30+
pydantic-core==2.33.2
31+
pygments==2.19.2
32+
pytest==8.4.1
33+
pytest-asyncio==0.21.1
34+
pytest-cov==6.2.1
35+
pytest-mock==3.14.1
36+
pytest-randomly==3.16.0
37+
pyyaml==6.0.2
38+
regex==2024.11.6
39+
requests==2.32.4
40+
sniffio==1.3.1
41+
sortedcontainers==2.4.0
42+
tiktoken==0.9.0
43+
tqdm==4.67.1
44+
typing-extensions==4.14.1
45+
typing-inspection==0.4.1
46+
urllib3==1.26.20
47+
vcrpy==7.0.0
48+
wrapt==1.17.2
49+
yarl==1.20.1

‎.riot/requirements/e9c67e1.txt

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
#
2+
# This file is autogenerated by pip-compile with Python 3.9
3+
# by the following command:
4+
#
5+
# pip-compile --allow-unsafe --no-annotate .riot/requirements/e9c67e1.in
6+
#
7+
annotated-types==0.7.0
8+
anyio==4.9.0
9+
attrs==25.3.0
10+
certifi==2025.7.14
11+
charset-normalizer==3.4.2
12+
coverage[toml]==7.9.2
13+
distro==1.9.0
14+
exceptiongroup==1.3.0
15+
h11==0.16.0
16+
httpcore==1.0.9
17+
httpx==0.28.1
18+
hypothesis==6.45.0
19+
idna==3.10
20+
importlib-metadata==8.7.0
21+
iniconfig==2.1.0
22+
jiter==0.10.0
23+
mock==5.2.0
24+
multidict==6.6.3
25+
openai==1.66.0
26+
opentracing==2.4.0
27+
packaging==25.0
28+
pillow==11.3.0
29+
pluggy==1.6.0
30+
propcache==0.3.2
31+
pydantic==2.11.7
32+
pydantic-core==2.33.2
33+
pygments==2.19.2
34+
pytest==8.4.1
35+
pytest-asyncio==0.21.1
36+
pytest-cov==6.2.1
37+
pytest-mock==3.14.1
38+
pytest-randomly==3.16.0
39+
pyyaml==6.0.2
40+
regex==2024.11.6
41+
requests==2.32.4
42+
sniffio==1.3.1
43+
sortedcontainers==2.4.0
44+
tiktoken==0.9.0
45+
tomli==2.2.1
46+
tqdm==4.67.1
47+
typing-extensions==4.14.1
48+
typing-inspection==0.4.1
49+
urllib3==1.26.20
50+
vcrpy==7.0.0
51+
wrapt==1.17.2
52+
yarl==1.20.1
53+
zipp==3.23.0

‎ddtrace/llmobs/_integrations/openai.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -153,9 +153,9 @@ def _extract_llmobs_metrics_tags(
153153

154154
# in the streamed responses case, `resp` is a list with `usage` being stored in the first element
155155
if resp and isinstance(resp, list) and _get_attr(resp[0], "usage", None):
156-
token_usage = resp[0].get("usage", {})
157-
elif resp and getattr(resp, "usage", None):
158-
token_usage = resp.usage
156+
token_usage = _get_attr(resp[0], "usage", {})
157+
elif resp and _get_attr(resp, "usage", None):
158+
token_usage = _get_attr(resp, "usage", {})
159159
if token_usage is not None:
160160
prompt_tokens = _get_attr(token_usage, "prompt_tokens", 0)
161161
completion_tokens = _get_attr(token_usage, "completion_tokens", 0)

‎releasenotes/notes/openai-responses-early-support-b44a3216b1adf575.yaml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
---
2+
fixes:
3+
- |
4+
openai, LLM Observability: fixed an issue when using the openai responses api with ``openai>=1.66.0,<1.66.2`` would result in an ``AttributeError``.

‎riotfile.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2632,7 +2632,7 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT
26322632
Venv(
26332633
pys=select_pys(min_version="3.8"),
26342634
pkgs={
2635-
"openai": [latest, "~=1.76.2"],
2635+
"openai": [latest, "~=1.76.2", "==1.66.0"],
26362636
"tiktoken": latest,
26372637
"pillow": latest,
26382638
},

‎tests/contrib/openai/test_openai_llmobs.py

Lines changed: 2 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -1794,43 +1794,12 @@ def test_responses_reasoning_stream(self, openai, ddtrace_global_config, mock_ll
17941794
for event in stream:
17951795
pass
17961796

1797-
span = mock_tracer.pop_traces()[0][0]
1798-
assert mock_llmobs_writer.enqueue.call_count == 1
1799-
mock_llmobs_writer.enqueue.assert_called_with(
1800-
_expected_llmobs_llm_span_event(
1801-
span,
1802-
model_name="o4-mini-2025-04-16",
1803-
model_provider="openai",
1804-
input_messages=[{"content": "If one plus a number is 10, what is the number?", "role": "user"}],
1805-
output_messages=[
1806-
{"role": "reasoning", "content": mock.ANY},
1807-
{"role": "assistant", "content": "The number is 9, since 1 + x = 10 ⇒ x = 10 − 1 = 9."},
1808-
],
1809-
metadata={
1810-
"reasoning": {"effort": "medium", "summary": "detailed"},
1811-
"stream": True,
1812-
"temperature": 1.0,
1813-
"top_p": 1.0,
1814-
"tools": [],
1815-
"tool_choice": "auto",
1816-
"truncation": "disabled",
1817-
"text": {"format": {"type": "text"}},
1818-
"reasoning_tokens": 128,
1819-
},
1820-
token_metrics={
1821-
"input_tokens": mock.ANY,
1822-
"output_tokens": mock.ANY,
1823-
"total_tokens": mock.ANY,
1824-
"cache_read_input_tokens": mock.ANY,
1825-
},
1826-
tags={"ml_app": "<ml-app-name>", "service": "tests.contrib.openai"},
1827-
)
1828-
)
1829-
18301797
# special assertion on rough reasoning content
18311798
span_event = mock_llmobs_writer.enqueue.call_args[0][0]
18321799
reasoning_content = json.loads(span_event["meta"]["output"]["messages"][0]["content"])
1800+
assistant_content = span_event["meta"]["output"]["messages"][1]["content"]
18331801
assert reasoning_content["summary"] is not None
1802+
assert assistant_content == "The number is 9, since 1 + x = 10 ⇒ x = 10 − 1 = 9."
18341803

18351804
@pytest.mark.skipif(
18361805
parse_version(openai_module.version.VERSION) < (1, 66), reason="Response options only available openai >= 1.66"

0 commit comments

Comments
 (0)