Skip to content

Commit 1f86d5a

Browse files
rachelyangdogmabdinur
rachelyangdog
and
mabdinur
authored
chore(telemetry): adding http propagation telemetry (#13992)
- Adding telemetry metrics for successful header injections and extractions for baggage, datadog, tracecontext, b3single, b3multi - adding telemetry metrics for malformed baggage header extractions, and when max bytes or max item limits in baggage are exceeded - for more information: [baggage telemetry RFC ](https://docs.google.com/document/d/1N-dux1_n1vVu391Z5IDT_zWUwZp-MPqD7k7bhND8SK8/edit?tab=t.0) ## Checklist - [x] PR author has checked that all the criteria below are met - The PR description includes an overview of the change - The PR description articulates the motivation for the change - The change includes tests OR the PR description describes a testing strategy - The PR description notes risks associated with the change, if any - Newly-added code is easy to change - The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) - The change includes or references documentation updates if necessary - Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) ## Reviewer Checklist - [x] Reviewer has checked that all the criteria below are met - Title is accurate - All changes are related to the pull request's stated goal - Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - Testing strategy adequately addresses listed risks - Newly-added code is easy to change - Release note makes sense to a user of the library - If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting) --------- Co-authored-by: Munir Abdinur <[email protected]>
1 parent 73233d2 commit 1f86d5a

File tree

3 files changed

+270
-14
lines changed

3 files changed

+270
-14
lines changed

‎.gitlab/benchmarks/bp-runner.microbenchmarks.fail-on-breach.yml

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -153,31 +153,31 @@ experiments:
153153
# httppropagationextract
154154
- name: httppropagationextract-all_styles_all_headers
155155
thresholds:
156-
- execution_time < 0.07 ms
156+
- execution_time < 0.084 ms
157157
- max_rss_usage < 31.00 MB
158158
- name: httppropagationextract-b3_headers
159159
thresholds:
160160
- execution_time < 0.02 ms
161161
- max_rss_usage < 31.00 MB
162162
- name: httppropagationextract-b3_single_headers
163163
thresholds:
164-
- execution_time < 0.01 ms
164+
- execution_time < 0.015 ms
165165
- max_rss_usage < 31.00 MB
166166
- name: httppropagationextract-datadog_tracecontext_tracestate_not_propagated_on_trace_id_no_match
167167
thresholds:
168-
- execution_time < 0.06 ms
168+
- execution_time < 0.066 ms
169169
- max_rss_usage < 31.00 MB
170170
- name: httppropagationextract-datadog_tracecontext_tracestate_propagated_on_trace_id_match
171171
thresholds:
172-
- execution_time < 0.06 ms
172+
- execution_time < 0.069 ms
173173
- max_rss_usage < 31.00 MB
174174
- name: httppropagationextract-empty_headers
175175
thresholds:
176176
- execution_time < 0.01 ms
177177
- max_rss_usage < 31.00 MB
178178
- name: httppropagationextract-full_t_id_datadog_headers
179179
thresholds:
180-
- execution_time < 0.02 ms
180+
- execution_time < 0.025 ms
181181
- max_rss_usage < 31.00 MB
182182
- name: httppropagationextract-invalid_priority_header
183183
thresholds:
@@ -217,7 +217,7 @@ experiments:
217217
- max_rss_usage < 31.00 MB
218218
- name: httppropagationextract-tracecontext_headers
219219
thresholds:
220-
- execution_time < 0.03 ms
220+
- execution_time < 0.035 ms
221221
- max_rss_usage < 31.00 MB
222222
- name: httppropagationextract-valid_headers_all
223223
thresholds:
@@ -279,27 +279,27 @@ experiments:
279279
- max_rss_usage < 31.00 MB
280280
- name: httppropagationinject-with_all
281281
thresholds:
282-
- execution_time < 0.03 ms
282+
- execution_time < 0.036 ms
283283
- max_rss_usage < 31.00 MB
284284
- name: httppropagationinject-with_dd_origin
285285
thresholds:
286-
- execution_time < 0.02 ms
286+
- execution_time < 0.027 ms
287287
- max_rss_usage < 31.00 MB
288288
- name: httppropagationinject-with_priority_and_origin
289289
thresholds:
290-
- execution_time < 0.02 ms
290+
- execution_time < 0.029 ms
291291
- max_rss_usage < 31.00 MB
292292
- name: httppropagationinject-with_sampling_priority
293293
thresholds:
294-
- execution_time < 0.02 ms
294+
- execution_time < 0.023 ms
295295
- max_rss_usage < 31.00 MB
296296
- name: httppropagationinject-with_tags
297297
thresholds:
298-
- execution_time < 0.02 ms
298+
- execution_time < 0.029 ms
299299
- max_rss_usage < 31.00 MB
300300
- name: httppropagationinject-with_tags_invalid
301301
thresholds:
302-
- execution_time < 0.03 ms
302+
- execution_time < 0.032 ms
303303
- max_rss_usage < 31.00 MB
304304
- name: httppropagationinject-with_tags_max_size
305305
thresholds:

‎ddtrace/propagation/http.py

Lines changed: 63 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@
1919
from ddtrace._trace.span import _MetaDictType
2020
from ddtrace.appsec._constants import APPSEC
2121
from ddtrace.internal import core
22+
from ddtrace.internal.telemetry import telemetry_writer
23+
from ddtrace.internal.telemetry.constants import TELEMETRY_NAMESPACE
2224
from ddtrace.settings._config import config
2325
from ddtrace.settings.asm import config as asm_config
2426

@@ -151,6 +153,20 @@ def _dd_id_to_b3_id(dd_id):
151153
return "{:016x}".format(dd_id)
152154

153155

156+
def _record_http_telemetry(metric_name: str, header_style: str) -> None:
157+
"""Record telemetry metric for HTTP propagation operations.
158+
159+
:param metric_name: The name of the metric to record
160+
:param tags: Tuple of tag key-value pairs to include with the metric
161+
"""
162+
telemetry_writer.add_count_metric(
163+
namespace=TELEMETRY_NAMESPACE.TRACERS,
164+
name=metric_name,
165+
value=1,
166+
tags=(("header_style", header_style),),
167+
)
168+
169+
154170
class _DatadogMultiHeader:
155171
"""Helper class for injecting/extract Datadog multi header format
156172
@@ -288,6 +304,9 @@ def _inject(span_context, headers):
288304
span_context._meta["_dd.propagation_error"] = "encoding_error"
289305
log.warning("failed to encode x-datadog-tags", exc_info=True)
290306

307+
# Record telemetry for successful injection
308+
_record_http_telemetry("context_header_style.injected", PROPAGATION_STYLE_DATADOG)
309+
291310
@staticmethod
292311
def _extract(headers):
293312
# type: (Dict[str, str]) -> Optional[Context]
@@ -445,6 +464,9 @@ def _inject(span_context, headers):
445464
elif sampling_priority > 1:
446465
headers[_HTTP_HEADER_B3_FLAGS] = "1"
447466

467+
# Record telemetry for successful injection
468+
_record_http_telemetry("context_header_style.injected", PROPAGATION_STYLE_B3_MULTI)
469+
448470
@staticmethod
449471
def _extract(headers):
450472
# type: (Dict[str, str]) -> Optional[Context]
@@ -561,6 +583,9 @@ def _inject(span_context, headers):
561583
single_header += "-d"
562584
headers[_HTTP_HEADER_B3_SINGLE] = single_header
563585

586+
# Record telemetry for successful injection
587+
_record_http_telemetry("context_header_style.injected", PROPAGATION_STYLE_B3_SINGLE)
588+
564589
@staticmethod
565590
def _extract(headers):
566591
# type: (Dict[str, str]) -> Optional[Context]
@@ -873,6 +898,9 @@ def _inject(span_context, headers):
873898
else:
874899
headers[_HTTP_HEADER_TRACESTATE] = span_context._tracestate
875900

901+
# Record telemetry for successful injection
902+
_record_http_telemetry("context_header_style.injected", _PROPAGATION_STYLE_W3C_TRACECONTEXT)
903+
876904

877905
class _BaggageHeader:
878906
"""Helper class to inject/extract Baggage Headers"""
@@ -897,6 +925,13 @@ def _inject(span_context: Context, headers: Dict[str, str]) -> None:
897925
try:
898926
if len(baggage_items) > DD_TRACE_BAGGAGE_MAX_ITEMS:
899927
log.warning("Baggage item limit exceeded, dropping excess items")
928+
# Record telemetry for baggage item count exceeding limit
929+
telemetry_writer.add_count_metric(
930+
namespace=TELEMETRY_NAMESPACE.TRACERS,
931+
name="context_header_style.truncated",
932+
value=1,
933+
tags=(("truncation_reason", "baggage_item_count_exceeded"),),
934+
)
900935
baggage_items = itertools.islice(baggage_items, DD_TRACE_BAGGAGE_MAX_ITEMS) # type: ignore
901936

902937
encoded_items: List[str] = []
@@ -906,16 +941,37 @@ def _inject(span_context: Context, headers: Dict[str, str]) -> None:
906941
item_size = len(item.encode("utf-8")) + (1 if encoded_items else 0) # +1 for comma if not first item
907942
if total_size + item_size > DD_TRACE_BAGGAGE_MAX_BYTES:
908943
log.warning("Baggage header size exceeded, dropping excess items")
944+
# Record telemetry for baggage header size exceeding limit
945+
telemetry_writer.add_count_metric(
946+
namespace=TELEMETRY_NAMESPACE.TRACERS,
947+
name="context_header_style.truncated",
948+
value=1,
949+
tags=(("truncation_reason", "baggage_byte_count_exceeded"),),
950+
)
909951
break # stop adding items when size limit is reached
910952
encoded_items.append(item)
911953
total_size += item_size
912954

913955
header_value = ",".join(encoded_items)
914956
headers[_HTTP_HEADER_BAGGAGE] = header_value
915957

958+
# Record telemetry for successful baggage injection
959+
_record_http_telemetry("context_header_style.injected", _PROPAGATION_STYLE_BAGGAGE)
960+
916961
except Exception:
917962
log.warning("Failed to encode and inject baggage header")
918963

964+
@staticmethod
965+
def _record_malformed_and_return_empty() -> Context:
966+
"""Record telemetry for malformed baggage header and return empty context."""
967+
telemetry_writer.add_count_metric(
968+
namespace=TELEMETRY_NAMESPACE.TRACERS,
969+
name="context_header_style.malformed",
970+
value=1,
971+
tags=(("header_style", _PROPAGATION_STYLE_BAGGAGE),),
972+
)
973+
return Context(baggage={})
974+
919975
@staticmethod
920976
def _extract(headers: Dict[str, str]) -> Context:
921977
header_value = _extract_header_value(_POSSIBLE_HTTP_BAGGAGE_HEADER, headers)
@@ -927,12 +983,12 @@ def _extract(headers: Dict[str, str]) -> Context:
927983
baggages = header_value.split(",")
928984
for key_value in baggages:
929985
if "=" not in key_value:
930-
return Context(baggage={})
986+
return _BaggageHeader._record_malformed_and_return_empty()
931987
key, value = key_value.split("=", 1)
932988
key = urllib.parse.unquote(key.strip())
933989
value = urllib.parse.unquote(value.strip())
934990
if not key or not value:
935-
return Context(baggage={})
991+
return _BaggageHeader._record_malformed_and_return_empty()
936992
baggage[key] = value
937993

938994
return Context(baggage=baggage)
@@ -964,6 +1020,7 @@ def _extract_configured_contexts_avail(normalized_headers: Dict[str, str]) -> Tu
9641020
propagator = _PROP_STYLES[prop_style]
9651021
context = propagator._extract(normalized_headers) # type: ignore
9661022
if context:
1023+
_record_http_telemetry("context_header_style.extracted", prop_style)
9671024
contexts.append(context)
9681025
styles_w_ctx.append(prop_style)
9691026
return contexts, styles_w_ctx
@@ -1130,6 +1187,8 @@ def my_controller(url, headers):
11301187
propagator = _PROP_STYLES[prop_style]
11311188
context = propagator._extract(normalized_headers)
11321189
style = prop_style
1190+
if context:
1191+
_record_http_telemetry("context_header_style.extracted", prop_style)
11331192
if config._propagation_http_baggage_enabled is True:
11341193
_attach_baggage_to_context(normalized_headers, context)
11351194
break
@@ -1150,6 +1209,8 @@ def my_controller(url, headers):
11501209
if _PROPAGATION_STYLE_BAGGAGE in config._propagation_style_extract:
11511210
baggage_context = _BaggageHeader._extract(normalized_headers)
11521211
if baggage_context._baggage != {}:
1212+
# Record telemetry for successful baggage extraction
1213+
_record_http_telemetry("context_header_style.extracted", _PROPAGATION_STYLE_BAGGAGE)
11531214
if context:
11541215
context._baggage = baggage_context.get_all_baggage_items()
11551216
else:

0 commit comments

Comments
 (0)