DO NOT MERGE: Tracing Tutorial

docs/tutorials/agent_tracing/data-prepper-config.yaml

@@ -0,0 +1,3 @@
+# Sample Data Prepper configuration for agent tracing
+# This is an example configuration for development purposes
+ssl: false

docs/tutorials/agent_tracing/docker-compose.yml

@@ -0,0 +1,97 @@
+# Sample Docker Compose configuration for agent tracing setup
+# This is an example configuration for development purposes
+version: '3'
+services:
+  opensearch-node1:
+    image: opensearchproject/opensearch:3.1.0 # NOTE: Ensure versions are consistent across all OpenSearch services (OpenSearch, Dashboards, plugins)
+    container_name: opensearch-node1
+    environment:
+      - cluster.name=opensearch-cluster
+      - node.name=opensearch-node1
+      - discovery.type=single-node
+      - bootstrap.memory_lock=true # along with the memlock settings below, disables swapping
+      - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m -Dopensearch.experimental.feature.telemetry.enabled=true"
+      - OPENSEARCH_INITIAL_ADMIN_PASSWORD=<PASSWORD>
+      - DISABLE_SECURITY_PLUGIN=true
+      - opensearch.experimental.feature.telemetry.enabled=true
+      - telemetry.feature.tracer.enabled=true
+      - telemetry.tracer.enabled=true
+      - telemetry.tracer.sampler.probability=1.0
+      - telemetry.otel.tracer.span.exporter.class=io.opentelemetry.exporter.otlp.trace.OtlpGrpcSpanExporter
+      - plugins.ml_commons.tracing_enabled=true
+      - plugins.ml_commons.agent_tracing_enabled=true
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+      nofile:
+        soft: 65536 # maximum number of open files for the OpenSearch user, set to at least 65536 on modern systems
+        hard: 65536
+    volumes:
+      - opensearch-data1:/usr/share/opensearch/data
+      # USE THE BELOW TO MOUNT YOUR LOCAL BUILD TO AN INSTANCE
+      - <YOUR_ML_COMMONS_DIRECTORY>/plugin/build/distributions:/usr/share/opensearch/<YOUR_NAME> 
+    command: >
+      bash -c "
+      bin/opensearch-plugin remove opensearch-skills; 
+      bin/opensearch-plugin remove opensearch-ml; 
+      bin/opensearch-plugin install --batch telemetry-otel;
+      bin/opensearch-plugin install --batch file:///usr/share/opensearch/<YOUR_NAME>/opensearch-ml-3.1.0.0-SNAPSHOT.zip;
+      OPENSEARCH_LOG4J_CONFIG_FILE=/usr/share/opensearch/config/telemetry-log4j2.xml ./opensearch-docker-entrypoint.sh opensearch"
+    ports:
+      - 9200:9200
+      - 9600:9600 # required for Performance Analyzer
+    networks:
+      - opensearch-net
+    depends_on:
+      - otel-collector
+    extra_hosts:
+      - "localhost:172.17.0.1"  # This maps localhost to the Docker host IP
+
+  opensearch-dashboards:
+    image: opensearchproject/opensearch-dashboards:3.1.0 # Must match OpenSearch version
+    container_name: opensearch-dashboards
+    ports:
+      - 5601:5601
+    expose:
+      - "5601"
+    environment:
+      OPENSEARCH_HOSTS: '["http://opensearch-node1:9200"]'
+      DISABLE_SECURITY_DASHBOARDS_PLUGIN: "true"
+    networks:
+      - opensearch-net
+    depends_on:
+      - opensearch-node1
+
+  data-prepper:
+    restart: unless-stopped
+    container_name: data-prepper
+    image: opensearchproject/data-prepper:2
+    volumes:
+      - ./data-prepper-config.yaml:/usr/share/data-prepper/config/data-prepper-config.yaml
+      - ./pipelines.yaml:/usr/share/data-prepper/pipelines/pipelines.yaml
+      - <PATH_TO_INDEX_MAPPING>/ml_agent_trace.json:/usr/share/data-prepper/ml_agent_trace.json
+    ports:
+      - "21890:21890"
+    networks:
+      - opensearch-net
+
+  otel-collector:
+    image: otel/opentelemetry-collector-contrib:latest
+    container_name: otel-collector
+    command: ["--config=/etc/otel-collector-config.yaml"]
+    volumes:
+      - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml
+    ports:
+      - "4317:4317"  # OTLP gRPC port
+      - "4318:4318"  # OTLP HTTP port
+    networks:
+      - opensearch-net
+    depends_on:
+      - data-prepper
+
+volumes:
+  opensearch-data1:
+
+networks:
+  opensearch-net:

docs/tutorials/agent_tracing/ml_agent_trace.json

@@ -0,0 +1,175 @@
+{
+    "version": 1,
+    "template": {
+      "mappings": {
+        "date_detection": false,
+        "_source": {
+          "enabled": true
+        },
+        "properties": {
+          "traceId": {
+            "ignore_above": 256,
+            "type": "keyword"
+          },
+          "spanId": {
+            "ignore_above": 256,
+            "type": "keyword"
+          },
+          "parentSpanId": {
+            "ignore_above": 256,
+            "type": "keyword"
+          },
+          "name": {
+            "ignore_above": 1024,
+            "type": "keyword"
+          },
+          "traceGroup": {
+            "ignore_above": 1024,
+            "type": "keyword"
+          },
+          "traceGroupFields": {
+            "properties": {
+              "endTime": {
+                "type": "date_nanos"
+              },
+              "durationInNanos": {
+                "type": "long"
+              },
+              "statusCode": {
+                "type": "integer"
+              }
+            }
+          },
+          "kind": {
+            "ignore_above": 128,
+            "type": "keyword"
+          },
+          "startTime": {
+            "type": "date_nanos"
+          },
+          "endTime": {
+            "type": "date_nanos"
+          },
+          "status": {
+            "properties": {
+              "code": {
+                "type": "integer"
+              },
+              "message": {
+                "type": "keyword"
+              }
+            }
+          },
+          "serviceName": {
+            "type": "keyword"
+          },
+          "durationInNanos": {
+            "type": "long"
+          },
+          "events": {
+            "type": "nested",
+            "properties": {
+              "time": {
+                "type": "date_nanos"
+              }
+            }
+          },
+          "links": {
+            "type": "nested"
+          },
+          "resource": {
+            "properties": {
+              "attributes": {
+                "properties": {
+                  "service.name": {
+                    "type": "keyword"
+                  }
+                }
+              }
+            }
+          },
+          "span": {
+            "properties": {
+              "attributes": {
+                "properties": {
+                  "service@type": {
+                    "type": "keyword"
+                  },
+                  "thread@name": {
+                    "type": "keyword"
+                  },
+                  "gen_ai@operation@name": {
+                    "type": "keyword"
+                  },
+                  "gen_ai@system": {
+                    "type": "keyword"
+                  },
+                  "gen_ai@agent@name": {
+                    "type": "text"
+                  },
+                  "gen_ai@agent@task": {
+                    "type": "text"
+                  },
+                  "gen_ai@agent@result": {
+                    "type": "text"
+                  },
+                  "gen_ai@agent@phase": {
+                    "type": "keyword"
+                  },
+                  "gen_ai@agent@step@number": {
+                    "type": "integer"
+                  },
+                  "gen_ai@agent@latency": {
+                    "type": "integer"
+                  },
+                  "gen_ai@request@model": {
+                    "type": "keyword"
+                  },
+                  "gen_ai@usage@input_tokens": {
+                    "type": "integer"
+                  },
+                  "gen_ai@usage@output_tokens": {
+                    "type": "integer"
+                  },
+                  "gen_ai@usage@total_tokens": {
+                    "type": "integer"
+                  },
+                  "gen_ai@system@message": {
+                    "type": "text"
+                  },
+                  "gen_ai@tool@name": {
+                    "type": "keyword"
+                  },
+                  "gen_ai@tool@description": {
+                    "type": "text"
+                  },
+                  "ml@connector@id": {
+                    "type": "keyword"
+                  },
+                  "ml@connector@name": {
+                    "type": "keyword"
+                  },
+                  "ml@model@id": {
+                    "type": "keyword"
+                  },
+                  "ml@model@name": {
+                    "type": "keyword"
+                  },
+                  "ml@model@request_body": {
+                    "type": "text"
+                  }
+                }
+              }
+            }
+          },
+          "instrumentationScope": {
+            "properties": {
+              "name": {
+                "type": "keyword"
+              }
+            }
+          }
+        }
+      }
+    }
+  }

docs/tutorials/agent_tracing/otel-collector-config.yaml

@@ -0,0 +1,36 @@
+# Sample OpenTelemetry Collector configuration for agent tracing
+# This is an example configuration for development purposes
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: 0.0.0.0:4317
+      http:
+        endpoint: 0.0.0.0:4318
+
+processors:
+  filter/traces:
+    spans:
+      include:
+        match_type: regexp
+        attributes:
+          - key: service.type
+            value: tracer
+  batch/traces:
+    timeout: 5s
+    send_batch_size: 50
+
+exporters:
+  debug:
+    verbosity: detailed
+  otlp/data-prepper:
+    endpoint: data-prepper:21890
+    tls:
+      insecure: true
+
+service:
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [filter/traces, batch/traces]
+      exporters: [debug, otlp/data-prepper]

docs/tutorials/agent_tracing/pipelines.yaml

@@ -0,0 +1,66 @@
+# Sample Data Prepper pipelines configuration for agent tracing
+# This is an example configuration for development purposes
+entry-pipeline:
+  workers: 8
+  source:
+    otel_trace_source:
+      ssl: false
+      authentication:
+        unauthenticated:
+  buffer:
+    bounded_blocking:
+      buffer_size: 512
+      batch_size: 8
+  sink:
+    - pipeline:
+        name: "raw-trace-pipeline"
+    - pipeline:
+        name: "service-map-pipeline"
+
+raw-trace-pipeline:
+  workers: 8
+  source:
+    pipeline:
+      name: "entry-pipeline"
+  buffer:
+    bounded_blocking:
+      buffer_size: 512
+      batch_size: 64
+  processor:
+    - otel_trace_raw:
+        # flatten_attributes: false
+    - otel_trace_group:
+        hosts: [ "http://opensearch-node1:9200" ]
+        username: "admin"
+        password: "<PASSWORD>"
+  sink:
+    - opensearch:
+        hosts: ["http://opensearch-node1:9200"]
+        insecure: true
+        username: admin
+        password: "<PASSWORD>"
+        # index_type: trace-analytics-raw # Uses default index mappings
+        index_type: custom
+        index: otel-v1-apm-span-agent
+        template_file: ml_agent_trace.json
+        template_type: index-template
+
+service-map-pipeline:
+  workers: 8
+  source:
+    pipeline:
+      name: "entry-pipeline"
+  processor:
+    - service_map_stateful:
+        window_duration: 360
+  buffer:
+    bounded_blocking:
+      buffer_size: 512
+      batch_size: 8
+  sink:
+    - opensearch:
+        hosts: ["http://opensearch-node1:9200"]
+        insecure: true
+        username: admin
+        password: "<PASSWORD>"
+        index_type: trace-analytics-service-map