Merge pull request #7 from timberio/vector-to-datadog

Add demo for shipping Vector internal logs/metrics to Datadog

Author: lucperkins

datadog-agent/vector/vector.toml

@@ -33,4 +33,4 @@ codec = "json"
 timestamp_format = "rfc3339"
 
 [sinks.datadog.healthcheck]
-enabled = true
+enabled = true

datadog-observability/README.md

@@ -0,0 +1,19 @@
+# Send Vector logs and metrics to Datadog
+
+In this demo, Vector is running a simple topology in which the `generator` source emits random JSON
+logs twice per second and sends those logs off to stdout via the `console` sink. But even this
+simple pipeline is enough to generate internal logs and metrics for Vector (via the eponymous
+sources).
+
+To run this scenario, make sure you have the `DD_API_KEY` environment variable set to your Datadog
+API key and then run:
+
+```bash
+docker compose up --detach
+```
+
+Once Vector is up and running, check out the [Logs exporer][logs] and this simple
+[Metrics dashboard][metrics].
+
+[logs]: https://app.datadoghq.com/logs?query=%40tag%3Avector_internal
+[metrics]: https://app.datadoghq.com/metric/explorer?live=true&tile_size=m&exp_metric=vector.events_in_total&exp_agg=avg&exp_row_type=metric

datadog-observability/docker-compose.yml

@@ -0,0 +1,11 @@
+version: '3.9'
+services:
+  vector:
+    image: timberio/vector:0.16.1-alpine
+    ports:
+      - 8686:8686
+    environment:
+      - DD_API_KEY
+      - LOG=info
+    volumes:
+      - ./vector:/etc/vector:ro

datadog-observability/vector/vector.toml

@@ -0,0 +1,49 @@
+[api]
+enabled = true
+address = "0.0.0.0:8686"
+playground = true
+
+[sources.generator]
+type = "generator"
+format = "json"
+interval = 0.5
+
+[sinks.stdout]
+type = "console"
+inputs = ["generator"]
+target = "stdout"
+
+[sinks.stdout.encoding]
+codec = "json"
+
+[sources.vector_logs]
+type = "internal_logs"
+
+[sources.vector_metrics]
+type = "internal_metrics"
+
+[transforms.tag_vector_logs]
+type = "remap"
+inputs = ["vector_logs"]
+source = '''
+.tag = "vector_internal"
+'''
+
+[sinks.datadog_logs]
+type = "datadog_logs"
+inputs = ["tag_vector_logs"]
+default_api_key = "${DD_API_KEY}"
+
+[sinks.datadog_logs.healthcheck]
+enabled = true
+
+[sinks.datadog_logs.encoding]
+codec = "json"
+
+[sinks.datadog_metrics]
+type = "datadog_metrics"
+inputs = ["vector_metrics"]
+api_key = "${DD_API_KEY}"
+
+[sinks.datadog_metrics.healthcheck]
+enabled = true