Skip to content

Commit b2d7c25

Browse files
spencergilbertspencergilbert
authored
Merge pull request #9 from vectordotdev/spencer/eks-aggregator
spencer/eks aggregator
2 parents e47e72f + dc29456 commit b2d7c25

File tree

5 files changed

+448
-0
lines changed

5 files changed

+448
-0
lines changed

‎eks-aggregator/Makefile‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
ifndef ACCOUNT_ID
2+
$(error ACCOUNT_ID is not set)
3+
endif
4+
.PHONY: cluster-up
5+
cluster-up:
6+
eksctl create cluster --with-oidc --name vector-demo
7+
aws iam create-policy \
8+
--policy-name AWSLoadBalancerControllerIAMPolicy \
9+
--policy-document file://iam_policy.json | jq -r .Policy.Arn
10+
eksctl create iamserviceaccount \
11+
--cluster=vector-demo \
12+
--namespace=kube-system \
13+
--name=aws-load-balancer-controller \
14+
--attach-policy-arn=arn:aws:iam::$(ACCOUNT_ID):policy/AWSLoadBalancerControllerIAMPolicy \
15+
--override-existing-serviceaccounts \
16+
--approve
17+
helm repo add eks https://aws.github.io/eks-charts && \
18+
helm repo update
19+
kubectl apply -k "github.com/aws/eks-charts/stable/aws-load-balancer-controller/crds?ref=master"
20+
helm upgrade --install aws-load-balancer-controller eks/aws-load-balancer-controller \
21+
--set clusterName=vector-demo \
22+
--set serviceAccount.create=false \
23+
--set serviceAccount.name=aws-load-balancer-controller \
24+
--namespace kube-system
25+
kubectl apply -f "https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml"
26+
kubectl create namespace vector
27+
kubectl create namespace datadog
28+
29+
.PHONY: cluster-down
30+
cluster-down:
31+
helm uninstall aws-load-balancer-controller \
32+
--namespace kube-system
33+
kubectl delete namespace vector
34+
kubectl delete namespace datadog
35+
eksctl delete iamserviceaccount \
36+
--cluster=vector-demo \
37+
--namespace=kube-system \
38+
--name=aws-load-balancer-controller
39+
sleep 15
40+
aws iam delete-policy --policy-arn arn:aws:iam::$(ACCOUNT_ID):policy/AWSLoadBalancerControllerIAMPolicy
41+
eksctl delete cluster --name=vector-demo

‎eks-aggregator/README.md‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
# EKS Stateless Aggregator Demo
2+
3+
## Prerequisites
4+
5+
- [helm](https://helm.sh/docs/intro/install/)
6+
- [kubectl](https://kubernetes.io/docs/tasks/tools/)
7+
- [ekctl](https://eksctl.io/introduction/#installation) - if creating a new cluster with the Makefile
8+
9+
Your EKS cluster will need the [AWS Load Balancer Controller](https://github.com/kubernetes-sigs/aws-load-balancer-controller) installed, this will be installed via the `make` targets or you
10+
can follow [Amazon's instructions](https://docs.aws.amazon.com/eks/latest/userguide/aws-load-balancer-controller.html) for your own cluster. Additionally you'll also need a Datadog API key for Vector and the Datadog Agents.
11+
12+
## Getting started
13+
14+
Add the necessary Helm repositories for the Vector and Datadog charts:
15+
16+
```shell
17+
helm repo add datadog https://helm.datadoghq.com
18+
helm repo add vector https://helm.vector.dev
19+
helm repo update
20+
```
21+
22+
If you need to provision an EKS cluster with the AWS load balancer controller, you can use the included Makefile by running:
23+
24+
```shell
25+
ACCOUNT_ID=<AWS_ACCOUNT_ID> make cluster-up
26+
```
27+
28+
The following command will install Vector as an Aggregator using an Application Load Balancer to route requests from Datadog Agents.
29+
Vector is configured to process Datadog Agent logs in a similar fashion Datadog's [Pipelines](https://docs.datadoghq.com/logs/log_configuration/pipelines/)
30+
feature, allowing you to move your log processing onto your own hardware.
31+
32+
```shell
33+
helm upgrade --install vector vector/vector --devel \
34+
--namespace vector --values helm/vector.yaml \
35+
--set secrets.generic.DATADOG_API_KEY=<DD_API_KEY(base64 encoded)> \
36+
--set ingress.hosts[0].host=DUMMY_VAL
37+
```
38+
39+
Once your ALB is provisioned you can run the following command to extract it's generated hostname to replace the DUMMY_VAL above.
40+
41+
```shell
42+
export ALB_HOSTNAME=kubectl --namespace vector get ingress vector \
43+
--output go-template='{{(index .status.loadBalancer.ingress 0 ).hostname}}'
44+
```
45+
46+
The following command will upgrade your `vector` release with the created ALB hostname.
47+
48+
```shell
49+
helm upgrade --install vector vector/vector --devel \
50+
--namespace vector --values helm/vector.yaml \
51+
--set secrets.generic.DATADOG_API_KEY=<DD_API_KEY(base64 encoded)> \
52+
--set ingress.hosts[0].host=${ALB_HOSTNAME}
53+
```
54+
55+
Then install your Datadog Agents substituting the hostname from the previous step.
56+
57+
```shell
58+
helm upgrade --install datadog datadog/datadog \
59+
--namespace datadog --values helm/datadog.yaml \
60+
--set datadog.apiKey=<DD_API_KEY> \
61+
--set agents.customAgentConfig.logs_config.logs_dd_url="<ALB_HOSTNAME>:8080"
62+
```
63+
64+
Once all the pods have started, you should begin to see logs being ingested to your [Datadog account](https://app.datadoghq.com/logs) that are being aggregated and parsed by Vector.
65+
66+
## Cleaning up
67+
68+
The _cluster-down_ target will delete the Namespaces and Cluster created during this demo.
69+
70+
```shell
71+
make cluster-down
72+
```
73+
74+
## Notes
75+
76+
- A nightly image is currently used to leverage our rewritten `datadog_logs` sink
77+
- The `--devel` option is used to access our currently _pre-release_ [`vector`](https://github.com/vectordotdev/helm-charts/blob/develop/charts/vector/README.md) chart

‎eks-aggregator/helm/datadog.yaml‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
datadog:
2+
logLevel: debug
3+
clusterName: vector-demo
4+
logs:
5+
containerCollectAll: true
6+
enabled: true
7+
orchestratorExplorer:
8+
enabled: true
9+
processAgent:
10+
processCollection: true
11+
prometheusScrape:
12+
enabled: true
13+
agents:
14+
useConfigMap: true
15+
customAgentConfig:
16+
logs_config:
17+
logs_dd_url: "vector.mycompany.tld:8080"
18+
logs_no_ssl: true
19+
use_http: true
20+
use_v2_api: false

‎eks-aggregator/helm/vector.yaml‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
role: Stateless-Aggregator
2+
3+
image:
4+
tag: nightly-2021-09-16-distroless-libc
5+
6+
autoscaling:
7+
enabled: true
8+
minReplicas: 2
9+
maxReplicas: 5
10+
11+
resources:
12+
requests:
13+
cpu: 200m
14+
memory: 256Mi
15+
limits:
16+
cpu: 200m
17+
memory: 256Mi
18+
19+
env:
20+
- name: DATADOG_API_KEY
21+
valueFrom:
22+
secretKeyRef:
23+
name: vector
24+
key: DATADOG_API_KEY
25+
26+
podAnnotations:
27+
prometheus.io/scrape: "true"
28+
prometheus.io/port: "9090"
29+
30+
customConfig:
31+
api:
32+
enabled: true
33+
address: 0.0.0.0:8686
34+
sources:
35+
datadog_agents:
36+
type: datadog_agent
37+
acknowledgements: true
38+
address: 0.0.0.0:8080
39+
internal_metrics:
40+
type: internal_metrics
41+
transforms:
42+
remap:
43+
type: remap
44+
inputs:
45+
- datadog_agents
46+
drop_on_abort: true
47+
source: |
48+
# Parse the received .ddtags field so we can more easily access the contained tags
49+
.ddtags = parse_key_value!(.ddtags, key_value_delimiter: ":", field_delimiter: ",")
50+
.ddtags.sender = "vector"
51+
.ddtags.vector_aggregator = get_hostname!()
52+
53+
if .service == "agent" {
54+
parsed, err =
55+
parse_grok(.message, s'(?<timestamp>%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{TIME}) UTC \| %{NOTSPACE:agent} \| %{LOGLEVEL:status} \| \(%{NOTSPACE:filename}:%{NUMBER:lineno} in %{WORD:process}\) \|( %{NOTSPACE:kv} \|)?( - \|)?( \(%{NOTSPACE:pyFilename}:%{NUMBER:pyLineno}\) \|)?%{GREEDYDATA}', remove_empty: true) ??
56+
parse_grok(.message, s'(?<timestamp>%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{TIME}) UTC \| %{LOGLEVEL:status} \| \(%{NOTSPACE:filename}:%{NUMBER:lineno} in %{WORD:process}\)%{GREEDYDATA}') ??
57+
parse_grok(.message, s'(?<timestamp>%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{TIME}) UTC \| %{NOTSPACE:agent} \| %{LOGLEVEL:status}\s+\| %{WORD:class} \| %{GREEDYDATA}')
58+
if err != null { log("Failed to parse agent log: " + string!(.message), level: "error"); abort }
59+
60+
parsed |= parse_key_value(del(parsed.kv), key_value_delimiter: ":", field_delimiter: ",") ?? {}
61+
62+
ts = parse_timestamp!(parsed.timestamp, format: "%F %T")
63+
parsed.timestamp = to_unix_timestamp(ts, unit: "milliseconds")
64+
parsed.lineno = to_int!(parsed.lineno)
65+
if exists(parsed.pyLineno) { parsed.pyLineno = to_int!(parsed.pyLineno) }
66+
67+
. = merge(., parsed)
68+
}
69+
70+
# Re-encode Datadog tags as a string for the `datadog_logs` sink
71+
.ddtags = encode_key_value!(.ddtags, key_value_delimiter: ":", field_delimiter: ",")
72+
sinks:
73+
to_datadog:
74+
type: datadog_logs
75+
inputs:
76+
- remap
77+
default_api_key: "${DATADOG_API_KEY}"
78+
batch:
79+
timeout_secs: 5
80+
encoding:
81+
codec: json
82+
prom_exporter:
83+
type: prometheus_exporter
84+
inputs:
85+
- internal_metrics
86+
address: 0.0.0.0:9090
87+
88+
ingress:
89+
enabled: true
90+
annotations:
91+
alb.ingress.kubernetes.io/scheme: internal
92+
alb.ingress.kubernetes.io/healthcheck-port: '8686'
93+
alb.ingress.kubernetes.io/healthcheck-path: /health
94+
alb.ingress.kubernetes.io/listen-ports: '[{"HTTP": 8080}]'
95+
alb.ingress.kubernetes.io/target-type: ip
96+
kubernetes.io/ingress.class: alb
97+
hosts:
98+
- host: vector.mycompany.tld
99+
paths:
100+
- path: /
101+
pathType: Prefix
102+
port:
103+
name: datadog-agents

0 commit comments

Comments
 (0)