Merge pull request #11 from vectordotdev/redaction-example

Add simple redaction example

Author: lucperkins

aggregator/docker-compose.yml

@@ -8,6 +8,14 @@ services:
       - ./vector/agent/http:/etc/vector:ro
     depends_on:
       - aggregator
+  transaction_logs_agent:
+    image: timberio/vector:nightly-alpine
+    deploy:
+      replicas: 3
+    volumes:
+      - ./vector/agent/transaction:/etc/vector:ro
+    depends_on:
+      - aggregator
   syslog_logs_agent:
     image: timberio/vector:nightly-alpine
     deploy:

aggregator/vector/agent/transaction/vector.toml

@@ -0,0 +1,24 @@
+[sources.logs]
+type = "generator"
+format = "shuffle"
+lines = [
+  '{"transaction":{"id":"d71dc34a-ff0c-4708-afb3-bed232ac879e","result":"success"},"user":{"username":"victory1337","email":"[email protected]"}}',
+  '{"transaction":{"id":"1251afbc-481a-45a1-abe4-caa3a9dfc2b3","result":"cancel"},"user":{"username":"socrates21","email":"[email protected]"}}'
+]
+interval = 2
+
+[transforms.tag]
+type = "remap"
+inputs = ["logs"]
+source = '''
+.tag = "transaction_logs"
+'''
+
+[sinks.out]
+type = "vector"
+inputs = ["tag"]
+address = "aggregator:9000"
+version = "2"
+
+[sinks.out.healthcheck]
+enabled = true

aggregator/vector/aggregator/vector.toml

@@ -18,9 +18,23 @@ inputs = ["*_agents"]
 
 [transforms.logs_router.route]
 http_logs = '.tag == "http_logs"'
+transaction_logs = '.tag == "transaction_logs"'
 syslog_logs = '.tag == "syslog_logs"'
 fluent_bit_logs = '.tag == "fluent_bit_logs"'
 
+[transforms.redact_transaction_logs]
+type = "remap"
+inputs = ["logs_router.transaction_logs"]
+source = '''
+# Parse JSON string
+. = parse_json!(.message)
+
+# Apply email address redaction filter
+. = redact(., filters: [
+  r'(\w+)@(\w+).(\w+)'
+])
+'''
+
 [transforms.parse_and_sanitize_http_logs]
 type = "remap"
 inputs = ["logs_router.http_logs"]
@@ -95,7 +109,7 @@ condition = 'to_syslog_severity!(.severity) < 4'
 
 [sinks.json_out]
 type = "console"
-inputs = ["filter_*", "parse_and_sanitize_fluent_bit_logs"]
+inputs = ["filter_*", "parse_and_sanitize_fluent_bit_logs", "redact_transaction_logs"]
 target = "stdout"
 
 [sinks.json_out.encoding]