Merge pull request #10 from vectordotdev/csv-enrichment-schema

Add CSV enrichment example with schema

Author: lucperkins

csv-enrichment/README.md

@@ -1,20 +1,24 @@
 # CSV enrichment in Vector
 
-**Note:** This feature is unreleased and still under active development. However, if you want to try it out, we would
-love any feedback!
+> **Note:** This feature is unreleased and still under active development. However, if you want to try it out, we would
+> love any feedback!
 
 ## How to start it
 
-`docker-compose up`
+```shell
+docker-compose up
+```
 
 ## What it does
 
-This demo shows an example of using CSV enrichment in Vector to enrich events with data from a CSV file.
+This demo shows an example of using CSV enrichment in Vector to enrich events with data from two CSV files.
 
-The CSV file ([`./data/users.csv`](./data/users.csv) contains a list of users with their phone numbers and addresses.
+The [`users.csv`](./data/users.csv) file contains a list of users with their phone numbers and addresses, while the
+[`codes.csv`](./data/codes.csv) file contains a list of common Unix exit codes (with code, tag, and message info).
 
-The `vector.toml` configuration contains:
+The [`vector.toml`](./vector.toml) configuration contains:
 
-* A `random` source in the `vector.toml` that simply generates some fake events using names that match users in the CSV file
-* A `remap` transform then parses these events and then looks up the corresponding record from the CSV file to enrich the event with more metadata
+* Two `generator` sources in the that generate fake events that can be enriched using the CSV files (user data and coded error messages
+  respectively)
+* Two `remap` transforms that parse the events and then look up the corresponding records from the CSV files to enrich the events with more metadata
 * A `console` sink to print out the events

csv-enrichment/data/codes.csv

@@ -0,0 +1,4 @@
+code,tag,message
+1,"EPERM","Operation not permitted"
+2,"ENOENT","No such file or directory"
+3,"ESRCH","No such process"

csv-enrichment/vector.toml

@@ -1,9 +1,26 @@
+# User info table (all fields are string, thus no schema)
 [enrichment_tables.users]
 type = "file"
-file.path = "/var/lib/vector/data/users.csv"
-file.encoding.type = "csv"
 
-[sources.random]
+[enrichment_tables.users.file]
+path = "/var/lib/vector/data/users.csv"
+encoding = { type = "csv" }
+
+# Error codes table (with specified schema)
+[enrichment_tables.codes]
+type = "file"
+
+[enrichment_tables.codes.file]
+path = "/var/lib/vector/data/codes.csv"
+encoding = { type = "csv" }
+
+[enrichment_tables.codes.schema]
+code = "integer"
+tag = "string"
+message = "string"
+
+# Generate user info messages
+[sources.user_info_messages]
 type = "generator"
 format = "shuffle"
 lines = [
@@ -15,15 +32,41 @@ lines = [
 ]
 interval = 2
 
-[transforms.remap]
+# Generate coded error messages
+[sources.coded_error_messages]
+type = "generator"
+format = "shuffle"
+lines = [
+  '{"code":1,"device_id":"e5ad503d","timestamp":"2021-10-18T15:35:09.158139Z"}',
+  '{"code":2,"device_id":"a5b2401e","timestamp":"2021-10-18T15:35:28.517210Z"}',
+  '{"code":3,"device_id":"b48f41aa","timestamp":"2021-10-18T15:35:37.846783Z"}'
+]
+interval = 2
+
+[transforms.remap_user_info]
 type = "remap"
-inputs = ["random"]
+inputs = ["user_info_messages"]
 source = """
 . = parse_json(.message) ?? {}
 . |= get_enrichment_table_record("users", { "last_name": .last_name, "first_name": .first_name }) ?? {}
 """
 
+[transforms.remap_coded_errors]
+type = "remap"
+inputs = ["coded_error_messages"]
+source = """
+. = parse_json!(.message)
+
+row, err = get_enrichment_table_record("codes", { "code": del(.code) })
+
+if err != null {
+  log(err, level: "error")
+} else {
+  . |= merge(., {"message": row.message, "tag": row.tag})
+}
+"""
+
 [sinks.console]
 type = "console"
-inputs = ["remap"]
-encoding.codec = "json"
+inputs = ["remap_*"]
+encoding = { codec = "json" }