@@ -40,174 +40,64 @@ filelog/containers:
4040 include_file_name : false
4141 include_file_path : true
4242 operators :
43- # # Detect the container runtime log format
44- # # Can be: docker-shim, CRI-O and containerd
45- - id : get-format
43+ # # Parse the container runtime log format automatically
44+ - type : container
45+ add_metadata_from_filepath : true
46+
47+ # # Reorganise attributes according to Sumo Logic requirements
48+ # # - rename log.iostream to stream
49+ - type : move
50+ from : attributes["log.iostream"]
51+ to : attributes["stream"]
52+ # # Keep only the following attributes:
53+ # # - stream
54+ # # - k8s.pod.name
55+ # # - k8s.container.name
56+ # # - k8s.namespace.name
57+ # # - log.file.path
58+ # # - time if `sumologic.logs.container.keep_time_attribute` is set to `true`
59+ - type : retain
60+ id : keep-fields
61+ fields :
62+ - attributes["stream"]
63+ - attributes["k8s.pod.name"]
64+ - attributes["k8s.container.name"]
65+ - attributes["k8s.namespace.name"]
66+ - attributes["log.file.path"]
67+ {{ if .Values.sumologic.logs.container.keep_time_attribute }}
68+ - attributes["time"]
69+ {{ end }}
70+
71+ # # Strip trailing "\n" from the log body
72+ - id : strip-trailing-newline-router
73+ {{- if .Values.sumologic.logs.multiline.enabled }}
74+ default : multiline
75+ {{- else }}
76+ default : merge-multiline-logs
77+ {{- end }}
4678 routes :
47- - expr : ' body matches "^\\{"'
48- output : parser-docker
49- - expr : ' body matches "^[^ Z]+ "'
50- output : parser-crio
51- - expr : ' body matches "^[^ Z]+Z"'
52- output : parser-containerd
79+ - expr : body matches "^.*\n$"
80+ output : strip-trailing-newline
5381 type : router
5482
55- # # Parse CRI-O format
56- - id : parser-crio
57- output : merge-cri-lines
58- parse_to : body
59- regex : ' ^(?P<time>[^ Z]+) (?P<stream>stdout|stderr) (?P<logtag>[^ ]*)( |)(?P<log>.*)$'
60- timestamp :
61- layout : ' 2006-01-02T15:04:05.000000000-07:00'
62- layout_type : gotime
63- parse_from : body.time
64- type : regex_parser
65-
66- # # Parse CRI-Containerd format
67- - id : parser-containerd
68- output : merge-cri-lines
69- parse_to : body
70- regex : ' ^(?P<time>[^ ^Z]+Z) (?P<stream>stdout|stderr) (?P<logtag>[^ ]*)( |)(?P<log>.*)$'
71- timestamp :
72- layout : ' %Y-%m-%dT%H:%M:%S.%LZ'
73- parse_from : body.time
74- type : regex_parser
75-
76- # # Parse docker-shim format
77- # # parser-docker interprets the input string as JSON and moves the `time` field from the JSON to Timestamp field in the OTLP log
78- # # record.
79- # # Input Body (string): '{"log":"2001-02-03 04:05:06 first line\n","stream":"stdout","time":"2021-11-25T09:59:13.23887954Z"}'
80- # # Output Body (JSON): { "log": "2001-02-03 04:05:06 first line\n", "stream": "stdout" }
81- # # Input Timestamp: _empty_
82- # # Output Timestamp: 2021-11-25 09:59:13.23887954 +0000 UTC
83- - id : parser-docker
84- output : merge-docker-lines
85- parse_to : body
86- timestamp :
87- layout : ' %Y-%m-%dT%H:%M:%S.%LZ'
88- parse_from : body.time
89- type : json_parser
90-
91- # # merge-docker-lines stitches back together log lines split by Docker logging driver.
92- # # Input Body (JSON): { "log": "2001-02-03 04:05:06 very long li", "stream": "stdout" }
93- # # Input Body (JSON): { "log": "ne that was split by the logging driver\n", "stream": "stdout" }
94- # # Output Body (JSON): { "log": "2001-02-03 04:05:06 very long line that was split by the logging driver\n","stream":"stdout"}
95- - id : merge-docker-lines
96- combine_field : body.log
97- combine_with : " "
98- is_last_entry : body.log matches "\n$"
99- output : strip-trailing-newline
100- source_identifier : attributes["log.file.path"]
101- type : recombine
102- # # Ensure we combine everything up to `is_last_entry` even on the file beginning
103- max_unmatched_batch_size : 0
104-
105- # # merge-cri-lines stitches back together log lines split by CRI logging drivers.
106- # # Input Body (JSON): { "log": "2001-02-03 04:05:06 very long li", "logtag": "P" }
107- # # Input Body (JSON): { "log": "ne that was split by the logging driver", "logtag": "F" }
108- # # Output Body (JSON): { "log": "2001-02-03 04:05:06 very long line that was split by the logging driver", "logtag": "F" }
109- - id : merge-cri-lines
110- combine_field : body.log
111- combine_with : " "
112- is_last_entry : body.logtag == "F"
113- output : extract-metadata-from-filepath
114- overwrite_with : newest
115- source_identifier : attributes["log.file.path"]
116- type : recombine
117- # # Ensure we combine everything up to `is_last_entry` even on the file beginning
118- max_unmatched_batch_size : 0
119-
12083 # # strip-trailing-newline removes the trailing "\n" from the `log` key. This is required for logs coming from Docker container runtime.
84+ # # Uses attributes.log as temporary cotainer for new log
12185 # # Input Body (JSON): { "log": "2001-02-03 04:05:06 very long line that was split by the logging driver\n", "stream": "stdout" }
12286 # # Output Body (JSON): { "log": "2001-02-03 04:05:06 very long line that was split by the logging driver", "stream": "stdout" }
12387 - id : strip-trailing-newline
124- output : extract-metadata-from-filepath
125- parse_from : body.log
126- parse_to : body
88+ parse_from : body
89+ parse_to : attributes
90+ output : replace- body
12791 regex : " ^(?P<log>.*)\n $"
12892 type : regex_parser
12993
130- # # extract-metadata-from-filepath extracts data from the `log.file.path` Attribute into the Attributes
131- # # Input Attributes:
132- # # - log.file.path: '/var/log/pods/default_logger-multiline-4nvg4_aed49747-b541-4a07-8663-f7e1febc47d5/loggercontainer/0.log'
133- # # Output Attributes:
134- # # - log.file.path: '/var/log/pods/default_logger-multiline-4nvg4_aed49747-b541-4a07-8663-f7e1febc47d5/loggercontainer/0.log'
135- # # - container_name: "loggercontainer",
136- # # - namespace: "default",
137- # # - pod_name: "logger-multiline-4nvg4",
138- # # - run_id: "0",
139- # # - uid: "aed49747-b541-4a07-8663-f7e1febc47d5"
140- # # }
141- - id : extract-metadata-from-filepath
142- parse_from : attributes["log.file.path"]
143- {{ if eq .Type "linux" }}
144- regex : ' ^.*\/(?P<namespace>[^_]+)_(?P<pod_name>[^_]+)_(?P<uid>[a-f0-9\-]+)\/(?P<container_name>[^\._]+)\/(?P<run_id>\d+)\.log$'
145- {{ else if eq .Type "windows" }}
146- regex : ' ^.*\\(?P<namespace>[^_]+)_(?P<pod_name>[^_]+)_(?P<uid>[a-f0-9\-]+)\\(?P<container_name>[^\._]+)\\(?P<run_id>\d+)\.log$'
147- {{ else }}
148- fail "\nUnknown Type argument for `logs.collector.configuration.receivers.filelog-container` function"
149- {{ end }}
150- type : regex_parser
151-
152-
153- # # The following actions are being performed:
154- # # - renaming attributes
155- # # - moving stream from body to attributes
156- # # - using body.log as body
157- # # Input Body (JSON): {
158- # # "log": "2001-02-03 04:05:06 loggerlog 1 first line\n",
159- # # "stream": "stdout",
160- # # }
161- # # Output Body (String): "2001-02-03 04:05:06 loggerlog 1 first line\n"
162- # # Input Attributes:
163- # # - log.file.path: '/var/log/pods/default_logger-multiline-4nvg4_aed49747-b541-4a07-8663-f7e1febc47d5/loggercontainer/0.log'
164- # # - container_name: "loggercontainer",
165- # # - namespace: "default",
166- # # - pod_name: "logger-multiline-4nvg4",
167- # # - run_id: "0",
168- # # - uid: "aed49747-b541-4a07-8663-f7e1febc47d5"
169- # # Output Attributes:
170- # # - k8s.container.name: "loggercontainer"
171- # # - k8s.namespace.name: "default"
172- # # - k8s.pod.name: "logger-multiline-4nvg4"
173- # # - stream: "stdout"
174- # # - log.file.path: '/var/log/pods/default_logger-multiline-4nvg4_aed49747-b541-4a07-8663-f7e1febc47d5/loggercontainer/0.log'
175-
176- - id : move-attributes
177- from : body.stream
178- to : attributes["stream"]
179- type : move
180-
181- {{ if .Values.sumologic.logs.container.keep_time_attribute }}
182- - id : move-time-attribute
183- from : body.time
184- to : attributes["time"]
185- type : move
186- {{ end }}
187-
188- - from : attributes.container_name
189- to : attributes["k8s.container.name"]
94+ - id : replace-body
19095 type : move
191-
192- - from : attributes.namespace
193- to : attributes["k8s.namespace.name"]
194- type : move
195-
196- - from : attributes.pod_name
197- to : attributes["k8s.pod.name"]
198- type : move
199-
200- - from : body.log
96+ from : attributes.log
20197 to : body
202- type : move
203-
204- - field : attributes.run_id
205- type : remove
206-
207- - field : attributes.uid
208- type : remove
20998
21099{{- if .Values.sumologic.logs.multiline.enabled }}
100+ # # Perform multiline detection
211101 - id : multiline
212102 default : merge-multiline-logs
213103 routes :
0 commit comments