From c8ba3291ff1d468a542345b05cb120b5e32a3fb1 Mon Sep 17 00:00:00 2001 From: Jonah Feldman Date: Mon, 17 Nov 2025 11:54:55 -0500 Subject: [PATCH 01/12] OCSF pipeline for nginx --- nginx/assets/logs/nginx.yaml | 200 +++++++++++++++++++++++++++++++++-- 1 file changed, 193 insertions(+), 7 deletions(-) diff --git a/nginx/assets/logs/nginx.yaml b/nginx/assets/logs/nginx.yaml index e43e21e1a833c..1a637853d6ae3 100644 --- a/nginx/assets/logs/nginx.yaml +++ b/nginx/assets/logs/nginx.yaml @@ -59,7 +59,7 @@ facets: source: log pipeline: type: pipeline - name: Nginx + name: Nginx - OCSF enabled: true filter: query: source:nginx @@ -117,13 +117,13 @@ pipeline: samples: - HEAD http://174.138.82.103:80/sql/sql-admin/ HTTP/1.1 - type: url-parser - name: '' + name: "" enabled: true sources: - http.url target: http.url_details - type: user-agent-parser - name: '' + name: "" enabled: true sources: - http.useragent @@ -139,16 +139,16 @@ pipeline: enabled: true categories: - filter: - query: '@http.status_code:[200 TO 299]' + query: "@http.status_code:[200 TO 299]" name: OK - filter: - query: '@http.status_code:[300 TO 399]' + query: "@http.status_code:[300 TO 399]" name: notice - filter: - query: '@http.status_code:[400 TO 499]' + query: "@http.status_code:[400 TO 499]" name: warning - filter: - query: '@http.status_code:[500 TO 599]' + query: "@http.status_code:[500 TO 599]" name: error target: http.status_category - type: status-remapper @@ -157,3 +157,189 @@ pipeline: sources: - http.status_category - level + - type: pipeline + name: OCSF sub pipeline for HTTP Activity [4002] + enabled: true + ocsf: + isOcsf: true + filter: + query: "@http.method:*" + processors: + - type: string-builder-processor + name: Add ocsf.metadata.product.name + enabled: true + template: nginx + target: ocsf.metadata.product.name + replaceMissing: false + - type: schema-processor + name: OCSF Processor for HTTP Activity [4002] + enabled: true + mappers: + - type: schema-category-mapper + name: ocsf.activity_id + categories: + - filter: + query: "@http.method:CONNECT" + name: Connect + id: 1 + - filter: + query: "@http.method:DELETE" + name: Delete + id: 2 + - filter: + query: "@http.method:GET" + name: Get + id: 3 + - filter: + query: "@http.method:HEAD" + name: Head + id: 4 + - filter: + query: "@http.method:OPTIONS" + name: Options + id: 5 + - filter: + query: "@http.method:POST" + name: Post + id: 6 + - filter: + query: "@http.method:PUT" + name: Put + id: 7 + - filter: + query: "@http.method:TRACE" + name: Trace + id: 8 + - filter: + query: "@http.method:PATCH" + name: Patch + id: 9 + - filter: + query: "@http.method:*" + name: Other + id: 99 + targets: + name: ocsf.activity_id + id: ocsf.activity_id + - type: schema-category-mapper + name: ocsf.severity_id + categories: + - filter: + query: "@http.method:*" + name: Informational + id: 1 + targets: + name: ocsf.severity_id + id: ocsf.severity_id + - type: schema-remapper + name: Map `host` to `ocsf.dst_endpoint.ip` + sources: + - host + target: ocsf.dst_endpoint.ip + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `http.method` to `ocsf.http_request.http_method` + sources: + - http.method + target: ocsf.http_request.http_method + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `http.referer` to `ocsf.http_request.referrer` + sources: + - http.referer + target: ocsf.http_request.referrer + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `http.url_details.path` to `ocsf.http_request.url.path` + sources: + - http.url_details.path + target: ocsf.http_request.url.path + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `http.url_details.port` to `ocsf.http_request.url.port` + sources: + - http.url_details.port + target: ocsf.http_request.url.port + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `http.useragent` to `ocsf.http_request.user_agent` + sources: + - http.useragent + target: ocsf.http_request.user_agent + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `http.status_code` to `ocsf.http_response.code` + sources: + - http.status_code + target: ocsf.http_response.code + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `http.status_category` to `ocsf.http_response.status` + sources: + - http.status_category + target: ocsf.http_response.status + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `http.method` to `ocsf.metadata.event_code` + sources: + - http.method + target: ocsf.metadata.event_code + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `network.client.ip` to `ocsf.src_endpoint.ip` + sources: + - network.client.ip + target: ocsf.src_endpoint.ip + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `http.os.family` to `ocsf.src_endpoint.os` + sources: + - http.os.family + target: ocsf.src_endpoint.os + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `http.status_category` to `ocsf.status` + sources: + - http.status_category + target: ocsf.status + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `duration` to `ocsf.duration` + sources: + - http.os.family + target: ocsf.duration + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.product.name` to `ocsf.metadata.product.name` + sources: + - ocsf.metadata.product.name + target: ocsf.metadata.product.name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `date_access` to `ocsf.time` + sources: + - date_access + target: ocsf.time + preserveSource: true + overrideOnConflict: true + schema: + schema_type: ocsf + version: 1.5.0 + class_name: HTTP Activity + class_uid: 4002 + extensions: [] + profiles: [] From 649815dad35c9f9f318c8615518412f8da65bff0 Mon Sep 17 00:00:00 2001 From: Jonah Feldman Date: Mon, 17 Nov 2025 11:55:18 -0500 Subject: [PATCH 02/12] revert name --- nginx/assets/logs/nginx.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nginx/assets/logs/nginx.yaml b/nginx/assets/logs/nginx.yaml index 1a637853d6ae3..6776719fc1dff 100644 --- a/nginx/assets/logs/nginx.yaml +++ b/nginx/assets/logs/nginx.yaml @@ -59,7 +59,7 @@ facets: source: log pipeline: type: pipeline - name: Nginx - OCSF + name: Nginx enabled: true filter: query: source:nginx From 67cee7ae1592fd9ee59b0e58b482815e92e4593c Mon Sep 17 00:00:00 2001 From: Jonah Feldman Date: Mon, 17 Nov 2025 12:07:07 -0500 Subject: [PATCH 03/12] convert to camelcase --- nginx/assets/logs/nginx.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nginx/assets/logs/nginx.yaml b/nginx/assets/logs/nginx.yaml index 6776719fc1dff..b649f6862e67e 100644 --- a/nginx/assets/logs/nginx.yaml +++ b/nginx/assets/logs/nginx.yaml @@ -318,7 +318,7 @@ pipeline: - type: schema-remapper name: Map `duration` to `ocsf.duration` sources: - - http.os.family + - duration target: ocsf.duration preserveSource: true overrideOnConflict: true @@ -337,9 +337,9 @@ pipeline: preserveSource: true overrideOnConflict: true schema: - schema_type: ocsf + schemaType: ocsf version: 1.5.0 - class_name: HTTP Activity - class_uid: 4002 + clasName: HTTP Activity + clasUid: 4002 extensions: [] profiles: [] From 9e7728ed8ca6cc1785f052de4840a3dfbb695558 Mon Sep 17 00:00:00 2001 From: Jonah Feldman Date: Mon, 17 Nov 2025 12:22:49 -0500 Subject: [PATCH 04/12] fixed class name --- nginx/assets/logs/nginx.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nginx/assets/logs/nginx.yaml b/nginx/assets/logs/nginx.yaml index b649f6862e67e..313fb80a8e8f0 100644 --- a/nginx/assets/logs/nginx.yaml +++ b/nginx/assets/logs/nginx.yaml @@ -339,7 +339,7 @@ pipeline: schema: schemaType: ocsf version: 1.5.0 - clasName: HTTP Activity - clasUid: 4002 + className: HTTP Activity + classUid: 4002 extensions: [] profiles: [] From bfc17ecbe4a2ee5bfc471ab96da4330312db5882 Mon Sep 17 00:00:00 2001 From: Jonah Feldman Date: Mon, 17 Nov 2025 12:45:41 -0500 Subject: [PATCH 05/12] add sample logs --- nginx/assets/logs/nginx.yaml | 2 +- nginx/assets/logs/nginx_tests.yaml | 480 ++++++++++++++++++----------- 2 files changed, 298 insertions(+), 184 deletions(-) diff --git a/nginx/assets/logs/nginx.yaml b/nginx/assets/logs/nginx.yaml index 313fb80a8e8f0..065e26209823c 100644 --- a/nginx/assets/logs/nginx.yaml +++ b/nginx/assets/logs/nginx.yaml @@ -172,7 +172,7 @@ pipeline: target: ocsf.metadata.product.name replaceMissing: false - type: schema-processor - name: OCSF Processor for HTTP Activity [4002] + name: Apply OCSF schema for 4002 enabled: true mappers: - type: schema-category-mapper diff --git a/nginx/assets/logs/nginx_tests.yaml b/nginx/assets/logs/nginx_tests.yaml index 0ed058745fa61..34e88712a4c92 100644 --- a/nginx/assets/logs/nginx_tests.yaml +++ b/nginx/assets/logs/nginx_tests.yaml @@ -1,185 +1,299 @@ id: "nginx" tests: - - - sample: "127.0.0.1 - frank [13/Jul/2016:10:55:36 +0000] \"GET /apache_pb.gif HTTP/1.0\" 200 2326" - result: - custom: - date_access: 1468407336000 - http: - auth: "frank" - method: "GET" - status_category: "OK" - status_code: 200 - url: "/apache_pb.gif" - url_details: - path: "/apache_pb.gif" - version: "1.0" - network: - bytes_written: 2326 - client: - ip: "127.0.0.1" - message: "127.0.0.1 - frank [13/Jul/2016:10:55:36 +0000] \"GET /apache_pb.gif HTTP/1.0\" 200 2326" - status: "ok" - tags: - - "source:LOGS_SOURCE" - timestamp: 1468407336000 - - - sample: "172.17.0.1 - - [06/Jan/2017:16:16:37 +0000] \"GET /datadoghq/company?test=var1%20Pl HTTP/1.1\" 404 612 \"http://www.perdu.com/\" \"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36\" \"-\"" - result: - custom: - date_access: 1483719397000 - http: - method: "GET" - referer: "http://www.perdu.com/" - status_category: "warning" - status_code: 404 - url: "/datadoghq/company?test=var1%20Pl" - url_details: - path: "/datadoghq/company" - queryString: - test: "var1%20Pl" - useragent: "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36" - useragent_details: - browser: - family: "Chrome" - major: "55" - minor: "0" - patch: "2883" - patch_minor: "87" - device: - category: "Desktop" - family: "Other" - os: - family: "Linux" - version: "1.1" - network: - bytes_written: 612 - client: - ip: "172.17.0.1" - message: "172.17.0.1 - - [06/Jan/2017:16:16:37 +0000] \"GET /datadoghq/company?test=var1%20Pl HTTP/1.1\" 404 612 \"http://www.perdu.com/\" \"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36\" \"-\"" - status: "warn" - tags: - - "source:LOGS_SOURCE" - timestamp: 1483719397000 - - - sample: "172.17.0.1 - - [06/Jan/2017:16:16:37 +0000] \"GET /datadoghq/company?test=var1%20Pl HTTP/1.1\" 200 612 \"http://www.perdu.com/\" \"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36\" \"-\" those are random characters" - result: - custom: - date_access: 1483719397000 - http: - method: "GET" - referer: "http://www.perdu.com/" - status_category: "OK" - status_code: 200 - url: "/datadoghq/company?test=var1%20Pl" - url_details: - path: "/datadoghq/company" - queryString: - test: "var1%20Pl" - useragent: "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36" - useragent_details: - browser: - family: "Chrome" - major: "55" - minor: "0" - patch: "2883" - patch_minor: "87" - device: - category: "Desktop" - family: "Other" - os: - family: "Linux" - version: "1.1" - network: - bytes_written: 612 - client: - ip: "172.17.0.1" - message: "172.17.0.1 - - [06/Jan/2017:16:16:37 +0000] \"GET /datadoghq/company?test=var1%20Pl HTTP/1.1\" 200 612 \"http://www.perdu.com/\" \"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36\" \"-\" those are random characters" - status: "ok" - tags: - - "source:LOGS_SOURCE" - timestamp: 1483719397000 - - - sample: "2017/09/26 14:36:50 [error] 8409#8409: *317058 \"/usr/share/nginx/html/sql/sql-admin/index.html\" is not found (2: No such file or directory), client: 217.92.148.44, server: localhost, request: \"HEAD http://174.138.82.103:80/sql/sql-admin/ HTTP/1.1\", host: \"174.138.82.103\"" - result: - custom: - date_access: 1506436610000 - error: - message: "8409#8409: *317058 \"/usr/share/nginx/html/sql/sql-admin/index.html\" is not found (2: No such file or directory)" - host: "174.138.82.103" - http: - method: "HEAD" - url: "http://174.138.82.103:80/sql/sql-admin/" - url_details: - host: "174.138.82.103" - path: "/sql/sql-admin/" - port: 80 - scheme: "http" - version: "1.1" - level: "error" - network: - client: - ip: "217.92.148.44" - request: "HEAD http://174.138.82.103:80/sql/sql-admin/ HTTP/1.1" - server: "localhost" - message: "2017/09/26 14:36:50 [error] 8409#8409: *317058 \"/usr/share/nginx/html/sql/sql-admin/index.html\" is not found (2: No such file or directory), client: 217.92.148.44, server: localhost, request: \"HEAD http://174.138.82.103:80/sql/sql-admin/ HTTP/1.1\", host: \"174.138.82.103\"" - status: "error" - tags: - - "source:LOGS_SOURCE" - timestamp: 1506436610000 - - - sample: "2017/09/26 14:36:50 [info] 14#14: *285 client 172.17.0.27 closed keepalive connection" - result: - custom: - date_access: 1506436610000 - error: - message: "14#14: *285 client 172.17.0.27 closed keepalive connection" - level: "info" - message: "2017/09/26 14:36:50 [info] 14#14: *285 client 172.17.0.27 closed keepalive connection" - status: "info" - tags: - - "source:LOGS_SOURCE" - timestamp: 1506436610000 - - - sample: "127.0.0.1 - - [19/Feb/2015:15:50:36 -0500] \"GET /big.pdf HTTP/1.1\" 206 33973115 0.202 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.111 Safari/537.36\"" - result: - custom: - date_access: 1424379036000 - duration: 2.02E8 - http: - method: "GET" - referer: "-" - status_category: "OK" - status_code: 206 - url: "/big.pdf" - url_details: - path: "/big.pdf" - useragent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.111 Safari/537.36" - useragent_details: - browser: - family: "Chrome" - major: "40" - minor: "0" - patch: "2214" - patch_minor: "111" - device: - brand: "Apple" - category: "Desktop" - family: "Mac" - model: "Mac" - os: - family: "Mac OS X" - major: "10" - minor: "10" - patch: "1" - version: "1.1" - network: - bytes_written: 33973115 - client: - ip: "127.0.0.1" - message: "127.0.0.1 - - [19/Feb/2015:15:50:36 -0500] \"GET /big.pdf HTTP/1.1\" 206 33973115 0.202 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.111 Safari/537.36\"" - status: "ok" - tags: - - "source:LOGS_SOURCE" - timestamp: 1424379036000 - + - sample: '127.0.0.1 - frank [13/Jul/2016:10:55:36 +0000] "GET /apache_pb.gif HTTP/1.0" 200 2326' + result: + custom: + date_access: 1468407336000 + http: + auth: "frank" + method: "GET" + status_category: "OK" + status_code: 200 + url: "/apache_pb.gif" + url_details: + path: "/apache_pb.gif" + version: "1.0" + network: + bytes_written: 2326 + client: + ip: "127.0.0.1" + ocsf: + activity_id: 3 + category_name: "Network Activity" + category_uid: 4 + class_name: "HTTP Activity" + class_uid: 4002 + http_request: + http_method: "GET" + url: + path: "/apache_pb.gif" + http_response: + code: 200 + status: "OK" + metadata: + event_code: "GET" + product: + name: "nginx" + version: "1.5.0" + severity_id: 1 + src_endpoint: + ip: "127.0.0.1" + status: "OK" + time: 1468407336000 + message: '127.0.0.1 - frank [13/Jul/2016:10:55:36 +0000] "GET /apache_pb.gif HTTP/1.0" 200 2326' + status: "ok" + tags: + - "source:LOGS_SOURCE" + timestamp: 1468407336000 + - sample: '172.17.0.1 - - [06/Jan/2017:16:16:37 +0000] "GET /datadoghq/company?test=var1%20Pl HTTP/1.1" 404 612 "http://www.perdu.com/" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36" "-"' + result: + custom: + date_access: 1483719397000 + http: + method: "GET" + referer: "http://www.perdu.com/" + status_category: "warning" + status_code: 404 + url: "/datadoghq/company?test=var1%20Pl" + url_details: + path: "/datadoghq/company" + queryString: + test: "var1%20Pl" + useragent: "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36" + useragent_details: + browser: + family: "Chrome" + major: "55" + minor: "0" + patch: "2883" + patch_minor: "87" + device: + category: "Desktop" + family: "Other" + os: + family: "Linux" + version: "1.1" + network: + bytes_written: 612 + client: + ip: "172.17.0.1" + ocsf: + activity_id: 3 + category_name: "Network Activity" + category_uid: 4 + class_name: "HTTP Activity" + class_uid: 4002 + http_request: + http_method: "GET" + referrer: "http://www.perdu.com/" + url: + path: "/datadoghq/company" + user_agent: "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36" + http_response: + code: 404 + status: "warning" + metadata: + event_code: "GET" + product: + name: "nginx" + version: "1.5.0" + severity_id: 1 + src_endpoint: + ip: "172.17.0.1" + status: "warning" + time: 1483719397000 + message: '172.17.0.1 - - [06/Jan/2017:16:16:37 +0000] "GET /datadoghq/company?test=var1%20Pl HTTP/1.1" 404 612 "http://www.perdu.com/" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36" "-"' + status: "warn" + tags: + - "source:LOGS_SOURCE" + timestamp: 1483719397000 + - sample: '172.17.0.1 - - [06/Jan/2017:16:16:37 +0000] "GET /datadoghq/company?test=var1%20Pl HTTP/1.1" 200 612 "http://www.perdu.com/" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36" "-" those are random characters' + result: + custom: + date_access: 1483719397000 + http: + method: "GET" + referer: "http://www.perdu.com/" + status_category: "OK" + status_code: 200 + url: "/datadoghq/company?test=var1%20Pl" + url_details: + path: "/datadoghq/company" + queryString: + test: "var1%20Pl" + useragent: "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36" + useragent_details: + browser: + family: "Chrome" + major: "55" + minor: "0" + patch: "2883" + patch_minor: "87" + device: + category: "Desktop" + family: "Other" + os: + family: "Linux" + version: "1.1" + network: + bytes_written: 612 + client: + ip: "172.17.0.1" + ocsf: + activity_id: 3 + category_name: "Network Activity" + category_uid: 4 + class_name: "HTTP Activity" + class_uid: 4002 + http_request: + http_method: "GET" + referrer: "http://www.perdu.com/" + url: + path: "/datadoghq/company" + user_agent: "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36" + http_response: + code: 200 + status: "OK" + metadata: + event_code: "GET" + product: + name: "nginx" + version: "1.5.0" + severity_id: 1 + src_endpoint: + ip: "172.17.0.1" + status: "OK" + time: 1483719397000 + message: '172.17.0.1 - - [06/Jan/2017:16:16:37 +0000] "GET /datadoghq/company?test=var1%20Pl HTTP/1.1" 200 612 "http://www.perdu.com/" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36" "-" those are random characters' + status: "ok" + tags: + - "source:LOGS_SOURCE" + timestamp: 1483719397000 + - sample: '2017/09/26 14:36:50 [error] 8409#8409: *317058 "/usr/share/nginx/html/sql/sql-admin/index.html" is not found (2: No such file or directory), client: 217.92.148.44, server: localhost, request: "HEAD http://174.138.82.103:80/sql/sql-admin/ HTTP/1.1", host: "174.138.82.103"' + result: + custom: + date_access: 1506436610000 + error: + message: '8409#8409: *317058 "/usr/share/nginx/html/sql/sql-admin/index.html" is not found (2: No such file or directory)' + host: "174.138.82.103" + http: + method: "HEAD" + url: "http://174.138.82.103:80/sql/sql-admin/" + url_details: + host: "174.138.82.103" + path: "/sql/sql-admin/" + port: 80 + scheme: "http" + version: "1.1" + level: "error" + network: + client: + ip: "217.92.148.44" + ocsf: + activity_id: 4 + category_name: "Network Activity" + category_uid: 4 + class_name: "HTTP Activity" + class_uid: 4002 + dst_endpoint: + ip: "174.138.82.103" + http_request: + http_method: "HEAD" + url: + path: "/sql/sql-admin/" + port: 80 + metadata: + event_code: "HEAD" + product: + name: "nginx" + version: "1.5.0" + severity_id: 1 + src_endpoint: + ip: "217.92.148.44" + time: 1506436610000 + request: "HEAD http://174.138.82.103:80/sql/sql-admin/ HTTP/1.1" + server: "localhost" + message: '2017/09/26 14:36:50 [error] 8409#8409: *317058 "/usr/share/nginx/html/sql/sql-admin/index.html" is not found (2: No such file or directory), client: 217.92.148.44, server: localhost, request: "HEAD http://174.138.82.103:80/sql/sql-admin/ HTTP/1.1", host: "174.138.82.103"' + status: "error" + tags: + - "source:LOGS_SOURCE" + timestamp: 1506436610000 + - sample: "2017/09/26 14:36:50 [info] 14#14: *285 client 172.17.0.27 closed keepalive connection" + result: + custom: + date_access: 1506436610000 + error: + message: "14#14: *285 client 172.17.0.27 closed keepalive connection" + level: "info" + message: "2017/09/26 14:36:50 [info] 14#14: *285 client 172.17.0.27 closed keepalive connection" + status: "info" + tags: + - "source:LOGS_SOURCE" + timestamp: 1506436610000 + - sample: '127.0.0.1 - - [19/Feb/2015:15:50:36 -0500] "GET /big.pdf HTTP/1.1" 206 33973115 0.202 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.111 Safari/537.36"' + result: + custom: + date_access: 1424379036000 + duration: 2.02E8 + http: + method: "GET" + referer: "-" + status_category: "OK" + status_code: 206 + url: "/big.pdf" + url_details: + path: "/big.pdf" + useragent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.111 Safari/537.36" + useragent_details: + browser: + family: "Chrome" + major: "40" + minor: "0" + patch: "2214" + patch_minor: "111" + device: + brand: "Apple" + category: "Desktop" + family: "Mac" + model: "Mac" + os: + family: "Mac OS X" + major: "10" + minor: "10" + patch: "1" + version: "1.1" + network: + bytes_written: 33973115 + client: + ip: "127.0.0.1" + ocsf: + activity_id: 3 + category_name: "Network Activity" + category_uid: 4 + class_name: "HTTP Activity" + class_uid: 4002 + duration: 2.02E8 + http_request: + http_method: "GET" + referrer: "-" + url: + path: "/big.pdf" + user_agent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.111 Safari/537.36" + http_response: + code: 206 + status: "OK" + metadata: + event_code: "GET" + product: + name: "nginx" + version: "1.5.0" + severity_id: 1 + src_endpoint: + ip: "127.0.0.1" + status: "OK" + time: 1424379036000 + message: '127.0.0.1 - - [19/Feb/2015:15:50:36 -0500] "GET /big.pdf HTTP/1.1" 206 33973115 0.202 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.111 Safari/537.36"' + status: "ok" + tags: + - "source:LOGS_SOURCE" + timestamp: 1424379036000 From 03af30e3fbb5eebb1268e0c54932b1e44c045c20 Mon Sep 17 00:00:00 2001 From: Jonah Feldman Date: Tue, 18 Nov 2025 13:29:59 -0500 Subject: [PATCH 06/12] put status in category-mapper --- nginx/assets/logs/nginx.yaml | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/nginx/assets/logs/nginx.yaml b/nginx/assets/logs/nginx.yaml index 065e26209823c..444c7169b1270 100644 --- a/nginx/assets/logs/nginx.yaml +++ b/nginx/assets/logs/nginx.yaml @@ -231,6 +231,24 @@ pipeline: targets: name: ocsf.severity_id id: ocsf.severity_id + - type: schema-category-mapper + name: ocsf.status_id + categories: + - filter: + query: "@http.status_code:[200 TO 399]" + name: Success + id: 1 + - filter: + query: "@http.status_code:[400 TO 599]" + name: Failure + id: 2 + - filter: + query: "*" + name: Unknown + id: 99 + targets: + name: ocsf.status_id + id: ocsf.status_id - type: schema-remapper name: Map `host` to `ocsf.dst_endpoint.ip` sources: @@ -309,10 +327,10 @@ pipeline: preserveSource: true overrideOnConflict: true - type: schema-remapper - name: Map `http.status_category` to `ocsf.status` + name: Map `http.status_code` to `ocsf.status_code` sources: - http.status_category - target: ocsf.status + target: ocsf.status_code preserveSource: true overrideOnConflict: true - type: schema-remapper From 1199dd4513f18319adb3fe44592a9eb046b9c987 Mon Sep 17 00:00:00 2001 From: Jonah Feldman Date: Wed, 19 Nov 2025 10:11:33 -0500 Subject: [PATCH 07/12] fixed status mapping --- nginx/assets/logs/nginx.yaml | 2 +- nginx/assets/logs/nginx_tests.yaml | 18 ++++++++++++++---- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/nginx/assets/logs/nginx.yaml b/nginx/assets/logs/nginx.yaml index 444c7169b1270..6eb96a51fa759 100644 --- a/nginx/assets/logs/nginx.yaml +++ b/nginx/assets/logs/nginx.yaml @@ -329,7 +329,7 @@ pipeline: - type: schema-remapper name: Map `http.status_code` to `ocsf.status_code` sources: - - http.status_category + - http.status_code target: ocsf.status_code preserveSource: true overrideOnConflict: true diff --git a/nginx/assets/logs/nginx_tests.yaml b/nginx/assets/logs/nginx_tests.yaml index 34e88712a4c92..86ddff29e3a62 100644 --- a/nginx/assets/logs/nginx_tests.yaml +++ b/nginx/assets/logs/nginx_tests.yaml @@ -38,7 +38,9 @@ tests: severity_id: 1 src_endpoint: ip: "127.0.0.1" - status: "OK" + status: "Success" + status_code: 200 + status_id: 1 time: 1468407336000 message: '127.0.0.1 - frank [13/Jul/2016:10:55:36 +0000] "GET /apache_pb.gif HTTP/1.0" 200 2326' status: "ok" @@ -100,7 +102,9 @@ tests: severity_id: 1 src_endpoint: ip: "172.17.0.1" - status: "warning" + status: "Failure" + status_code: 404 + status_id: 2 time: 1483719397000 message: '172.17.0.1 - - [06/Jan/2017:16:16:37 +0000] "GET /datadoghq/company?test=var1%20Pl HTTP/1.1" 404 612 "http://www.perdu.com/" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36" "-"' status: "warn" @@ -162,7 +166,9 @@ tests: severity_id: 1 src_endpoint: ip: "172.17.0.1" - status: "OK" + status: "Success" + status_code: 200 + status_id: 1 time: 1483719397000 message: '172.17.0.1 - - [06/Jan/2017:16:16:37 +0000] "GET /datadoghq/company?test=var1%20Pl HTTP/1.1" 200 612 "http://www.perdu.com/" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36" "-" those are random characters' status: "ok" @@ -210,6 +216,8 @@ tests: severity_id: 1 src_endpoint: ip: "217.92.148.44" + status: Unknown + status_id: 99 time: 1506436610000 request: "HEAD http://174.138.82.103:80/sql/sql-admin/ HTTP/1.1" server: "localhost" @@ -290,7 +298,9 @@ tests: severity_id: 1 src_endpoint: ip: "127.0.0.1" - status: "OK" + status: "Success" + status_code: 206 + status_id: 1 time: 1424379036000 message: '127.0.0.1 - - [19/Feb/2015:15:50:36 -0500] "GET /big.pdf HTTP/1.1" 206 33973115 0.202 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.111 Safari/537.36"' status: "ok" From c1a608d483f6a9e2ecd0eb19c0ffff821b870bcb Mon Sep 17 00:00:00 2001 From: Jonah Feldman Date: Wed, 19 Nov 2025 11:14:51 -0500 Subject: [PATCH 08/12] mapping + vendor name --- nginx/assets/logs/nginx.yaml | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/nginx/assets/logs/nginx.yaml b/nginx/assets/logs/nginx.yaml index 6eb96a51fa759..26007d6b532f8 100644 --- a/nginx/assets/logs/nginx.yaml +++ b/nginx/assets/logs/nginx.yaml @@ -168,9 +168,16 @@ pipeline: - type: string-builder-processor name: Add ocsf.metadata.product.name enabled: true - template: nginx + template: "nginx" target: ocsf.metadata.product.name replaceMissing: false + - type: string-builder-processor + name: Add ocsf.metadata.product.vendor_name + enabled: true + template: "F5" + target: ocsf.metadata.product.name + replaceMissing: false + - type: schema-processor name: Apply OCSF schema for 4002 enabled: true @@ -219,7 +226,7 @@ pipeline: name: Other id: 99 targets: - name: ocsf.activity_id + name: ocsf.activity_name id: ocsf.activity_id - type: schema-category-mapper name: ocsf.severity_id @@ -229,7 +236,7 @@ pipeline: name: Informational id: 1 targets: - name: ocsf.severity_id + name: ocsf.severity id: ocsf.severity_id - type: schema-category-mapper name: ocsf.status_id @@ -247,7 +254,7 @@ pipeline: name: Unknown id: 99 targets: - name: ocsf.status_id + name: ocsf.status id: ocsf.status_id - type: schema-remapper name: Map `host` to `ocsf.dst_endpoint.ip` @@ -347,6 +354,13 @@ pipeline: target: ocsf.metadata.product.name preserveSource: true overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.product.vendor_name` to `ocsf.metadata.product.vendor_name` + sources: + - ocsf.metadata.product.vendor_name + target: ocsf.metadata.product.vendor_name + preserveSource: true + overrideOnConflict: true - type: schema-remapper name: Map `date_access` to `ocsf.time` sources: From 731aab20f5d3fc5f3ef9bfed6de6ce67fe29c77f Mon Sep 17 00:00:00 2001 From: Jonah Feldman Date: Wed, 19 Nov 2025 14:43:11 -0500 Subject: [PATCH 09/12] fix test output --- nginx/assets/logs/nginx_tests.yaml | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/nginx/assets/logs/nginx_tests.yaml b/nginx/assets/logs/nginx_tests.yaml index 86ddff29e3a62..f2c3a14b8a103 100644 --- a/nginx/assets/logs/nginx_tests.yaml +++ b/nginx/assets/logs/nginx_tests.yaml @@ -19,6 +19,7 @@ tests: ip: "127.0.0.1" ocsf: activity_id: 3 + activity_name: "Get" category_name: "Network Activity" category_uid: 4 class_name: "HTTP Activity" @@ -33,8 +34,9 @@ tests: metadata: event_code: "GET" product: - name: "nginx" + name: "F5" version: "1.5.0" + severity: "Informational" severity_id: 1 src_endpoint: ip: "127.0.0.1" @@ -81,6 +83,7 @@ tests: ip: "172.17.0.1" ocsf: activity_id: 3 + activity_name: "Get" category_name: "Network Activity" category_uid: 4 class_name: "HTTP Activity" @@ -97,8 +100,9 @@ tests: metadata: event_code: "GET" product: - name: "nginx" + name: "F5" version: "1.5.0" + severity: "Informational" severity_id: 1 src_endpoint: ip: "172.17.0.1" @@ -145,6 +149,7 @@ tests: ip: "172.17.0.1" ocsf: activity_id: 3 + activity_name: "Get" category_name: "Network Activity" category_uid: 4 class_name: "HTTP Activity" @@ -161,8 +166,9 @@ tests: metadata: event_code: "GET" product: - name: "nginx" + name: "F5" version: "1.5.0" + severity: "Informational" severity_id: 1 src_endpoint: ip: "172.17.0.1" @@ -197,6 +203,7 @@ tests: ip: "217.92.148.44" ocsf: activity_id: 4 + activity_name: "Head" category_name: "Network Activity" category_uid: 4 class_name: "HTTP Activity" @@ -211,12 +218,13 @@ tests: metadata: event_code: "HEAD" product: - name: "nginx" + name: "F5" version: "1.5.0" + severity: "Informational" severity_id: 1 src_endpoint: ip: "217.92.148.44" - status: Unknown + status: "Unknown" status_id: 99 time: 1506436610000 request: "HEAD http://174.138.82.103:80/sql/sql-admin/ HTTP/1.1" @@ -276,6 +284,7 @@ tests: ip: "127.0.0.1" ocsf: activity_id: 3 + activity_name: "Get" category_name: "Network Activity" category_uid: 4 class_name: "HTTP Activity" @@ -293,8 +302,9 @@ tests: metadata: event_code: "GET" product: - name: "nginx" + name: "F5" version: "1.5.0" + severity: "Informational" severity_id: 1 src_endpoint: ip: "127.0.0.1" From 6e99f32fdb5e2d48d8d70643a37d69f0df5091b0 Mon Sep 17 00:00:00 2001 From: Jonah Feldman Date: Mon, 24 Nov 2025 15:03:59 -0500 Subject: [PATCH 10/12] targetformat for status code --- nginx/assets/logs/nginx.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/nginx/assets/logs/nginx.yaml b/nginx/assets/logs/nginx.yaml index 26007d6b532f8..bbbf7df7dc70a 100644 --- a/nginx/assets/logs/nginx.yaml +++ b/nginx/assets/logs/nginx.yaml @@ -339,6 +339,7 @@ pipeline: - http.status_code target: ocsf.status_code preserveSource: true + targetFormat: string overrideOnConflict: true - type: schema-remapper name: Map `duration` to `ocsf.duration` From bbc9750bb11ddac4ea2da164c0d7418f429466dc Mon Sep 17 00:00:00 2001 From: Jonah Feldman Date: Wed, 26 Nov 2025 11:23:16 -0500 Subject: [PATCH 11/12] cast to string in test output --- nginx/assets/logs/nginx_tests.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nginx/assets/logs/nginx_tests.yaml b/nginx/assets/logs/nginx_tests.yaml index f2c3a14b8a103..70fc726ced460 100644 --- a/nginx/assets/logs/nginx_tests.yaml +++ b/nginx/assets/logs/nginx_tests.yaml @@ -41,7 +41,7 @@ tests: src_endpoint: ip: "127.0.0.1" status: "Success" - status_code: 200 + status_code: "200" status_id: 1 time: 1468407336000 message: '127.0.0.1 - frank [13/Jul/2016:10:55:36 +0000] "GET /apache_pb.gif HTTP/1.0" 200 2326' @@ -173,7 +173,7 @@ tests: src_endpoint: ip: "172.17.0.1" status: "Success" - status_code: 200 + status_code: "200" status_id: 1 time: 1483719397000 message: '172.17.0.1 - - [06/Jan/2017:16:16:37 +0000] "GET /datadoghq/company?test=var1%20Pl HTTP/1.1" 200 612 "http://www.perdu.com/" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36" "-" those are random characters' @@ -309,7 +309,7 @@ tests: src_endpoint: ip: "127.0.0.1" status: "Success" - status_code: 206 + status_code: "206" status_id: 1 time: 1424379036000 message: '127.0.0.1 - - [19/Feb/2015:15:50:36 -0500] "GET /big.pdf HTTP/1.1" 206 33973115 0.202 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.111 Safari/537.36"' From 608f29cea54b43267df42c9dc9a1b5c00c0f9f87 Mon Sep 17 00:00:00 2001 From: Jonah Feldman Date: Wed, 26 Nov 2025 11:41:59 -0500 Subject: [PATCH 12/12] one more sample log --- nginx/assets/logs/nginx_tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nginx/assets/logs/nginx_tests.yaml b/nginx/assets/logs/nginx_tests.yaml index 70fc726ced460..7e9570f9656d7 100644 --- a/nginx/assets/logs/nginx_tests.yaml +++ b/nginx/assets/logs/nginx_tests.yaml @@ -107,7 +107,7 @@ tests: src_endpoint: ip: "172.17.0.1" status: "Failure" - status_code: 404 + status_code: "404" status_id: 2 time: 1483719397000 message: '172.17.0.1 - - [06/Jan/2017:16:16:37 +0000] "GET /datadoghq/company?test=var1%20Pl HTTP/1.1" 404 612 "http://www.perdu.com/" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36" "-"'