Merge pull request #390 from splunk/requirement_test/modinput

feat: requirement test modinput TA's

Author: nandinivij

pytest_splunk_addon/standard_lib/event_ingestors/ingestor_helper.py

@@ -38,6 +38,21 @@ def get_event_ingestor(cls, input_type, ingest_meta_data):
         LOGGER.debug("Using the following HEC ingestor: {}".format(str(ingestor)))
         return ingestor
 
+    @classmethod
+    def get_consolidated_events(cls, events):
+        ingestor_dict = dict()
+        for event in events:
+            input_type = event.metadata.get("input_type")
+            if input_type in ["modinput", "windows_input", "syslog_tcp", "syslog_udp", "uf_file_monitor"]:
+                event.event = event.event.encode("utf-8").decode()
+            else:
+                event.event = event.event.encode("utf-8")
+            if input_type in ingestor_dict:
+                ingestor_dict[input_type].append(event)
+            else:
+                ingestor_dict[input_type] = [event]
+        return ingestor_dict
+
     @classmethod
     def ingest_events(
         cls,
@@ -60,23 +75,7 @@ def ingest_events(
         sample_generator = SampleXdistGenerator(addon_path, config_path)
         store_sample = sample_generator.get_samples(store_events)
         tokenized_events = store_sample.get("tokenized_events")
-        ingestor_dict = dict()
-        for event in tokenized_events:
-            input_type = event.metadata.get("input_type")
-            if input_type in [
-                "modinput",
-                "windows_input",
-                "syslog_tcp",
-                "syslog_udp",
-                "uf_file_monitor",
-            ]:
-                event.event = event.event.encode("utf-8").decode()
-            else:
-                event.event = event.event.encode("utf-8")
-            if input_type in ingestor_dict:
-                ingestor_dict[input_type].append(event)
-            else:
-                ingestor_dict[input_type] = [event]
+        ingestor_dict = cls.get_consolidated_events(tokenized_events)
         for input_type, events in ingestor_dict.items():
             LOGGER.debug(
                 "Received the following input type for HEC event: {}".format(input_type)
@@ -85,9 +84,12 @@ def ingest_events(
             event_ingestor.ingest(events, thread_count)
 
         if run_requirement_test != "None":
-            requirement_event = RequirementEventIngestor(run_requirement_test)
-            events = requirement_event.get_events()
-            input_type = "syslog_tcp"
-            event_ingestor = cls.get_event_ingestor(input_type, ingest_meta_data)
-            event_ingestor.ingest(events, thread_count)
+            requirement_events = RequirementEventIngestor(run_requirement_test)
+            requirement_events_get = requirement_events.get_events()
+            requirement_events_dict = cls.get_consolidated_events(requirement_events_get)
+            for input_type, events in requirement_events_dict.items():
+                LOGGER.debug(
+                    "Received the following input type for HEC event: {}".format(input_type))
+                event_ingestor = cls.get_event_ingestor(input_type, ingest_meta_data)
+                event_ingestor.ingest(events, thread_count)
             LOGGER.info("Ingestion Done")

pytest_splunk_addon/standard_lib/event_ingestors/requirement_event_ingester.py

@@ -73,9 +73,22 @@ def get_models(self, root):
             model_list.append(str(model.text))
         return model_list
 
+    #extract_params_transport
+    def extract_params(self, event):
+        host, source, source_type = "", "", ""
+        for transport in event.iter('transport'):
+            if transport.get('host'):
+                host = transport.get('host')
+            if transport.get('source'):
+                source = transport.get('source')
+            if transport.get('sourcetype'):
+                source_type = transport.get('sourcetype')
+        return host, source, source_type
+
     def get_events(self):
         req_file_path = self.requirement_file_path
         events = []
+        host, source, sourcetype = "", "",""
         if os.path.isdir(req_file_path):
             for file1 in os.listdir(req_file_path):
                 filename = os.path.join(req_file_path, file1)
@@ -87,24 +100,30 @@ def get_events(self):
                             if len(model_list) != 0:
                                 transport_type = self.extract_transport_tag(event_tag)
                                 if transport_type == "syslog":
-                                    LOGGER.info(
-                                        "sending data using sc4s {}".format(filename)
-                                    )
+                                    transport_type = "syslog_tcp"
+                                    LOGGER.info("sending data using sc4s {}".format(filename))
+                                elif transport_type in ("modinput","Modinput", "Mod input","Modular Input", "Modular input", "modular input","modular_input", "Mod Input","hec_event"):
+                                    transport_type = "modinput"
+                                    LOGGER.info("sending data via HEC {}".format(filename))
+                                    host, source, sourcetype = self.extract_params(event_tag)
+                                    LOGGER.info(f"sending data via HEC {host}, {source} {sourcetype}")
+                                elif transport_type == "dbx":
+                                    transport_type = "modinput"
+                                elif transport_type == "windows_input":
+                                    transport_type = "windows_input"
                                 else:
                                     transport_type = "default"
                                 unescaped_event = self.extract_raw_events(event_tag)
-                                escaped_ingest = self.escape_before_ingest(
-                                    unescaped_event
-                                )
-                                metadata = {
-                                    "input_type": transport_type,
-                                    "index": "main",
-                                }
-                                events.append(
-                                    SampleEvent(
-                                        escaped_ingest, metadata, "requirement_test"
-                                    )
-                                )
+                                escaped_ingest = self.escape_before_ingest(unescaped_event)
+                                metadata = {'input_type': transport_type,
+                                            'index': 'main',
+                                            "source": source,
+                                            "host": host,
+                                            "sourcetype": sourcetype,
+                                            "timestamp_type": "event",
+                                            }
+                                events.append(SampleEvent(escaped_ingest, metadata, "requirement_test"))
+
                             else:
                                 # if there is no model in event do not ingest that event
                                 continue

pytest_splunk_addon/standard_lib/requirement_tests/requirement_test_datamodel_tag_constants.py

@@ -81,5 +81,5 @@
     "Updates_Update_Errors": ["update", "error"],
     "Vulnerabilities": ["report", "vulnerability"],
     "Web": ["web"],
-    "Web_proxy": ["web", "proxy"],
+    "Web_Proxy": ["web", "proxy"],
 }

pytest_splunk_addon/standard_lib/requirement_tests/test_generator.py

@@ -97,6 +97,7 @@ def generate_cim_req_params(self):
         """
         req_file_path = self.folder_path
         req_test_id = 0
+        modinput_params = None
         if os.path.isdir(req_file_path):
             for file1 in os.listdir(req_file_path):
                 filename = os.path.join(req_file_path, file1)
@@ -113,17 +114,24 @@ def generate_cim_req_params(self):
                         event_no += 1
                         unescaped_event = self.get_event(event_tag)
                         transport_type = self.extract_transport_tag(event_tag)
-                        if transport_type == "syslog":
+                        if transport_type.lower() == "syslog":
                             stripped_event = self.strip_syslog_header(unescaped_event)
+                            unescaped_event = stripped_event
+                            if stripped_event is None:
+                                LOGGER.error("Syslog event do not match CEF, RFC_3164, RFC_5424 format")
+                                continue
+                        elif transport_type in ("modinput","Modinput", "Mod input","Modular Input", "Modular input", "modular input","modular_input", "Mod Input", "dbx", "windows_input","hec_event"):
+                            host, source, sourcetype = self.extract_params(event_tag)
+                            modinput_params = {
+                                "host": host,
+                                "source": source,
+                                "sourcetype": sourcetype
+                            }
                         else:
-                            # todo: non syslog events are skipped currently until we support it
+                            # todo: non syslog/modinput events are skipped currently until we support it
                             continue
-                        if stripped_event is None:
-                            LOGGER.error(
-                                "Syslog event do not match CEF, RFC_3164, RFC_5424 format"
-                            )
-                            continue
-                        escaped_event = self.escape_char_event(stripped_event)
+
+                        escaped_event = self.escape_char_event(unescaped_event)
                         model_list = self.get_models(event_tag)
                         # Fetching kay value pair from XML
                         key_value_dict = self.extract_key_value_xml(event_tag)
@@ -144,6 +152,8 @@ def generate_cim_req_params(self):
                                 "model_list": list_model_dataset_subdataset,
                                 "escaped_event": escaped_event,
                                 "Key_value_dict": key_value_dict,
+                                "modinput_params":  modinput_params,
+                                "transport_type": transport_type,
                             },
                             id=f"{model_list}::{filename}::event_no::{event_no}::req_test_id::{req_test_id}",
                         )
@@ -206,14 +216,26 @@ def check_xml_format(self, file_name):
         else:
             return False
 
+        # extract_params_transport
+
+    def extract_params(self, event):
+        host, source, source_type = "", "", ""
+        for transport in event.iter('transport'):
+            if transport.get('host'):
+                host = transport.get('host')
+            if transport.get('source'):
+                source = transport.get('source')
+            if transport.get('sourcetype'):
+                source_type = transport.get('sourcetype')
+        return host, source, source_type
+
     def escape_char_event(self, event):
         """
         Input: Event getting parsed
         Function to escape special characters in Splunk
         https://docs.splunk.com/Documentation/StyleGuide/current/StyleGuide/Specialcharacters
         """
         escape_splunk_chars = [
-            "\\",
             "`",
             "~",
             "!",
@@ -249,6 +271,8 @@ def escape_char_event(self, event):
             "WHERE",
             "LIKE",
         ]
+        event = event.replace("\\", "\\\\")
         for character in escape_splunk_chars:
-            event = event.replace(character, "\\" + character)
+            event = event.replace(character, '\\' + character)
+        event = event.replace("*", " ")
         return event

pytest_splunk_addon/standard_lib/requirement_tests/test_templates.py

@@ -1,5 +1,6 @@
 import logging
 import pytest
+import re
 from .requirement_test_datamodel_tag_constants import dict_datamodel_tag
 
 INTERVAL = 3
@@ -107,20 +108,50 @@ def datamodel_check_test(self, keyValue_dict_SPL, requrement_file_model_list):
         ) = self.compare_datamodel(requrement_file_model_list, datamodel_based_on_tag)
         return list_extra_datamodel_requirement_file, lis_extra_extracted_splunkside
 
+
+    def remove_empty_keys(self, event):
+        event = re.sub(
+            r"(\s[a-zA-Z0-9_]*(\\=|\\:)(\\\"\\\"|\\'\\'|\\-))",
+            '',
+            str(event)
+        )
+        return event
+
+
     @pytest.mark.splunk_searchtime_requirements
     def test_requirement_params(
         self, splunk_searchtime_requirement_param, splunk_search_util
     ):
         model_datalist = splunk_searchtime_requirement_param["model_list"]
         escaped_event = splunk_searchtime_requirement_param["escaped_event"]
         key_values_xml = splunk_searchtime_requirement_param["Key_value_dict"]
+        modinput_params = splunk_searchtime_requirement_param["modinput_params"]
+        transport_type = splunk_searchtime_requirement_param["transport_type"]
         # search = f" search source= pytest_splunk_addon:hec:raw sourcetype={sourcetype} {escaped_event} |fields * "
         # removed source and sourcetype as sc4s assigns it based on event
-        search = f"search index=* {escaped_event} |fields * "
+        if transport_type in (
+        "modinput", "Modinput", "Mod input", "Modular Input", "Modular input", "modular input", "modular_input",
+        "Mod Input"):
+            host = modinput_params["host"]
+            source = modinput_params["source"]
+            sourcetype = modinput_params["sourcetype"]
+            search = f"search index=* host={host} source={source} sourcetype={sourcetype} {escaped_event}|fields * "
+        else:
+            search = f"search index=* {escaped_event} |fields * "
         ingestion_check = splunk_search_util.checkQueryCountIsGreaterThanZero(
             search, interval=INTERVAL, retries=RETRIES
         )
-        assert ingestion_check, f"ingestion failure \nsearch={search}\n"
+
+        if not ingestion_check and transport_type.lower() == "syslog":
+            empty_field_removed = self.remove_empty_keys(escaped_event)
+            search = f"search index=* {empty_field_removed} |fields * "
+            ingestion_check = splunk_search_util.checkQueryCountIsGreaterThanZero(
+                search, interval=INTERVAL, retries=RETRIES
+            )
+
+        assert ingestion_check, (
+            f"ingestion failure \nsearch={search}\n"
+        )
         self.logger.info(f"ingestion_check: {ingestion_check}")
         keyValue_dict_SPL = splunk_search_util.getFieldValuesDict(
             search, interval=INTERVAL, retries=RETRIES
@@ -141,9 +172,9 @@ def test_requirement_params(
         sourcetype = keyValue_dict_SPL["_sourcetype"]
         assert datamodel_check, (
             f"data model check: {datamodel_check} \n"
-            f"data model in requirement file not in Splunk ingested event/ missing dataset {list_unmatched_datamodel_splunkside}\n "
-            f"data model extracted on Splunk side not in requirement file {list_unmatched_datamodel_requirement_file}\n"
-            f"source type of ingested event: {sourcetype} \n"
+            f"data model in requirement file  {list_unmatched_datamodel_splunkside}\n "
+            f"data model extracted by TA {list_unmatched_datamodel_requirement_file}\n"
+            f"sourcetype of ingested event: {sourcetype} \n"
         )
         field_extraction_check, missing_key_value = self.compare(
             keyValue_dict_SPL, key_values_xml
@@ -158,7 +189,7 @@ def test_requirement_params(
         assert field_extraction_check, (
             f"Issue with the field extraction.\nsearch={search}\n"
             f" Field_extraction_check: {field_extraction_check} \n"
-            f" Key value not extracted in splunk event: {missing_key_value} \n"
+            f" Key value not extracted by TA: {missing_key_value} \n"
             f" Mismatched key value: {mismapped_key_value_pair}\n"
-            f" source type of ingested event: {sourcetype} \n"
+            f" sourcetype of ingested event: {sourcetype} \n"
         )

tests/addons/TA_requirement_test_modinput/README.txt

@@ -0,0 +1,4 @@
+Splunk Add-on for Microsoft Sysmon version 11.0.0
+Copyright (C) 2021 Splunk Inc. All Rights Reserved.
+
+For documentation, see: https://docs.splunk.com/Documentation/AddOns/latest/MSSysmon

tests/addons/TA_requirement_test_modinput/app.manifest

@@ -0,0 +1,58 @@
+{
+  "schemaVersion": "2.0.0",
+  "info": {
+    "title": "Splunk Add-on for Microsoft Sysmon",
+    "id": {
+      "group": null,
+      "name": "Splunk_TA_microsoft_sysmon",
+      "version": "11.0.0"
+    },
+    "author": [
+      {
+        "name": "Splunk, Inc.",
+        "email": "[email protected]",
+        "company": "Splunk, Inc."
+      }
+    ],
+    "releaseDate": null,
+    "description": "Splunk Add-on for Microsoft Sysmon",
+    "classification": {
+      "intendedAudience": "IT Professionals",
+      "categories": [
+        "Security, Fraud & Compliance"
+      ],
+      "developmentStatus": "Production/Stable"
+    },
+    "commonInformationModels": null,
+    "license": {
+      "name": "Splunk General Terms",
+      "text": "LICENSES/LicenseRef-Splunk-8-2021.txt",
+      "uri": "https://www.splunk.com/en_us/legal/splunk-general-terms.html"
+    },
+    "privacyPolicy": {
+      "name": null,
+      "text": null,
+      "uri": null
+    },
+    "releaseNotes": {
+      "name": "README",
+      "text": "README.txt",
+      "uri": "https://docs.splunk.com/Documentation/AddOns/MSSysmon/About"
+    }
+  },
+  "dependencies": null,
+  "tasks": null,
+  "inputGroups": null,
+  "incompatibleApps": null,
+  "platformRequirements": null,
+  "supportedDeployments": [
+    "_standalone",
+    "_distributed",
+    "_search_head_clustering"
+  ],
+  "targetWorkloads": [
+    "_search_heads",
+    "_forwarders",
+    "_indexers"
+  ]
+}