Merge pull request #263 from splunk/Update-file-creation-uf-file-monitor

Updated file creation mechanism in file monitor

Author: nariyanibhargav

.circleci/config.yml

@@ -71,7 +71,7 @@ orbs:
 jobs:
   release:
     docker:
-      - image: circleci/node:11
+      - image: circleci/node:12
     steps:
       - checkout
       - run: npx semantic-release

docs/how_to_use.rst

@@ -42,7 +42,7 @@ There are three ways to execute the tests:
         pip install poetry
         poetry install
 
-    Create a Dockerfile-splunk file
+    Create a Dockerfile.splunk file
 
     .. dropdown:: Example Dockerfile
 
@@ -57,6 +57,19 @@ There are three ways to execute the tests:
             COPY deps/apps /opt/splunk/etc/apps/
             COPY $SPLUNK_APP_PACKAGE /opt/splunk/etc/apps/$SPLUNK_APP_ID
 
+    Create a Dockerfile.uf file
+
+    .. dropdown:: Example Dockerfile
+
+        .. code:: Dockerfile
+
+            ARG SPLUNK_VERSION=latest
+            FROM splunk/universalforwarder:$SPLUNK_VERSION
+            ARG SPLUNK_VERSION=latest
+            ARG SPLUNK_APP_ID=TA_UNKNOWN
+            ARG SPLUNK_APP_PACKAGE=$SPLUNK_APP_PACKAGE
+            COPY $SPLUNK_APP_PACKAGE /opt/splunkforwarder/etc/apps/$SPLUNK_APP_ID
+
     Create docker-compose.yml
 
     .. dropdown:: Example docker-compose file
@@ -287,4 +300,4 @@ Extending pytest-splunk-addon
 
    <hr width=100%>
 
-.. [#] xfail indicates that you expect a test to fail for some reason. A common example is a test for a feature not yet implemented, or a bug not yet fixed. When a test passes despite being expected to fail, it's an xpass and will be reported in the test summary.
+.. [#] xfail indicates that you expect a test to fail for some reason. A common example is a test for a feature not yet implemented, or a bug not yet fixed. When a test passes despite being expected to fail, it's an xpass and will be reported in the test summary.

pytest_splunk_addon/splunk.py

@@ -426,16 +426,10 @@ def uf(request):
         uf = request.getfixturevalue("uf_docker")
     else:
         raise Exception
-    uf["uf_username"] = request.config.getoption("splunk_uf_user")
-    uf["uf_password"] = request.config.getoption("splunk_uf_password")
-    for _ in range(RESPONSIVE_SPLUNK_TIMEOUT):
-        if is_responsive_uf(uf):
-            break
-        sleep(1)
     yield uf
 
 @pytest.fixture(scope="session")
-def uf_docker(docker_services, tmp_path_factory, worker_id):
+def uf_docker(docker_services, tmp_path_factory, worker_id, request):
     """
     Provides IP of the uf server and management port based on pytest-args(splunk_type)
     """
@@ -450,7 +444,13 @@ def uf_docker(docker_services, tmp_path_factory, worker_id):
     uf_info = {
         "uf_host": docker_services.docker_ip,
         "uf_port": docker_services.port_for("uf", 8089),
+        "uf_username": request.config.getoption("splunk_uf_user"),
+        "uf_password": request.config.getoption("splunk_uf_password"),
     }
+    for _ in range(RESPONSIVE_SPLUNK_TIMEOUT):
+        if is_responsive_uf(uf_info):
+            break
+        sleep(1)
     return uf_info
 
 @pytest.fixture(scope="session")
@@ -461,6 +461,8 @@ def uf_external(request):
     uf_info = {
         "uf_host": request.config.getoption("splunk_uf_host"),
         "uf_port": request.config.getoption("splunk_uf_port"),
+        "uf_username": request.config.getoption("splunk_uf_user"),
+        "uf_password": request.config.getoption("splunk_uf_password"),
     }
     return uf_info
 
@@ -704,9 +706,13 @@ def file_system_prerequisite():
     """
     UF_FILE_MONTOR_DIR = "uf_files"
     monitor_dir = os.path.join(os.getcwd(), UF_FILE_MONTOR_DIR)
-    if os.path.exists(monitor_dir):
-        shutil.rmtree(UF_FILE_MONTOR_DIR, ignore_errors=True)
-    os.mkdir(monitor_dir)
+    if (
+        "PYTEST_XDIST_WORKER" not in os.environ
+        or os.environ.get("PYTEST_XDIST_WORKER") == "gw0"
+    ):
+        if os.path.exists(monitor_dir):
+            shutil.rmtree(monitor_dir, ignore_errors=True)
+        os.mkdir(monitor_dir)
 
 def is_responsive_uf(uf):
     """

pytest_splunk_addon/standard_lib/event_ingestors/file_monitor_ingestor.py

@@ -89,6 +89,9 @@ def create_inputs_stanza(self, event):
             event (SampleEvent): Instance containing event info
         """
         file_path = self.get_file_path(event)
+        host_segment = len(file_path.split(os.sep)) - 2 
+        # If file path is /home/uf_files/host_name/sample_name
+        # Then split will return ['', home, 'uf_files', 'host_name', 'sample_name']
         sourcetype = event.metadata.get("sourcetype")
         if not sourcetype:
             sourcetype = event.metadata.get("sourcetype_to_search", "pytest_splunk_addon")
@@ -100,15 +103,13 @@ def create_inputs_stanza(self, event):
             "crc-salt": "<SOURCE>"
         }
         if event.metadata.get("host_type") in ("plugin"):
-            stanza["host"] = event.metadata.get("host")
-        if event.metadata.get("source"):
-            stanza["rename-source"] = event.metadata.get("source")
+            stanza["host_segment"] = host_segment
         LOGGER.info("Making rest call to create stanza in inputs.conf file with following endpoint : {}".format(self.inputs_endpoint))
         LOGGER.debug("Creating following stanza in inputs.conf : {}".format(stanza))
         try:
             response = requests.post(self.inputs_endpoint, stanza, auth=(self.uf_username, self.uf_password), verify=False)
             if response.status_code not in (200, 201):
-                LOGGER.warning("Unable to add stanza in inputs.conf\nStatus code: {} \nReason: {} \ntext:{}".format(response.status_code, response.reason, response.text))
+                LOGGER.warning("Unable to add stanza in inputs.conf for Path : {} \nStatus code: {} \nReason: {} \ntext:{}".format(file_path, response.status_code, response.reason, response.text))
         except ConnectionError as e:
             LOGGER.error("Unable to connect to Universal forwarder, {}".format(e))
 
@@ -119,4 +120,7 @@ def get_file_path(self, event):
         Args:
             event (SampleEvent): Instance containing event info
         """
-        return "{}/{}/{}".format(os.getcwd(), MONITOR_DIR, event.metadata.get("host"))
+        host_dir = os.path.join(os.getcwd(), MONITOR_DIR, event.metadata.get("host"))
+        if not os.path.exists(host_dir):
+            os.mkdir(host_dir)
+        return os.path.join(host_dir, event.metadata.get("source"))

pytest_splunk_addon/standard_lib/sample_generation/sample_stanza.py

@@ -160,6 +160,8 @@ def _parse_meta(self, eventgen_params):
             metadata.update(count="100")
         if metadata.get("index") is not None and metadata.get("input_type") in ["syslog_tcp", "tcp", "udp"]:
             raise_warning("For input_type '{}', there should be no index set".format(metadata.get("input_type")))
+        if metadata.get("input_type") == "uf_file_monitor":
+            metadata["host"] = metadata.get("host").replace("_", "-").replace(".", "-")
         return metadata
 
     def get_eventmetadata(self):

tests/constants.py

@@ -616,8 +616,8 @@
     '*test_splunk_fiction_indextime.py::Test_App::test_indextime_line_breaker*test:indextime:uf_file_monitor_host_event::uf_file_monitor_host_event.sample* PASSED*',
     '*test_splunk_fiction_indextime.py::Test_App::test_indextime_line_breaker*test:indextime:uf_file_monitor_host_plugin::uf_file_monitor_host_plugin.sample* PASSED*',
     '*test_splunk_fiction_indextime.py::Test_App::test_indextime_line_breaker*test:indextime:uf_file_monitor_rename_sourcetype:source::uf_file_monitor_rename_sourcetype.sample* PASSED*',
-    '*test_splunk_fiction_indextime.py::Test_App::test_indextime_key_fields*test:indextime:uf_file_monitor_indexed_extraction::uf_file_monitor_indexed_extraction.sample* PASSED*',
-    '*test_splunk_fiction_indextime.py::Test_App::test_indextime_time*test:indextime:uf_file_monitor_indexed_extraction::uf_file_monitor_indexed_extraction.sample* PASSED*',
+    '*test_splunk_fiction_indextime.py::Test_App::test_indextime_key_fields*test:indextime:uf_file_monitor_indexed_extraction::uf-file-monitor-indexed-extraction-sample* PASSED*',
+    '*test_splunk_fiction_indextime.py::Test_App::test_indextime_time*test:indextime:uf_file_monitor_indexed_extraction::uf-file-monitor-indexed-extraction-sample* PASSED*',
     '*test_splunk_fiction_indextime.py::Test_App::test_indextime_line_breaker*test:indextime:uf_file_monitor_indexed_extraction::uf_file_monitor_indexed_extraction.sample* PASSED*',
 ]