remap_acl speedup via config reload (#12540)

This changes the remap_acl.test.py test to use config reloads instead of
spinning up new ATS insteads for every TestRun. Now that we've reduced
our reload scanning interval via #12541, these reloads in test are
pretty quick.

Before:
```
./autest.sh --sandbox /tmp/sb --clean=none -f remap_acl  171.62s user 57.13s system 52% cpu 7:12.57 total
```

After:
```
./autest.sh --sandbox /tmp/sb --clean=none -f remap_acl  14.39s user 10.78s system 19% cpu 2:09.23 total
```

So the time to run this test goes from about 7 minutes to about 2 minutes.

Author: bneradt

tests/gold_tests/remap/remap_acl.test.py

@@ -24,15 +24,36 @@
 from yaml import CLoader as Loader
 from typing import List, Tuple
 
+from ports import get_port
+
 Test.Summary = '''
 Verify remap.config acl behavior.
 '''
 
 
+def update_config_file(path1: str, content1: str, path2: str, content2: str) -> None:
+    """Update two config files.
+
+    This is used for some of the updates to the config files between test runs.
+
+    :param path1: The path to the first config file.
+    :param content1: The content to write to the first config file.
+    :param path2: The path to the second config file.
+    :param content2: The content to write to the second config file.
+    """
+    with open(path1, 'w') as f:
+        f.write(content1 + '\n')
+    with open(path2, 'w') as f:
+        f.write(content2 + '\n')
+
+
 class Test_remap_acl:
     """Configure a test to verify remap.config acl behavior."""
 
-    _ts_counter: int = 0
+    _ts: 'TestProcess' = None
+    _ts_reload_counter: int = 0
+    _ts_is_started: bool = False
+
     _server_counter: int = 0
     _client_counter: int = 0
 
@@ -50,48 +71,47 @@ def __init__(
         :param expect_responses: The in-order expected responses from the proxy.
         """
         self._replay_file = replay_file
-        self._ip_allow_content = ip_allow_content
+        self._ip_allow_lines = ip_allow_content.split("\n")
         self._deactivate_ip_allow = deactivate_ip_allow
         self._acl_behavior_policy = acl_behavior_policy
         self._acl_configuration = acl_configuration
         self._named_acls = named_acls
         self._expected_responses = expected_responses
 
+        # Usually we configure the server first and use the server port to
+        # configure ATS to remap to it. In this case, though, we want a
+        # long-lived ATS process that spans TestRuns. So we let ATS choose an
+        # arbitrary availble server port, and then tell the TestRun-specific
+        # server to use that port.
+        server_port = self._configure_traffic_server()
         tr = Test.AddTestRun(name)
-        self._configure_server(tr)
-        self._configure_traffic_server(tr, proxy_protocol)
+        self._configure_server(tr, server_port)
         self._configure_client(tr, proxy_protocol)
 
-    def _configure_server(self, tr: 'TestRun') -> None:
+    def _configure_server(self, tr: 'TestRun', server_port: int) -> None:
         """Configure the server.
-
-        :param tr: The TestRun object to associate the server process with.
         """
         name = f"server-{Test_remap_acl._server_counter}"
-        server = tr.AddVerifierServerProcess(name, self._replay_file)
+        server = tr.AddVerifierServerProcess(name, self._replay_file, http_ports=[server_port])
         Test_remap_acl._server_counter += 1
         self._server = server
 
-    def _configure_traffic_server(self, tr: 'TestRun', proxy_protocol: bool) -> None:
+    def _configure_traffic_server(self) -> int:
         """Configure Traffic Server.
 
-        :param tr: The TestRun object to associate the Traffic Server process with.
+        :return: The listening port that the server should use.
         """
 
-        name = f"ts-{Test_remap_acl._ts_counter}"
-        ts = tr.MakeATSProcess(name, enable_cache=False, enable_proxy_protocol=proxy_protocol, enable_uds=False)
-        Test_remap_acl._ts_counter += 1
+        call_reload: bool = False
+        if Test_remap_acl._ts is not None:
+            ts = Test_remap_acl._ts
+            call_reload = True
+        else:
+            ts = Test.MakeATSProcess("ts", enable_cache=False, enable_proxy_protocol=True, enable_uds=False)
+            Test_remap_acl._ts = ts
         self._ts = ts
-
-        ts.Disk.records_config.update(
-            {
-                'proxy.config.diags.debug.enabled': 1,
-                'proxy.config.diags.debug.tags': 'http|url|remap|ip_allow|proxyprotocol',
-                'proxy.config.http.push_method_enabled': 1,
-                'proxy.config.http.connect_ports': self._server.Variables.http_port,
-                'proxy.config.url_remap.acl_behavior_policy': self._acl_behavior_policy,
-                'proxy.config.acl.subjects': 'PROXY,PEER',
-            })
+        port_name = f'ServerPort-{Test_remap_acl._ts_reload_counter}'
+        server_port: int = get_port(ts, port_name)
 
         remap_config_lines = []
         if self._deactivate_ip_allow:
@@ -104,9 +124,64 @@ def _configure_traffic_server(self, tr: 'TestRun', proxy_protocol: bool) -> None
         for name, _ in self._named_acls:
             remap_config_lines.append(f'.activatefilter {name}')
 
-        remap_config_lines.append(f'map / http://127.0.0.1:{self._server.Variables.http_port} {self._acl_configuration}')
-        ts.Disk.remap_config.AddLines(remap_config_lines)
-        ts.Disk.ip_allow_yaml.AddLines(self._ip_allow_content.split("\n"))
+        remap_config_lines.append(f'map / http://127.0.0.1:{server_port} {self._acl_configuration}')
+
+        if call_reload:
+            #
+            # Update the ATS configuration.
+            #
+            tr = Test.AddTestRun("Change the ATS configuration")
+            p = tr.Processes.Default
+            p.Command = (
+                f'traffic_ctl config set proxy.config.http.connect_ports {server_port} && '
+                f'traffic_ctl config set proxy.config.url_remap.acl_behavior_policy {self._acl_behavior_policy}')
+
+            p.Env = ts.Env
+            tr.StillRunningAfter = ts
+
+            remap_cfg_path = os.path.join(ts.Variables.CONFIGDIR, 'remap.config')
+            ip_allow_path = os.path.join(ts.Variables.CONFIGDIR, 'ip_allow.yaml')
+            p.Setup.Lambda(
+                lambda: update_config_file(
+                    remap_cfg_path, '\n'.join(remap_config_lines), ip_allow_path, '\n'.join(self._ip_allow_lines)))
+
+            #
+            # Kick off the ATS config reload.
+            #
+            tr = Test.AddTestRun("Reload the ATS configuration")
+            p = tr.Processes.Default
+            p.Command = 'traffic_ctl config reload'
+            p.Env = ts.Env
+            tr.StillRunningAfter = ts
+
+            #
+            # Await the config reload to finish.
+            #
+            tr = Test.AddTestRun("Await config reload")
+            p = tr.Processes.Default
+            p.Command = 'echo awaiting config reload'
+            p.Env = ts.Env
+            Test_remap_acl._ts_reload_counter += 1
+            count = Test_remap_acl._ts_reload_counter
+            await_config_reload = tr.Processes.Process(f'config_reload_succeeded_{count}', 'sleep 30')
+            await_config_reload.Ready = When.FileContains(ts.Disk.diags_log.Name, "remap.config finished loading", count)
+            p.StartBefore(await_config_reload)
+
+        else:
+            record_config = {
+                'proxy.config.diags.debug.enabled': 1,
+                'proxy.config.diags.debug.tags': 'http|url|remap|ip_allow|proxyprotocol',
+                'proxy.config.http.push_method_enabled': 1,
+                'proxy.config.http.connect_ports': server_port,
+                'proxy.config.url_remap.acl_behavior_policy': self._acl_behavior_policy,
+                'proxy.config.acl.subjects': 'PROXY,PEER',
+            }
+
+            ts.Disk.records_config.update(record_config)
+            ts.Disk.remap_config.AddLines(remap_config_lines)
+            ts.Disk.ip_allow_yaml.AddLines(self._ip_allow_lines)
+
+        return server_port
 
     def _configure_client(self, tr: 'TestRun', proxy_protocol: bool) -> None:
         """Run the test.
@@ -115,11 +190,14 @@ def _configure_client(self, tr: 'TestRun', proxy_protocol: bool) -> None:
         """
 
         name = f"client-{Test_remap_acl._client_counter}"
-        port = self._ts.Variables.port if proxy_protocol == False else self._ts.Variables.proxy_protocol_port
+        ts = Test_remap_acl._ts
+        port = ts.Variables.port if proxy_protocol == False else ts.Variables.proxy_protocol_port
         p = tr.AddVerifierClientProcess(name, self._replay_file, http_ports=[port])
         Test_remap_acl._client_counter += 1
         p.StartBefore(self._server)
-        p.StartBefore(self._ts)
+        if not Test_remap_acl._ts_is_started:
+            p.StartBefore(ts)
+            Test_remap_acl._ts_is_started = True
 
         if self._expected_responses == [None, None]:
             # If there are no expected responses, expect the Warning about the rejected ip.