Merge bitcoin/bitcoin#33313: test/refactor: use test deque to avoid quadratic iteration

75e6984 test/refactor: use test deque to avoid quadratic iteration (Lőrinc)

Pull request description:

  Extracted from bitcoin/bitcoin#33141 (comment).

  -----

  In Python, [list `pop(0)` is linear](https://docs.python.org/3/tutorial/datastructures.html#using-lists-as-queues), so consuming all items in the test results in quadratic iteration.

  Switching to `collections.deque` with `popleft()` expresses FIFO intent and avoids the O(n^2) path.
  Behavior is unchanged - for a few hundred items the perf impact is likely negligible.

ACKs for top commit:
  maflcko:
    lgtm ACK 75e6984
  theStack:
    re-ACK 75e6984
  enirox001:
    reACK 75e6984
  w0xlt:
    reACK bitcoin/bitcoin@75e6984

Tree-SHA512: 290f6aeeb33d8b12b7acbbfede7ce0bef1c831a7ab9efc9c3a08c049986572e289cdece0844db908cf198395f574575ce4073c268033bf6dbaadc3828c96c1d8

Author: glozow

test/functional/test_runner.py

@@ -454,8 +454,8 @@ def main():
         print("Re-compile with the -DBUILD_DAEMON=ON build option")
         sys.exit(1)
 
-    # Build list of tests
-    test_list = []
+    # Build tests
+    test_list = deque()
     if tests:
         # Individual tests have been specified. Run specified tests that exist
         # in the ALL_SCRIPTS list. Accept names with or without a .py extension.
@@ -474,7 +474,7 @@ def main():
             script = script + ".py" if ".py" not in script else script
             matching_scripts = [s for s in ALL_SCRIPTS if s.startswith(script)]
             if matching_scripts:
-                test_list.extend(matching_scripts)
+                test_list += matching_scripts
             else:
                 print("{}WARNING!{} Test '{}' not found in full test list.".format(BOLD[1], BOLD[0], test))
     elif args.extended:
@@ -509,7 +509,7 @@ def remove_tests(exclude_list):
                 remove_tests([test for test in test_list if test.split('.py')[0] == exclude_test.split('.py')[0]])
 
     if args.filter:
-        test_list = list(filter(re.compile(args.filter).search, test_list))
+        test_list = deque(filter(re.compile(args.filter).search, test_list))
 
     if not test_list:
         print("No valid test scripts specified. Check that your test is in one "
@@ -726,7 +726,7 @@ def done(self):
     def get_next(self):
         while len(self.jobs) < self.num_jobs and self.test_list:
             # Add tests
-            test = self.test_list.pop(0)
+            test = self.test_list.popleft()
             portseed = len(self.test_list)
             portseed_arg = ["--portseed={}".format(portseed)]
             log_stdout = tempfile.SpooledTemporaryFile(max_size=2**16)
@@ -754,8 +754,7 @@ def proc_wait(task):
             ]
             fut = self.executor.submit(proc_wait, task)
             self.jobs[fut] = test
-        if not self.jobs:
-            raise IndexError('pop from empty list')
+        assert self.jobs  # Must not be empty here
 
         # Print remaining running jobs when all jobs have been started.
         if not self.test_list: