ci: correct Github Actions CI instability for iOS. (#2337)

* Correct Github Actions CI instability for iOS.

* Add a dummy serial test so CircleCI passes serial test discovery.

Author: freakboy3742

.github/workflows/test.yml

@@ -36,7 +36,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, macos-13, macos-14]
+        os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, macos-13, macos-15]
         python_version: ['3.13']
         include:
         - os: ubuntu-latest

bin/run_tests.py

@@ -34,15 +34,39 @@
         if args.run_podman:
             unit_test_args += ["--run-podman"]
 
+    print(
+        "\n\n================================== UNIT TESTS ==================================",
+        flush=True,
+    )
     subprocess.run(unit_test_args, check=True)
 
-    # integration tests
+    # Run the serial integration tests without multiple processes
+    serial_integration_test_args = [
+        sys.executable,
+        "-m",
+        "pytest",
+        "-m",
+        "serial",
+        "-x",
+        "--durations",
+        "0",
+        "--timeout=2400",
+        "test",
+        "-vv",
+    ]
+    print(
+        "\n\n=========================== SERIAL INTEGRATION TESTS ===========================",
+        flush=True,
+    )
+    subprocess.run(serial_integration_test_args, check=True)
+
+    # Non-serial integration tests
     integration_test_args = [
         sys.executable,
         "-m",
         "pytest",
-        "--dist",
-        "loadgroup",
+        "-m",
+        "not serial",
         f"--numprocesses={args.num_processes}",
         "-x",
         "--durations",
@@ -55,4 +79,8 @@
     if sys.platform.startswith("linux") and args.run_podman:
         integration_test_args += ["--run-podman"]
 
+    print(
+        "\n\n========================= NON-SERIAL INTEGRATION TESTS =========================",
+        flush=True,
+    )
     subprocess.run(integration_test_args, check=True)

pyproject.toml

@@ -102,7 +102,9 @@ junit_family = "xunit2"
 xfail_strict = true
 filterwarnings = ["error"]
 log_cli_level = "info"
-
+markers = [
+    "serial: tests that must *not* be run in parallel (deselect with '-m \"not serial\"')",
+]
 
 [tool.mypy]
 python_version = "3.11"

test/test_0_basic.py

@@ -18,6 +18,15 @@
 )
 
 
+@pytest.mark.serial
+def test_dummy_serial():
+    """A no-op test to ensure that at least one serial test is always found.
+
+    Without this no-op test, CI fails on CircleCI because no serial tests are
+    found, and pytest errors if a test suite finds no tests.
+    """
+
+
 def test(tmp_path, build_frontend_env, capfd):
     project_dir = tmp_path / "project"
     basic_project.generate(project_dir)

test/test_ios.py

@@ -20,10 +20,12 @@ def test_platform(self):
 """
 
 
-# iOS tests shouldn't be run in parallel, because they're dependent on starting
-# a simulator. It's *possible* to start multiple simulators, but not advisable
-# to start as many simulators as there are CPUs on the test machine.
-@pytest.mark.xdist_group(name="ios")
+# iOS tests shouldn't be run in parallel, because they're dependent on calling
+# Xcode, and starting a simulator. These are both multi-threaded operations, and
+# it's easy to overload the CI machine if there are multiple test processes
+# running multithreaded processes. Therefore, they're put in the serial group,
+# which is guaranteed to run single-process.
+@pytest.mark.serial
 @pytest.mark.parametrize(
     "build_config",
     [
@@ -48,6 +50,7 @@ def test_ios_platforms(tmp_path, build_config):
             "CIBW_BUILD": "cp313-*",
             "CIBW_TEST_SOURCES": "tests",
             "CIBW_TEST_COMMAND": "unittest discover tests test_platform.py",
+            "CIBW_BUILD_VERBOSITY": "1",
             **build_config,
         },
     )
@@ -71,7 +74,7 @@ def test_ios_platforms(tmp_path, build_config):
     assert set(actual_wheels) == expected_wheels
 
 
-@pytest.mark.xdist_group(name="ios")
+@pytest.mark.serial
 def test_no_test_sources(tmp_path, capfd):
     if utils.platform != "macos":
         pytest.skip("this test can only run on macOS")