Improve integration test options (#2385)

* Improve integration test options

- Keep the platform and enable options in os.environ, don't duplicate state in utils
- Provide more user-friendly options for setting enable and platform in integration tests

* Ensure that CIBW_ENABLE is set in the test process

* Use get_enable_groups() where necessary

Author: joerick

.github/workflows/test.yml

@@ -83,7 +83,7 @@ jobs:
           CIBW_ENABLE=all
         else
           # get the default CIBW_ENABLE value from the test module
-          CIBW_ENABLE=$(uv run --no-sync python -c 'import sys, test.conftest as c; sys.stdout.write(c.DEFAULT_CIBW_ENABLE)')
+          CIBW_ENABLE=$(uv run --no-sync python -c 'import sys, test.utils as c; sys.stdout.write(c.DEFAULT_CIBW_ENABLE)')
 
           # if this is a PR, check for labels
           if [[ -n "$GITHUB_PR_LABEL_CI_PYPY" ]]; then

docs/contributing.md

@@ -78,11 +78,11 @@ nox -s tests -- test -k before_build
 
 A few notes-
 
-- Because they run inside a container, Linux tests can run on all platforms where Docker is installed, so they're convenient for running integration tests locally. Set CIBW_PLATFORM to do this: `CIBW_PLATFORM=linux nox -s tests -- test`.
+- Because they run inside a container, Linux tests can run on all platforms where Docker is installed, so they're convenient for running integration tests locally. Set the `--platform` flag on pytest to do this: `nox -s tests -- test --platform linux`.
 
 - Running the macOS integration tests requires _system installs_ of Python from python.org for all the versions that are tested. We won't attempt to install these when running locally, but you can do so manually using the URL in the error message that is printed when the install is not found.
 
-- The 'enable groups' run by default are just 'cpython-prerelease' and 'cpython-freethreading'. You can add other groups like pypy or graalpy by setting the [CIBW_ENABLE](options.md#enable) environment variable. On GitHub PRs, you can add a label to the PR to enable these groups.
+- The ['enable groups'](options.md#enable) run by default are just 'cpython-prerelease' and 'cpython-freethreading'. You can add other groups like pypy or graalpy by passing the `--enable` argument to pytest, i.e. `nox -s tests -- test --enable pypy`. On GitHub PRs, you can add a label to the PR to enable these groups.
 
 #### Running pytest directly
 

test/conftest.py

@@ -9,12 +9,11 @@
 from cibuildwheel.architecture import Architecture
 from cibuildwheel.ci import detect_ci_provider
 from cibuildwheel.options import CommandLineArguments, Options
+from cibuildwheel.selector import EnableGroup
+from cibuildwheel.typing import PLATFORMS
 from cibuildwheel.venv import find_uv
 
-from .utils import EMULATED_ARCHS, platform
-
-# default to just cpython
-DEFAULT_CIBW_ENABLE = "cpython-freethreading cpython-prerelease cpython-experimental-riscv64"
+from .utils import DEFAULT_CIBW_ENABLE, EMULATED_ARCHS, get_platform
 
 
 def pytest_addoption(parser: pytest.Parser) -> None:
@@ -32,8 +31,47 @@ def pytest_addoption(parser: pytest.Parser) -> None:
         default=False,
         help="macOS cp38 uses the universal2 installer",
     )
+    parser.addoption(
+        "--enable",
+        action="store",
+        default=None,
+        help="Set the CIBW_ENABLE environment variable for all tests.",
+    )
+    parser.addoption(
+        "--platform",
+        action="store",
+        default=None,
+        help="Set the CIBW_PLATFORM environment variable for all tests.",
+    )
+
+
+def pytest_configure(config):
+    flag_enable = config.getoption("--enable")
+    flag_platform = config.getoption("--platform")
+
+    if flag_enable is not None and "CIBW_ENABLE" in os.environ:
+        msg = (
+            "Both --enable pytest option and CIBW_ENABLE environment variable are set. "
+            "Please specify only one."
+        )
+        raise pytest.UsageError(msg)
+    if flag_platform is not None and "CIBW_PLATFORM" in os.environ:
+        msg = (
+            "Both --platform pytest option and CIBW_PLATFORM environment variable are set. "
+            "Please specify only one."
+        )
+        raise pytest.UsageError(msg)
+
+    if flag_enable is not None:
+        EnableGroup.parse_option_value(flag_enable)
+        os.environ["CIBW_ENABLE"] = flag_enable
+    if flag_enable is None and "CIBW_ENABLE" not in os.environ:
+        # Set default value for CIBW_ENABLE
+        os.environ["CIBW_ENABLE"] = DEFAULT_CIBW_ENABLE
 
-    os.environ.setdefault("CIBW_ENABLE", DEFAULT_CIBW_ENABLE)
+    if flag_platform is not None:
+        assert flag_platform in PLATFORMS, f"Invalid platform: {flag_platform}"
+        os.environ["CIBW_PLATFORM"] = flag_platform
 
 
 def docker_warmup(request: pytest.FixtureRequest) -> None:
@@ -91,7 +129,7 @@ def docker_warmup_fixture(
     request: pytest.FixtureRequest, tmp_path_factory: pytest.TempPathFactory, worker_id: str
 ) -> None:
     # if we're in CI testing linux, let's warm-up docker images
-    if detect_ci_provider() is None or platform != "linux":
+    if detect_ci_provider() is None or get_platform() != "linux":
         return None
     if request.config.getoption("--run-emulation", default=None) is not None:
         # emulation tests only run one test in CI, caching the image only slows down the test
@@ -116,7 +154,7 @@ def docker_warmup_fixture(
 @pytest.fixture(params=["pip", "build"])
 def build_frontend_env_nouv(request: pytest.FixtureRequest) -> dict[str, str]:
     frontend = request.param
-    if platform == "pyodide" and frontend == "pip":
+    if get_platform() == "pyodide" and frontend == "pip":
         pytest.skip("Can't use pip as build frontend for pyodide platform")
 
     return {"CIBW_BUILD_FRONTEND": frontend}
@@ -125,7 +163,7 @@ def build_frontend_env_nouv(request: pytest.FixtureRequest) -> dict[str, str]:
 @pytest.fixture
 def build_frontend_env(build_frontend_env_nouv: dict[str, str]) -> dict[str, str]:
     frontend = build_frontend_env_nouv["CIBW_BUILD_FRONTEND"]
-    if frontend != "build" or platform == "pyodide" or find_uv() is None:
+    if frontend != "build" or get_platform() == "pyodide" or find_uv() is None:
         return build_frontend_env_nouv
 
     return {"CIBW_BUILD_FRONTEND": "build[uv]"}
@@ -134,7 +172,7 @@ def build_frontend_env(build_frontend_env_nouv: dict[str, str]) -> dict[str, str
 @pytest.fixture
 def docker_cleanup() -> Generator[None, None, None]:
     def get_images() -> set[str]:
-        if detect_ci_provider() is None or platform != "linux":
+        if detect_ci_provider() is None or get_platform() != "linux":
             return set()
         images = subprocess.run(
             ["docker", "image", "ls", "--format", "{{json .ID}}"],

test/test_0_basic.py

@@ -1,4 +1,3 @@
-import os
 import textwrap
 
 import pytest
@@ -40,7 +39,7 @@ def test(tmp_path, build_frontend_env, capfd):
     expected_wheels = utils.expected_wheels("spam", "0.1.0")
     assert set(actual_wheels) == set(expected_wheels)
 
-    enable_groups = EnableGroup.parse_option_value(os.environ.get("CIBW_ENABLE", ""))
+    enable_groups = utils.get_enable_groups()
     if EnableGroup.GraalPy not in enable_groups:
         # Verify pip warning not shown
         captured = capfd.readouterr()

test/test_abi_variants.py

@@ -47,7 +47,7 @@ def test_abi3(tmp_path):
     )
 
     # check that the expected wheels are produced
-    if utils.platform == "pyodide":
+    if utils.get_platform() == "pyodide":
         # there's only 1 possible configuration for pyodide, cp312
         expected_wheels = utils.expected_wheels("spam", "0.1.0", python_abi_tags=["cp310-abi3"])
     else:
@@ -196,7 +196,7 @@ def test_abi_none(tmp_path, capfd):
     # check that each wheel was built once, and reused
     captured = capfd.readouterr()
     assert "Building wheel..." in captured.out
-    if utils.platform == "pyodide":
+    if utils.get_platform() == "pyodide":
         # there's only 1 possible configuration for pyodide, we won't see the message expected on following builds
         assert "Found previously built wheel" not in captured.out
     else:

test/test_before_build.py

@@ -6,7 +6,7 @@
 from . import test_projects, utils
 
 # pyodide does not support building without isolation, need to check the base_prefix
-SYS_PREFIX = f"sys.{'base_' if utils.platform == 'pyodide' else ''}prefix"
+SYS_PREFIX = f"sys.{'base_' if utils.get_platform() == 'pyodide' else ''}prefix"
 
 
 project_with_before_build_asserts = test_projects.new_c_project(
@@ -45,7 +45,7 @@ def test(tmp_path):
         f'''python -c "import pathlib, sys; pathlib.Path('{{project}}/pythonprefix_bb.txt').write_text({SYS_PREFIX})"'''
     )
     frontend = "build"
-    if utils.platform != "pyodide":
+    if utils.get_platform() != "pyodide":
         before_build = f"python -m pip install setuptools && {before_build}"
         frontend = f"{frontend};args: --no-isolation"
 

test/test_before_test.py

@@ -43,7 +43,7 @@ def test(tmp_path, build_frontend_env):
         '''python -c "import pathlib, sys; pathlib.Path('{project}/pythonprefix_bt.txt').write_text(sys.prefix)"''',
     ]
 
-    if utils.platform == "pyodide":
+    if utils.get_platform() == "pyodide":
         before_test_steps.extend(
             ["pyodide build {project}/dependency", "pip install --find-links dist/ spam"]
         )

test/test_build_frontend_args.py

@@ -20,7 +20,7 @@ def test_build_frontend_args(tmp_path, capfd, frontend_name):
 
     # the build will fail because the frontend is called with '-h' - it prints the help message
     add_env = {"CIBW_BUILD_FRONTEND": f"{frontend_name}; args: -h"}
-    if utils.platform == "pyodide":
+    if utils.get_platform() == "pyodide":
         add_env["TERM"] = "dumb"  # disable color / style
         add_env["NO_COLOR"] = "1"
     with pytest.raises(subprocess.CalledProcessError):
@@ -33,7 +33,7 @@ def test_build_frontend_args(tmp_path, capfd, frontend_name):
     if frontend_name == "pip":
         assert "Usage:" in captured.out
         assert "Wheel Options:" in captured.out
-    elif utils.platform == "pyodide":
+    elif utils.get_platform() == "pyodide":
         assert "Usage: pyodide build" in captured.out
     else:
         assert "usage:" in captured.out

test/test_container_engine.py

@@ -6,7 +6,7 @@
 
 
 def test_podman(tmp_path, capfd, request):
-    if utils.platform != "linux":
+    if utils.get_platform() != "linux":
         pytest.skip("the test is only relevant to the linux build")
 
     if not request.config.getoption("--run-podman"):
@@ -36,7 +36,7 @@ def test_podman(tmp_path, capfd, request):
 
 
 def test_create_args(tmp_path, capfd):
-    if utils.platform != "linux":
+    if utils.get_platform() != "linux":
         pytest.skip("the test is only relevant to the linux build")
 
     project_dir = tmp_path / "project"

test/test_container_images.py

@@ -23,7 +23,7 @@
 
 @pytest.mark.usefixtures("docker_cleanup")
 def test(tmp_path):
-    if utils.platform != "linux":
+    if utils.get_platform() != "linux":
         pytest.skip("the test is only relevant to the linux build")
     if platform.machine() not in ["x86_64", "i686"]:
         pytest.skip(