fix: change to the iOS testing option semantics (#2363)

* Change to the iOS testing option semantics

Don't assume the presence of `python -m` in the test command. Less magic
and allows more option reuse between platforms.

* Update schema

* Add a test for this warning

* Don't try to execute a test-command when it doesn't look like a module

* Update docs/options.md

Co-authored-by: Malcolm Smith <[email protected]>

* Only allow invalid test command if the first part is 'pytest'

* Responses to code review from @freakboy3742

* Fixes post-merge

---------

Co-authored-by: Malcolm Smith <[email protected]>

Author: joerick

cibuildwheel/platforms/ios.py

@@ -32,7 +32,7 @@
     download,
     move_file,
 )
-from ..util.helpers import prepare_command
+from ..util.helpers import prepare_command, unwrap_preserving_paragraphs
 from ..util.packaging import (
     combine_constraints,
     find_compatible_wheel,
@@ -593,14 +593,51 @@ def build(options: Options, tmp_path: Path) -> None:
                     )
 
                     log.step("Running test suite...")
+
+                    test_command_parts = shlex.split(build_options.test_command)
+                    if test_command_parts[0:2] != ["python", "-m"]:
+                        first_part = test_command_parts[0]
+                        if first_part == "pytest":
+                            # pytest works exactly the same as a module, so we
+                            # can just run it as a module.
+                            log.warning(
+                                unwrap_preserving_paragraphs(f"""
+                                    iOS tests configured with a test command which doesn't start
+                                    with 'python -m'. iOS tests must execute python modules - other
+                                    entrypoints are not supported.
+
+                                    cibuildwheel will try to execute it as if it started with
+                                    'python -m'. If this works, all you need to do is add that to
+                                    your test command.
+
+                                    Test command: {build_options.test_command!r}
+                                """)
+                            )
+                        else:
+                            msg = unwrap_preserving_paragraphs(
+                                f"""
+                                    iOS tests configured with a test command which doesn't start
+                                    with 'python -m'. iOS tests must execute python modules - other
+                                    entrypoints are not supported.
+
+                                    Test command: {build_options.test_command!r}
+                                """
+                            )
+                            raise errors.FatalError(msg)
+                    else:
+                        # the testbed run command actually doesn't want the
+                        # python -m prefix - it's implicit, so we remove it
+                        # here.
+                        test_command_parts = test_command_parts[2:]
+
                     try:
                         call(
                             "python",
                             testbed_path,
                             "run",
                             *(["--verbose"] if build_options.build_verbosity > 0 else []),
                             "--",
-                            *(shlex.split(build_options.test_command)),
+                            *test_command_parts,
                             env=build_env,
                         )
                         failed = False

cibuildwheel/util/helpers.py

@@ -76,6 +76,21 @@ def unwrap(text: str) -> str:
     return re.sub(r"\s+", " ", text)
 
 
+def unwrap_preserving_paragraphs(text: str) -> str:
+    """
+    Unwraps multi-line text to a single line, but preserves paragraphs
+    """
+    # remove initial line indent
+    text = textwrap.dedent(text)
+    # remove leading/trailing whitespace
+    text = text.strip()
+
+    paragraphs = text.split("\n\n")
+    # remove consecutive whitespace
+    paragraphs = [re.sub(r"\s+", " ", paragraph) for paragraph in paragraphs]
+    return "\n\n".join(paragraphs)
+
+
 def parse_key_value_string(
     key_value_string: str,
     positional_arg_names: Sequence[str] | None = None,

docs/options.md

@@ -1253,7 +1253,7 @@ run your test suite.
 
 On all platforms other than iOS, the command is run in a shell, so you can write things like `cmd1 && cmd2`.
 
-On iOS, the value of the `CIBW_TEST_COMMAND` setting is interpreted as the arguments to pass to `python -m` - that is, a Python module name, followed by arguments that will be assigned to `sys.argv`. Shell commands cannot be used.
+On iOS, the value of the `CIBW_TEST_COMMAND` setting must follow the format `python -m MODULE [ARGS...]` - where MODULE is a Python module name, followed by arguments that will be assigned to `sys.argv`. Other commands cannot be used.
 
 Platform-specific environment variables are also available:<br/>
 `CIBW_TEST_COMMAND_MACOS` | `CIBW_TEST_COMMAND_WINDOWS` | `CIBW_TEST_COMMAND_LINUX` | `CIBW_TEST_COMMAND_IOS` | `CIBW_TEST_COMMAND_PYODIDE`
@@ -1273,6 +1273,10 @@ Platform-specific environment variables are also available:<br/>
     CIBW_TEST_COMMAND: >
       pytest ./tests &&
       python ./test.py
+
+    # run tests on ios
+    CIBW_TEST_SOURCES_IOS: tests
+    CIBW_TEST_COMMAND_IOS: python -m pytest ./tests
     ```
 
 !!! tab examples "pyproject.toml"
@@ -1290,6 +1294,11 @@ Platform-specific environment variables are also available:<br/>
       "pytest ./tests",
       "python ./test.py",
     ]
+
+    # run tests on ios
+    [tool.cibuildwheel.ios]
+    test-sources = ["tests"]
+    test-command = "python -m pytest ./tests"
     ```
 
     In configuration files, you can use an array, and the items will be joined with `&&`.

test/test_ios.py

@@ -4,6 +4,7 @@
 import platform
 import shutil
 import subprocess
+import textwrap
 
 import pytest
 
@@ -73,28 +74,17 @@ def test_ios_platforms(tmp_path, build_config, monkeypatch, capfd):
             "CIBW_BUILD": "cp313-*",
             "CIBW_XBUILD_TOOLS": "does-exist",
             "CIBW_TEST_SOURCES": "tests",
-            "CIBW_TEST_COMMAND": "unittest discover tests test_platform.py",
+            "CIBW_TEST_COMMAND": "python -m unittest discover tests test_platform.py",
             "CIBW_BUILD_VERBOSITY": "1",
             **build_config,
         },
     )
 
     # The expected wheels were produced.
-    ios_version = os.getenv("IPHONEOS_DEPLOYMENT_TARGET", "13.0").replace(".", "_")
-    platform_machine = platform.machine()
-
-    if platform_machine == "x86_64":
-        expected_wheels = {
-            f"spam-0.1.0-cp313-cp313-ios_{ios_version}_x86_64_iphonesimulator.whl",
-        }
-
-    elif platform_machine == "arm64":
-        expected_wheels = {
-            f"spam-0.1.0-cp313-cp313-ios_{ios_version}_arm64_iphoneos.whl",
-            f"spam-0.1.0-cp313-cp313-ios_{ios_version}_arm64_iphonesimulator.whl",
-        }
-
-    assert set(actual_wheels) == expected_wheels
+    expected_wheels = utils.expected_wheels(
+        "spam", "0.1.0", platform="ios", python_abi_tags=["cp313-cp313"]
+    )
+    assert set(actual_wheels) == set(expected_wheels)
 
     # The user was notified that the cross-build tool was found.
     captured = capfd.readouterr()
@@ -119,7 +109,7 @@ def test_no_test_sources(tmp_path, capfd):
             add_env={
                 "CIBW_PLATFORM": "ios",
                 "CIBW_BUILD": "cp313-*",
-                "CIBW_TEST_COMMAND": "tests",
+                "CIBW_TEST_COMMAND": "python -m tests",
             },
         )
 
@@ -146,7 +136,7 @@ def test_missing_xbuild_tool(tmp_path, capfd):
             add_env={
                 "CIBW_PLATFORM": "ios",
                 "CIBW_BUILD": "cp313-*",
-                "CIBW_TEST_COMMAND": "tests",
+                "CIBW_TEST_COMMAND": "python -m tests",
                 "CIBW_XBUILD_TOOLS": "does-not-exist",
             },
         )
@@ -180,21 +170,13 @@ def test_no_xbuild_tool_definition(tmp_path, capfd):
     )
 
     # The expected wheels were produced.
-    ios_version = os.getenv("IPHONEOS_DEPLOYMENT_TARGET", "13.0").replace(".", "_")
-    platform_machine = platform.machine()
-
-    if platform_machine == "x86_64":
-        expected_wheels = {
-            f"spam-0.1.0-cp313-cp313-ios_{ios_version}_x86_64_iphonesimulator.whl",
-        }
-
-    elif platform_machine == "arm64":
-        expected_wheels = {
-            f"spam-0.1.0-cp313-cp313-ios_{ios_version}_arm64_iphoneos.whl",
-            f"spam-0.1.0-cp313-cp313-ios_{ios_version}_arm64_iphonesimulator.whl",
-        }
-
-    assert set(actual_wheels) == expected_wheels
+    expected_wheels = utils.expected_wheels(
+        "spam",
+        "0.1.0",
+        platform="ios",
+        python_abi_tags=["cp313-cp313"],
+    )
+    assert set(actual_wheels) == set(expected_wheels)
 
     # The user was notified that there was no cross-build tool definition.
     captured = capfd.readouterr()
@@ -225,23 +207,79 @@ def test_empty_xbuild_tool_definition(tmp_path, capfd):
         },
     )
 
-    # The expected wheels were produced.
-    ios_version = os.getenv("IPHONEOS_DEPLOYMENT_TARGET", "13.0").replace(".", "_")
-    platform_machine = platform.machine()
-
-    if platform_machine == "x86_64":
-        expected_wheels = {
-            f"spam-0.1.0-cp313-cp313-ios_{ios_version}_x86_64_iphonesimulator.whl",
-        }
-
-    elif platform_machine == "arm64":
-        expected_wheels = {
-            f"spam-0.1.0-cp313-cp313-ios_{ios_version}_arm64_iphoneos.whl",
-            f"spam-0.1.0-cp313-cp313-ios_{ios_version}_arm64_iphonesimulator.whl",
-        }
-
-    assert set(actual_wheels) == expected_wheels
+    expected_wheels = utils.expected_wheels(
+        "spam", "0.1.0", platform="ios", python_abi_tags=["cp313-cp313"]
+    )
+    assert set(actual_wheels) == set(expected_wheels)
 
     # The warnings about cross-build notifications were silenced.
     captured = capfd.readouterr()
     assert "Your project configuration does not define any cross-build tools." not in captured.err
+
+
+@pytest.mark.serial
+def test_ios_test_command_without_python_dash_m(tmp_path, capfd):
+    """pytest should be able to run without python -m, but it should warn."""
+    if utils.get_platform() != "macos":
+        pytest.skip("this test can only run on macOS")
+    if utils.get_xcode_version() < (13, 0):
+        pytest.skip("this test only works with Xcode 13.0 or greater")
+
+    project_dir = tmp_path / "project"
+
+    project = test_projects.new_c_project()
+    project.files["tests/__init__.py"] = ""
+    project.files["tests/test_spam.py"] = textwrap.dedent("""
+        import spam
+        def test_spam():
+            assert spam.filter("spam") == 0
+            assert spam.filter("ham") != 0
+    """)
+    project.generate(project_dir)
+
+    actual_wheels = utils.cibuildwheel_run(
+        project_dir,
+        add_env={
+            "CIBW_PLATFORM": "ios",
+            "CIBW_BUILD": "cp313-*",
+            "CIBW_TEST_COMMAND": "pytest ./tests",
+            "CIBW_TEST_SOURCES": "tests",
+            "CIBW_TEST_REQUIRES": "pytest",
+            "CIBW_XBUILD_TOOLS": "",
+        },
+    )
+
+    expected_wheels = utils.expected_wheels(
+        "spam", "0.1.0", platform="ios", python_abi_tags=["cp313-cp313"]
+    )
+    assert set(actual_wheels) == set(expected_wheels)
+
+    out, err = capfd.readouterr()
+
+    assert "iOS tests configured with a test command which doesn't start with 'python -m'" in err
+
+
+def test_ios_test_command_invalid(tmp_path, capfd):
+    """Test command should raise an error if it's clearly invalid."""
+    if utils.get_platform() != "macos":
+        pytest.skip("this test can only run on macOS")
+    if utils.get_xcode_version() < (13, 0):
+        pytest.skip("this test only works with Xcode 13.0 or greater")
+
+    project_dir = tmp_path / "project"
+    basic_project = test_projects.new_c_project()
+    basic_project.files["./my_test_script.sh"] = "echo hello"
+    basic_project.generate(project_dir)
+
+    with pytest.raises(subprocess.CalledProcessError):
+        utils.cibuildwheel_run(
+            project_dir,
+            add_env={
+                "CIBW_PLATFORM": "ios",
+                "CIBW_TEST_COMMAND": "./my_test_script.sh",
+                "CIBW_TEST_SOURCES": "./my_test_script.sh",
+                "CIBW_XBUILD_TOOLS": "",
+            },
+        )
+    out, err = capfd.readouterr()
+    assert "iOS tests configured with a test command which doesn't start with 'python -m'" in err