From a64a087e45e7ee18d2da822ce641e5009f13e659 Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Thu, 13 Mar 2025 13:37:09 +0800 Subject: [PATCH 01/12] Add the ability to declare safe tools in a cross-build environment. --- README.md | 1 + cibuildwheel/ios.py | 31 +++++++++++++++++-- cibuildwheel/options.py | 5 +++ .../resources/cibuildwheel.schema.json | 21 +++++++++++++ cibuildwheel/resources/defaults.toml | 1 + docs/options.md | 29 +++++++++++++++++ test/test_ios.py | 27 +++++++++++++++- unit_test/options_test.py | 3 ++ unit_test/options_toml_test.py | 5 +++ 9 files changed, 119 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 028b8e53d..0d25f2634 100644 --- a/README.md +++ b/README.md @@ -139,6 +139,7 @@ Options | | [`CIBW_ENVIRONMENT_PASS_LINUX`](https://cibuildwheel.pypa.io/en/stable/options/#environment-pass) | Set environment variables on the host to pass-through to the container during the build. | | | [`CIBW_BEFORE_ALL`](https://cibuildwheel.pypa.io/en/stable/options/#before-all) | Execute a shell command on the build system before any wheels are built. | | | [`CIBW_BEFORE_BUILD`](https://cibuildwheel.pypa.io/en/stable/options/#before-build) | Execute a shell command preparing each wheel's build | +| | [`CIBW_SAFE_TOOLS`](https://cibuildwheel.pypa.io/en/stable/options/#safe-tools) | Binaries on the path that are safe to include in an isolated cross-build environment. | | | [`CIBW_REPAIR_WHEEL_COMMAND`](https://cibuildwheel.pypa.io/en/stable/options/#repair-wheel-command) | Execute a shell command to repair each built wheel | | | [`CIBW_MANYLINUX_*_IMAGE`
`CIBW_MUSLLINUX_*_IMAGE`](https://cibuildwheel.pypa.io/en/stable/options/#linux-image) | Specify alternative manylinux / musllinux Docker images | | | [`CIBW_CONTAINER_ENGINE`](https://cibuildwheel.pypa.io/en/stable/options/#container-engine) | Specify which container engine to use when building Linux wheels | diff --git a/cibuildwheel/ios.py b/cibuildwheel/ios.py index e3b6c0a47..92029075f 100644 --- a/cibuildwheel/ios.py +++ b/cibuildwheel/ios.py @@ -147,6 +147,7 @@ def cross_virtualenv( build_python: Path, venv_path: Path, dependency_constraint_flags: Sequence[PathOrStr], + safe_tools: Sequence[str], ) -> dict[str, str]: """Create a cross-compilation virtual environment. @@ -174,6 +175,8 @@ def cross_virtualenv( created. :param dependency_constraint_flags: Any flags that should be used when constraining dependencies in the environment. + :param safe_tools: A list of executable names (without paths) that are + on the path, but must be preserved in the cross environment. """ # Create an initial macOS virtual environment env = virtualenv( @@ -206,14 +209,33 @@ def cross_virtualenv( # # To prevent problems, set the PATH to isolate the build environment from # sources that could introduce incompatible binaries. + # + # However, there may be some tools on the path that are needed for the + # build. Find their location on the path, and link the underlying binaries + # (fully resolving symlinks) to a "safe" location that will *only* contain + # those tools. This avoids needing to add *all* of Homebrew to the path just + # to get access to (for example) cmake for build purposes. + safe_tools_path = venv_path / "cibw_safe_tools" + safe_tools_path.mkdir() + for tool in safe_tools: + tool_path = shutil.which(tool) + if tool_path is None: + msg = f"Could not find a {tool!r} executable on the path." + raise errors.FatalError(msg) + + # Link the binary into the safe tools directory + original = Path(tool_path).resolve() + print(f"{tool!r} is a safe tool in the cross-build environment (using {original})") + (safe_tools_path / tool).symlink_to(original) + env["PATH"] = os.pathsep.join( [ # The target python's binary directory str(target_python.parent), - # The cross-platform environments binary directory + # The cross-platform environment's binary directory str(venv_path / "bin"), - # Cargo's binary directory (to allow for Rust compilation) - str(Path.home() / ".cargo" / "bin"), + # The directory of safe tools + str(safe_tools_path), # The bare minimum Apple system paths. "/usr/bin", "/bin", @@ -235,6 +257,7 @@ def setup_python( dependency_constraint_flags: Sequence[PathOrStr], environment: ParsedEnvironment, build_frontend: BuildFrontendName, + safe_tools: Sequence[str], ) -> tuple[Path, dict[str, str]]: if build_frontend == "build[uv]": msg = "uv doesn't support iOS" @@ -287,6 +310,7 @@ def setup_python( build_python=build_python, venv_path=venv_path, dependency_constraint_flags=dependency_constraint_flags, + safe_tools=safe_tools, ) venv_bin_path = venv_path / "bin" assert venv_bin_path.exists() @@ -414,6 +438,7 @@ def build(options: Options, tmp_path: Path) -> None: dependency_constraint_flags, build_options.environment, build_frontend.name, + safe_tools=build_options.safe_tools, ) pip_version = get_pip_version(env) diff --git a/cibuildwheel/options.py b/cibuildwheel/options.py index aff0566c2..e5520eb1a 100644 --- a/cibuildwheel/options.py +++ b/cibuildwheel/options.py @@ -94,6 +94,7 @@ class BuildOptions: environment: ParsedEnvironment before_all: str before_build: str | None + safe_tools: list[str] repair_command: str manylinux_images: dict[str, str] | None musllinux_images: dict[str, str] | None @@ -696,6 +697,9 @@ def _compute_build_options(self, identifier: str | None) -> BuildOptions: test_command = self.reader.get("test-command", option_format=ListFormat(sep=" && ")) before_test = self.reader.get("before-test", option_format=ListFormat(sep=" && ")) + safe_tools = shlex.split( + self.reader.get("safe-tools", option_format=ListFormat(sep=" ", quote=shlex.quote)) + ) test_sources = shlex.split( self.reader.get( "test-sources", option_format=ListFormat(sep=" ", quote=shlex.quote) @@ -838,6 +842,7 @@ def _compute_build_options(self, identifier: str | None) -> BuildOptions: before_build=before_build, before_all=before_all, build_verbosity=build_verbosity, + safe_tools=safe_tools, repair_command=repair_command, environment=environment, dependency_constraints=dependency_constraints, diff --git a/cibuildwheel/resources/cibuildwheel.schema.json b/cibuildwheel/resources/cibuildwheel.schema.json index 4a84606cf..19d6a98c9 100644 --- a/cibuildwheel/resources/cibuildwheel.schema.json +++ b/cibuildwheel/resources/cibuildwheel.schema.json @@ -397,6 +397,21 @@ "description": "Specify alternative manylinux / musllinux container images", "title": "CIBW_MUSLLINUX_X86_64_IMAGE" }, + "safe-tools": { + "description": "Binaries on the path that are safe to include in an isolated cross-build environment", + "oneOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ], + "title": "CIBW_SAFE_TOOLS" + }, "repair-wheel-command": { "description": "Execute a shell command to repair each built wheel.", "oneOf": [ @@ -566,6 +581,9 @@ "environment-pass": { "$ref": "#/$defs/inherit" }, + "safe-tools": { + "$ref": "#/$defs/inherit" + }, "repair-wheel-command": { "$ref": "#/$defs/inherit" }, @@ -991,6 +1009,9 @@ "repair-wheel-command": { "$ref": "#/properties/repair-wheel-command" }, + "safe-tools": { + "$ref": "#/properties/safe-tools" + }, "test-command": { "$ref": "#/properties/test-command" }, diff --git a/cibuildwheel/resources/defaults.toml b/cibuildwheel/resources/defaults.toml index 6b65e6369..c6124afe1 100644 --- a/cibuildwheel/resources/defaults.toml +++ b/cibuildwheel/resources/defaults.toml @@ -14,6 +14,7 @@ build-verbosity = 0 before-all = "" before-build = "" +safe-tools = [] repair-wheel-command = "" test-command = "" diff --git a/docs/options.md b/docs/options.md index 080c66bb8..4073c8b5a 100644 --- a/docs/options.md +++ b/docs/options.md @@ -1044,6 +1044,35 @@ Platform-specific environment variables are also available:
[PEP 517]: https://www.python.org/dev/peps/pep-0517/ [PEP 518]: https://www.python.org/dev/peps/pep-0517/ +### `CIBW_SAFE_TOOLS` {: #safe-tools} +> Binaries on the path that are safe to include in an isolated cross-build environment. + +When building in a cross-platform environment, it is sometimes necessary to isolate the ``PATH`` so that binaries from the build machine don't accidentally get linked into the cross-platform binary. However, this isolation process will also hide tools that might be required to build your wheel. + +If there are binaries present on the `PATH` when you invoke cibuildwheel, and those binaries are required to build your wheels, those binaries can be explicitly declared as "safe" using `CIBW_SAFE_TOOLS`. These binaries will be linked into an isolated location, and that isolated location will be put on the `PATH` of the isolated environment. You do not need to provide the full path to the binary - only the executable name that would be found by the shell. + +If you declare a tool as safe, and that tool cannot be found in the runtime environment, an error will be raised. + +Platform-specific environment variables are also available on platforms that use cross-platform environment isolation:
+ `CIBW_SAFE_TOOLS_IOS` + +#### Examples + +!!! tab examples "Environment variables" + + ```yaml + # Allow access to the cmake, ninja and rustc binaries in the isolated cross-build environment. + CIBW_SAFE_TOOLS: cmake ninja rustc + ``` + +!!! tab examples "pyproject.toml" + + ```toml + [tool.cibuildwheel] + + # Allow access to the cmake, ninja and rustc binaries in the isolated cross-build environment. + safe-tools = ["cmake", "ninja", "rustc"] + ``` ### `CIBW_REPAIR_WHEEL_COMMAND` {: #repair-wheel-command} > Execute a shell command to repair each built wheel diff --git a/test/test_ios.py b/test/test_ios.py index 6dc7b40d3..fa57247de 100644 --- a/test/test_ios.py +++ b/test/test_ios.py @@ -31,6 +31,8 @@ def test_platform(self): {"CIBW_PLATFORM": "ios"}, # Also check the build frontend {"CIBW_PLATFORM": "ios", "CIBW_BUILD_FRONTEND": "build"}, + # With a safe tool declaration + {"CIBW_PLATFORM": "ios", "CIBW_SAFE_TOOLS": "cmake"}, ], ) def test_ios_platforms(tmp_path, build_config): @@ -71,7 +73,6 @@ def test_ios_platforms(tmp_path, build_config): assert set(actual_wheels) == expected_wheels -@pytest.mark.xdist_group(name="ios") def test_no_test_sources(tmp_path, capfd): if utils.platform != "macos": pytest.skip("this test can only run on macOS") @@ -93,3 +94,27 @@ def test_no_test_sources(tmp_path, capfd): captured = capfd.readouterr() assert "Testing on iOS requires a definition of test-sources." in captured.err + + +def test_missing_safe_tool(tmp_path, capfd): + if utils.platform != "macos": + pytest.skip("this test can only run on macOS") + if utils.get_xcode_version() < (13, 0): + pytest.skip("this test only works with Xcode 13.0 or greater") + + project_dir = tmp_path / "project" + basic_project.generate(project_dir) + + with pytest.raises(subprocess.CalledProcessError): + utils.cibuildwheel_run( + project_dir, + add_env={ + "CIBW_PLATFORM": "ios", + "CIBW_BUILD": "cp313-*", + "CIBW_TEST_COMMAND": "tests", + "CIBW_SAFE_TOOLS": "does-not-exist", + }, + ) + + captured = capfd.readouterr() + assert "Could not find a 'does-not-exist' executable on the path." in captured.err diff --git a/unit_test/options_test.py b/unit_test/options_test.py index 1bd67aa77..239e8cd76 100644 --- a/unit_test/options_test.py +++ b/unit_test/options_test.py @@ -22,6 +22,7 @@ build = ["cp38-*", "cp313-*"] skip = ["*musllinux*"] environment = {FOO="BAR"} +safe-tools = ["cmake", "rustc"] test-command = "pyproject" test-sources = ["test", "other dir"] @@ -73,12 +74,14 @@ def test_options_1(tmp_path, monkeypatch): pinned_x86_64_container_image = all_pinned_container_images["x86_64"] local = options.build_options("cp38-manylinux_x86_64") + assert local.safe_tools == ["cmake", "rustc"] assert local.manylinux_images is not None assert local.test_command == "pyproject" assert local.test_sources == ["test", "other dir"] assert local.manylinux_images["x86_64"] == pinned_x86_64_container_image["manylinux1"] local = options.build_options("cp313-manylinux_x86_64") + assert local.safe_tools == ["cmake", "rustc"] assert local.manylinux_images is not None assert local.test_command == "pyproject-override" assert local.test_sources == ["test", "other dir"] diff --git a/unit_test/options_toml_test.py b/unit_test/options_toml_test.py index 749ff4b50..b31933bd1 100644 --- a/unit_test/options_toml_test.py +++ b/unit_test/options_toml_test.py @@ -89,6 +89,7 @@ def test_envvar_override(tmp_path, platform): env={ "CIBW_BUILD": "cp38*", "CIBW_MANYLINUX_X86_64_IMAGE": "manylinux_2_24", + "CIBW_SAFE_TOOLS": "cmake rustc", "CIBW_TEST_COMMAND": "mytest", "CIBW_TEST_REQUIRES": "docs", "CIBW_TEST_GROUPS": "mgroup two", @@ -104,6 +105,10 @@ def test_envvar_override(tmp_path, platform): assert options_reader.get("manylinux-x86_64-image") == "manylinux_2_24" assert options_reader.get("manylinux-i686-image") == "manylinux2014" + assert ( + options_reader.get("safe-tools", option_format=ListFormat(" ", quote=shlex.quote)) + == "cmake rustc" + ) assert ( options_reader.get("test-sources", option_format=ListFormat(" ", quote=shlex.quote)) == 'first "second third"' From 471632ebb16906f9cfdf9e91402615ff7fb1aed0 Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Thu, 13 Mar 2025 14:05:34 +0800 Subject: [PATCH 02/12] Add an xfail if cmake isn't available on the test machine. --- test/test_ios.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/test_ios.py b/test/test_ios.py index fa57247de..efc31eaff 100644 --- a/test/test_ios.py +++ b/test/test_ios.py @@ -2,6 +2,7 @@ import os import platform +import shutil import subprocess import pytest @@ -40,6 +41,8 @@ def test_ios_platforms(tmp_path, build_config): pytest.skip("this test can only run on macOS") if utils.get_xcode_version() < (13, 0): pytest.skip("this test only works with Xcode 13.0 or greater") + if "CIBW_SAFE_TOOLS" in build_config and shutil.which("cmake") is None: + pytest.xfail("test machine doesn't have cmake installed") project_dir = tmp_path / "project" basic_project.generate(project_dir) From 598af3c8f0866aaa36be540a504d87cac5830f02 Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Mon, 17 Mar 2025 10:02:47 +0800 Subject: [PATCH 03/12] Placate linter regarding positional args. --- cibuildwheel/ios.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/cibuildwheel/ios.py b/cibuildwheel/ios.py index 92029075f..f9742bd53 100644 --- a/cibuildwheel/ios.py +++ b/cibuildwheel/ios.py @@ -253,6 +253,7 @@ def cross_virtualenv( def setup_python( tmp: Path, + *, python_configuration: PythonConfiguration, dependency_constraint_flags: Sequence[PathOrStr], environment: ParsedEnvironment, @@ -434,10 +435,10 @@ def build(options: Options, tmp_path: Path) -> None: target_install_path, env = setup_python( identifier_tmp_dir / "build", - config, - dependency_constraint_flags, - build_options.environment, - build_frontend.name, + python_configuration=config, + dependency_constraint_flags=dependency_constraint_flags, + environment=build_options.environment, + build_frontend=build_frontend.name, safe_tools=build_options.safe_tools, ) pip_version = get_pip_version(env) From 27af6e0dfb51e77df6b9146c4037f73e71b2b7e7 Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Mon, 17 Mar 2025 10:32:40 +0800 Subject: [PATCH 04/12] Rework test to provide more robust confirmation of safe tools. --- test/test_ios.py | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/test/test_ios.py b/test/test_ios.py index efc31eaff..886ac7318 100644 --- a/test/test_ios.py +++ b/test/test_ios.py @@ -9,8 +9,8 @@ from . import test_projects, utils -basic_project = test_projects.new_c_project() -basic_project.files["tests/test_platform.py"] = f""" +basic_project_files = { + "tests/test_platform.py": f""" import platform from unittest import TestCase @@ -19,6 +19,7 @@ def test_platform(self): self.assertEqual(platform.machine(), "{platform.machine()}") """ +} # iOS tests shouldn't be run in parallel, because they're dependent on starting @@ -32,11 +33,9 @@ def test_platform(self): {"CIBW_PLATFORM": "ios"}, # Also check the build frontend {"CIBW_PLATFORM": "ios", "CIBW_BUILD_FRONTEND": "build"}, - # With a safe tool declaration - {"CIBW_PLATFORM": "ios", "CIBW_SAFE_TOOLS": "cmake"}, ], ) -def test_ios_platforms(tmp_path, build_config): +def test_ios_platforms(tmp_path, build_config, monkeypatch): if utils.platform != "macos": pytest.skip("this test can only run on macOS") if utils.get_xcode_version() < (13, 0): @@ -44,13 +43,31 @@ def test_ios_platforms(tmp_path, build_config): if "CIBW_SAFE_TOOLS" in build_config and shutil.which("cmake") is None: pytest.xfail("test machine doesn't have cmake installed") + # Create a temporary "bin" directory, symlink a tool that we know eixsts + # (/usr/bin/true) into that location under a name that should be unique, + # and add the temp bin directory to the PATH. + tools_dir = tmp_path / "bin" + tools_dir.mkdir() + tools_dir.joinpath("does-exist").symlink_to(shutil.which("true")) + + monkeypatch.setenv("PATH", str(tools_dir), prepend=os.pathsep) + + # Generate a test project that has an additional before-build step using the + # known-to-exist tool. project_dir = tmp_path / "project" + setup_py_add = "import subprocess\nsubprocess.run('does-exist', check=True)\n" + basic_project = test_projects.new_c_project(setup_py_add=setup_py_add) + basic_project.files.update(basic_project_files) basic_project.generate(project_dir) + # Build the wheels. Mark the "does-exist" tool as safe, and invoke it during + # a `before-build` step. It will also be invoked when `setup.py` is invoked. actual_wheels = utils.cibuildwheel_run( project_dir, add_env={ + "CIBW_BEFORE_BUILD": "does-exist", "CIBW_BUILD": "cp313-*", + "CIBW_SAFE_TOOLS": "does-exist", "CIBW_TEST_SOURCES": "tests", "CIBW_TEST_COMMAND": "unittest discover tests test_platform.py", **build_config, @@ -83,6 +100,8 @@ def test_no_test_sources(tmp_path, capfd): pytest.skip("this test only works with Xcode 13.0 or greater") project_dir = tmp_path / "project" + basic_project = test_projects.new_c_project() + basic_project.files.update(basic_project_files) basic_project.generate(project_dir) with pytest.raises(subprocess.CalledProcessError): @@ -106,6 +125,8 @@ def test_missing_safe_tool(tmp_path, capfd): pytest.skip("this test only works with Xcode 13.0 or greater") project_dir = tmp_path / "project" + basic_project = test_projects.new_c_project() + basic_project.files.update(basic_project_files) basic_project.generate(project_dir) with pytest.raises(subprocess.CalledProcessError): From 3f13cb590cc9aaa7f900b10c630ae380121b9ae2 Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Tue, 18 Mar 2025 06:10:39 +0800 Subject: [PATCH 05/12] Remove a test skip condition that is no longer needed. Co-authored-by: Joe Rickerby --- test/test_ios.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/test_ios.py b/test/test_ios.py index 886ac7318..d7f315c4a 100644 --- a/test/test_ios.py +++ b/test/test_ios.py @@ -40,8 +40,6 @@ def test_ios_platforms(tmp_path, build_config, monkeypatch): pytest.skip("this test can only run on macOS") if utils.get_xcode_version() < (13, 0): pytest.skip("this test only works with Xcode 13.0 or greater") - if "CIBW_SAFE_TOOLS" in build_config and shutil.which("cmake") is None: - pytest.xfail("test machine doesn't have cmake installed") # Create a temporary "bin" directory, symlink a tool that we know eixsts # (/usr/bin/true) into that location under a name that should be unique, From f7b60ddd1bf406bace64065480337efb03ef6a17 Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Wed, 19 Mar 2025 08:54:49 +0800 Subject: [PATCH 06/12] Rename the setting to xbuild-tools. --- README.md | 2 +- cibuildwheel/ios.py | 24 +++++++++---------- cibuildwheel/options.py | 10 ++++---- .../resources/cibuildwheel.schema.json | 12 +++++----- cibuildwheel/resources/defaults.toml | 2 +- docs/options.md | 19 +++++++-------- test/test_ios.py | 11 +++++---- unit_test/options_test.py | 6 ++--- unit_test/options_toml_test.py | 4 ++-- 9 files changed, 46 insertions(+), 44 deletions(-) diff --git a/README.md b/README.md index 0d25f2634..525fa31b8 100644 --- a/README.md +++ b/README.md @@ -139,7 +139,7 @@ Options | | [`CIBW_ENVIRONMENT_PASS_LINUX`](https://cibuildwheel.pypa.io/en/stable/options/#environment-pass) | Set environment variables on the host to pass-through to the container during the build. | | | [`CIBW_BEFORE_ALL`](https://cibuildwheel.pypa.io/en/stable/options/#before-all) | Execute a shell command on the build system before any wheels are built. | | | [`CIBW_BEFORE_BUILD`](https://cibuildwheel.pypa.io/en/stable/options/#before-build) | Execute a shell command preparing each wheel's build | -| | [`CIBW_SAFE_TOOLS`](https://cibuildwheel.pypa.io/en/stable/options/#safe-tools) | Binaries on the path that are safe to include in an isolated cross-build environment. | +| | [`CIBW_XBUILD_TOOLS`](https://cibuildwheel.pypa.io/en/stable/options/#xbuild-tools) | Binaries on the path that should be included in an isolated cross-build environment. | | | [`CIBW_REPAIR_WHEEL_COMMAND`](https://cibuildwheel.pypa.io/en/stable/options/#repair-wheel-command) | Execute a shell command to repair each built wheel | | | [`CIBW_MANYLINUX_*_IMAGE`
`CIBW_MUSLLINUX_*_IMAGE`](https://cibuildwheel.pypa.io/en/stable/options/#linux-image) | Specify alternative manylinux / musllinux Docker images | | | [`CIBW_CONTAINER_ENGINE`](https://cibuildwheel.pypa.io/en/stable/options/#container-engine) | Specify which container engine to use when building Linux wheels | diff --git a/cibuildwheel/ios.py b/cibuildwheel/ios.py index f9742bd53..39211cb1f 100644 --- a/cibuildwheel/ios.py +++ b/cibuildwheel/ios.py @@ -147,7 +147,7 @@ def cross_virtualenv( build_python: Path, venv_path: Path, dependency_constraint_flags: Sequence[PathOrStr], - safe_tools: Sequence[str], + xbuild_tools: Sequence[str], ) -> dict[str, str]: """Create a cross-compilation virtual environment. @@ -175,7 +175,7 @@ def cross_virtualenv( created. :param dependency_constraint_flags: Any flags that should be used when constraining dependencies in the environment. - :param safe_tools: A list of executable names (without paths) that are + :param xbuild_tools: A list of executable names (without paths) that are on the path, but must be preserved in the cross environment. """ # Create an initial macOS virtual environment @@ -215,9 +215,9 @@ def cross_virtualenv( # (fully resolving symlinks) to a "safe" location that will *only* contain # those tools. This avoids needing to add *all* of Homebrew to the path just # to get access to (for example) cmake for build purposes. - safe_tools_path = venv_path / "cibw_safe_tools" - safe_tools_path.mkdir() - for tool in safe_tools: + xbuild_tools_path = venv_path / "cibw_xbuild_tools" + xbuild_tools_path.mkdir() + for tool in xbuild_tools: tool_path = shutil.which(tool) if tool_path is None: msg = f"Could not find a {tool!r} executable on the path." @@ -225,8 +225,8 @@ def cross_virtualenv( # Link the binary into the safe tools directory original = Path(tool_path).resolve() - print(f"{tool!r} is a safe tool in the cross-build environment (using {original})") - (safe_tools_path / tool).symlink_to(original) + print(f"{tool!r} will be included in the cross-build environment (using {original})") + (xbuild_tools_path / tool).symlink_to(original) env["PATH"] = os.pathsep.join( [ @@ -234,8 +234,8 @@ def cross_virtualenv( str(target_python.parent), # The cross-platform environment's binary directory str(venv_path / "bin"), - # The directory of safe tools - str(safe_tools_path), + # The directory of cross-build tools + str(xbuild_tools_path), # The bare minimum Apple system paths. "/usr/bin", "/bin", @@ -258,7 +258,7 @@ def setup_python( dependency_constraint_flags: Sequence[PathOrStr], environment: ParsedEnvironment, build_frontend: BuildFrontendName, - safe_tools: Sequence[str], + xbuild_tools: Sequence[str], ) -> tuple[Path, dict[str, str]]: if build_frontend == "build[uv]": msg = "uv doesn't support iOS" @@ -311,7 +311,7 @@ def setup_python( build_python=build_python, venv_path=venv_path, dependency_constraint_flags=dependency_constraint_flags, - safe_tools=safe_tools, + xbuild_tools=xbuild_tools, ) venv_bin_path = venv_path / "bin" assert venv_bin_path.exists() @@ -439,7 +439,7 @@ def build(options: Options, tmp_path: Path) -> None: dependency_constraint_flags=dependency_constraint_flags, environment=build_options.environment, build_frontend=build_frontend.name, - safe_tools=build_options.safe_tools, + xbuild_tools=build_options.xbuild_tools, ) pip_version = get_pip_version(env) diff --git a/cibuildwheel/options.py b/cibuildwheel/options.py index e5520eb1a..84a006df2 100644 --- a/cibuildwheel/options.py +++ b/cibuildwheel/options.py @@ -94,7 +94,7 @@ class BuildOptions: environment: ParsedEnvironment before_all: str before_build: str | None - safe_tools: list[str] + xbuild_tools: list[str] repair_command: str manylinux_images: dict[str, str] | None musllinux_images: dict[str, str] | None @@ -697,8 +697,10 @@ def _compute_build_options(self, identifier: str | None) -> BuildOptions: test_command = self.reader.get("test-command", option_format=ListFormat(sep=" && ")) before_test = self.reader.get("before-test", option_format=ListFormat(sep=" && ")) - safe_tools = shlex.split( - self.reader.get("safe-tools", option_format=ListFormat(sep=" ", quote=shlex.quote)) + xbuild_tools = shlex.split( + self.reader.get( + "xbuild-tools", option_format=ListFormat(sep=" ", quote=shlex.quote) + ) ) test_sources = shlex.split( self.reader.get( @@ -842,7 +844,7 @@ def _compute_build_options(self, identifier: str | None) -> BuildOptions: before_build=before_build, before_all=before_all, build_verbosity=build_verbosity, - safe_tools=safe_tools, + xbuild_tools=xbuild_tools, repair_command=repair_command, environment=environment, dependency_constraints=dependency_constraints, diff --git a/cibuildwheel/resources/cibuildwheel.schema.json b/cibuildwheel/resources/cibuildwheel.schema.json index 19d6a98c9..10e07cc15 100644 --- a/cibuildwheel/resources/cibuildwheel.schema.json +++ b/cibuildwheel/resources/cibuildwheel.schema.json @@ -397,8 +397,8 @@ "description": "Specify alternative manylinux / musllinux container images", "title": "CIBW_MUSLLINUX_X86_64_IMAGE" }, - "safe-tools": { - "description": "Binaries on the path that are safe to include in an isolated cross-build environment", + "xbuild-tools": { + "description": "Binaries on the path that should be included in an isolated cross-build environment", "oneOf": [ { "type": "string" @@ -410,7 +410,7 @@ } } ], - "title": "CIBW_SAFE_TOOLS" + "title": "CIBW_XBUILD_TOOLS" }, "repair-wheel-command": { "description": "Execute a shell command to repair each built wheel.", @@ -581,7 +581,7 @@ "environment-pass": { "$ref": "#/$defs/inherit" }, - "safe-tools": { + "xbuild-tools": { "$ref": "#/$defs/inherit" }, "repair-wheel-command": { @@ -1009,8 +1009,8 @@ "repair-wheel-command": { "$ref": "#/properties/repair-wheel-command" }, - "safe-tools": { - "$ref": "#/properties/safe-tools" + "xbuild-tools": { + "$ref": "#/properties/xbuild-tools" }, "test-command": { "$ref": "#/properties/test-command" diff --git a/cibuildwheel/resources/defaults.toml b/cibuildwheel/resources/defaults.toml index c6124afe1..547b5d831 100644 --- a/cibuildwheel/resources/defaults.toml +++ b/cibuildwheel/resources/defaults.toml @@ -14,7 +14,7 @@ build-verbosity = 0 before-all = "" before-build = "" -safe-tools = [] +xbuild-tools = [] repair-wheel-command = "" test-command = "" diff --git a/docs/options.md b/docs/options.md index 4073c8b5a..cb4f2cf4c 100644 --- a/docs/options.md +++ b/docs/options.md @@ -1044,34 +1044,33 @@ Platform-specific environment variables are also available:
[PEP 517]: https://www.python.org/dev/peps/pep-0517/ [PEP 518]: https://www.python.org/dev/peps/pep-0517/ -### `CIBW_SAFE_TOOLS` {: #safe-tools} -> Binaries on the path that are safe to include in an isolated cross-build environment. +### `CIBW_XBUILD_TOOLS` {: #xbuild-tools} +> Binaries on the path that should be included in an isolated cross-build environment. When building in a cross-platform environment, it is sometimes necessary to isolate the ``PATH`` so that binaries from the build machine don't accidentally get linked into the cross-platform binary. However, this isolation process will also hide tools that might be required to build your wheel. -If there are binaries present on the `PATH` when you invoke cibuildwheel, and those binaries are required to build your wheels, those binaries can be explicitly declared as "safe" using `CIBW_SAFE_TOOLS`. These binaries will be linked into an isolated location, and that isolated location will be put on the `PATH` of the isolated environment. You do not need to provide the full path to the binary - only the executable name that would be found by the shell. +If there are binaries present on the `PATH` when you invoke cibuildwheel, and those binaries are required to build your wheels, those binaries can be explicitly included in the isolated cross-build environment using `CIBW_XBUILD_TOOLS`. The binaries listed in this setting will be linked into an isolated location, and that isolated location will be put on the `PATH` of the isolated environment. You do not need to provide the full path to the binary - only the executable name that would be found by the shell. -If you declare a tool as safe, and that tool cannot be found in the runtime environment, an error will be raised. +If you declare a tool as a cross-build tool, and that tool cannot be found in the runtime environment, an error will be raised. Platform-specific environment variables are also available on platforms that use cross-platform environment isolation:
- `CIBW_SAFE_TOOLS_IOS` + `CIBW_XBUILD_TOOLS_IOS` #### Examples !!! tab examples "Environment variables" ```yaml - # Allow access to the cmake, ninja and rustc binaries in the isolated cross-build environment. - CIBW_SAFE_TOOLS: cmake ninja rustc + # Allow access to the cmake and rustc binaries in the isolated cross-build environment. + CIBW_XBUILD_TOOLS: cmake rustc ``` !!! tab examples "pyproject.toml" ```toml [tool.cibuildwheel] - - # Allow access to the cmake, ninja and rustc binaries in the isolated cross-build environment. - safe-tools = ["cmake", "ninja", "rustc"] + # Allow access to the cmake and rustc binaries in the isolated cross-build environment. + xbuild-tools = ["cmake", "rustc"] ``` ### `CIBW_REPAIR_WHEEL_COMMAND` {: #repair-wheel-command} diff --git a/test/test_ios.py b/test/test_ios.py index d7f315c4a..8326d5f72 100644 --- a/test/test_ios.py +++ b/test/test_ios.py @@ -58,14 +58,15 @@ def test_ios_platforms(tmp_path, build_config, monkeypatch): basic_project.files.update(basic_project_files) basic_project.generate(project_dir) - # Build the wheels. Mark the "does-exist" tool as safe, and invoke it during - # a `before-build` step. It will also be invoked when `setup.py` is invoked. + # Build the wheels. Mark the "does-exist" tool as a cross-build tool, and + # invoke it during a `before-build` step. It will also be invoked when + # `setup.py` is invoked. actual_wheels = utils.cibuildwheel_run( project_dir, add_env={ "CIBW_BEFORE_BUILD": "does-exist", "CIBW_BUILD": "cp313-*", - "CIBW_SAFE_TOOLS": "does-exist", + "CIBW_XBUILD_TOOLS": "does-exist", "CIBW_TEST_SOURCES": "tests", "CIBW_TEST_COMMAND": "unittest discover tests test_platform.py", **build_config, @@ -116,7 +117,7 @@ def test_no_test_sources(tmp_path, capfd): assert "Testing on iOS requires a definition of test-sources." in captured.err -def test_missing_safe_tool(tmp_path, capfd): +def test_missing_xbuild_tool(tmp_path, capfd): if utils.platform != "macos": pytest.skip("this test can only run on macOS") if utils.get_xcode_version() < (13, 0): @@ -134,7 +135,7 @@ def test_missing_safe_tool(tmp_path, capfd): "CIBW_PLATFORM": "ios", "CIBW_BUILD": "cp313-*", "CIBW_TEST_COMMAND": "tests", - "CIBW_SAFE_TOOLS": "does-not-exist", + "CIBW_XBUILD_TOOLS": "does-not-exist", }, ) diff --git a/unit_test/options_test.py b/unit_test/options_test.py index 239e8cd76..a654d8f2f 100644 --- a/unit_test/options_test.py +++ b/unit_test/options_test.py @@ -22,7 +22,7 @@ build = ["cp38-*", "cp313-*"] skip = ["*musllinux*"] environment = {FOO="BAR"} -safe-tools = ["cmake", "rustc"] +xbuild-tools = ["cmake", "rustc"] test-command = "pyproject" test-sources = ["test", "other dir"] @@ -74,14 +74,14 @@ def test_options_1(tmp_path, monkeypatch): pinned_x86_64_container_image = all_pinned_container_images["x86_64"] local = options.build_options("cp38-manylinux_x86_64") - assert local.safe_tools == ["cmake", "rustc"] + assert local.xbuild_tools == ["cmake", "rustc"] assert local.manylinux_images is not None assert local.test_command == "pyproject" assert local.test_sources == ["test", "other dir"] assert local.manylinux_images["x86_64"] == pinned_x86_64_container_image["manylinux1"] local = options.build_options("cp313-manylinux_x86_64") - assert local.safe_tools == ["cmake", "rustc"] + assert local.xbuild_tools == ["cmake", "rustc"] assert local.manylinux_images is not None assert local.test_command == "pyproject-override" assert local.test_sources == ["test", "other dir"] diff --git a/unit_test/options_toml_test.py b/unit_test/options_toml_test.py index b31933bd1..39ad88e65 100644 --- a/unit_test/options_toml_test.py +++ b/unit_test/options_toml_test.py @@ -89,7 +89,7 @@ def test_envvar_override(tmp_path, platform): env={ "CIBW_BUILD": "cp38*", "CIBW_MANYLINUX_X86_64_IMAGE": "manylinux_2_24", - "CIBW_SAFE_TOOLS": "cmake rustc", + "CIBW_XBUILD_TOOLS": "cmake rustc", "CIBW_TEST_COMMAND": "mytest", "CIBW_TEST_REQUIRES": "docs", "CIBW_TEST_GROUPS": "mgroup two", @@ -106,7 +106,7 @@ def test_envvar_override(tmp_path, platform): assert options_reader.get("manylinux-i686-image") == "manylinux2014" assert ( - options_reader.get("safe-tools", option_format=ListFormat(" ", quote=shlex.quote)) + options_reader.get("xbuild-tools", option_format=ListFormat(" ", quote=shlex.quote)) == "cmake rustc" ) assert ( From d2dcabb51aec20e1ba5ee4b29002174dfa8ba60e Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Wed, 19 Mar 2025 08:58:52 +0800 Subject: [PATCH 07/12] Add docs to clarify that xbuild-tools is transitive. --- docs/options.md | 2 ++ docs/platforms/ios.md | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/options.md b/docs/options.md index cb4f2cf4c..dbfa5d4f0 100644 --- a/docs/options.md +++ b/docs/options.md @@ -1053,6 +1053,8 @@ If there are binaries present on the `PATH` when you invoke cibuildwheel, and th If you declare a tool as a cross-build tool, and that tool cannot be found in the runtime environment, an error will be raised. +*Any* tool used by the build process must be included in the `CIBW_XBUILD_TOOLS` list, not just tools that cibuildwheel will invoke directly. For example, if your build invokes `cmake`, and the `cmake` script invokes `magick` to perform some image transformations, both `cmake` and `magick` must be included in your safe tools list. + Platform-specific environment variables are also available on platforms that use cross-platform environment isolation:
`CIBW_XBUILD_TOOLS_IOS` diff --git a/docs/platforms/ios.md b/docs/platforms/ios.md index a58c0132b..72da6987a 100644 --- a/docs/platforms/ios.md +++ b/docs/platforms/ios.md @@ -57,7 +57,9 @@ iOS builds support both the `pip` and `build` build frontends. In principle, sup ## Build environment -The environment used to run builds does not inherit the full user environment - in particular, `PATH` is deliberately re-written. This is because UNIX C tooling doesn't do a great job differentiating between "macOS ARM64" and "iOS ARM64" binaries. If (for example) Homebrew is on the path when compilation commands are invoked, it's easy for a macOS version of a library to be linked into the iOS binary, rendering it unusable on iOS. To prevent this, iOS builds always force `PATH` to a "known minimal" path, that includes only the bare system utilities, plus the current user's cargo folder (to facilitate Rust builds). +The environment used to run builds does not inherit the full user environment - in particular, `PATH` is deliberately re-written. This is because UNIX C tooling doesn't do a great job differentiating between "macOS ARM64" and "iOS ARM64" binaries. If (for example) Homebrew is on the path when compilation commands are invoked, it's easy for a macOS version of a library to be linked into the iOS binary, rendering it unusable on iOS. To prevent this, iOS builds always force `PATH` to a "known minimal" path, that includes only the bare system utilities, and the iOS compiler toolchain. + +If your project requires additional tools to build (such as `cmake`, `ninja`, or `rustc`), those tools must be explicitly declared as cross-build tools using [`CIBW_XBUILD_TOOLS`](../../options#xbuild-tools). *Any* tool used by the build process must be included in the `CIBW_XBUILD_TOOLS` list, not just tools that cibuildwheel will invoke directly. For example, if your build script invokes `cmake`, and the `cmake` script invokes `magick` to perform some image transformations, both `cmake` and `magick` must be included in your safe tools list. ## Tests From a775dff85e68416a1afef91a477fe075a64dbb67 Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Tue, 1 Apr 2025 14:03:37 +0800 Subject: [PATCH 08/12] Raise a warning if xbuild-tools isn't defined. --- bin/generate_schema.py | 9 ++- cibuildwheel/platforms/ios.py | 43 +++++++--- cibuildwheel/resources/defaults.toml | 3 +- docs/options.md | 13 +++ docs/platforms/ios.md | 2 +- test/test_ios.py | 115 +++++++++++++++++++++++++-- 6 files changed, 163 insertions(+), 22 deletions(-) diff --git a/bin/generate_schema.py b/bin/generate_schema.py index b82a91136..0d471e4ff 100755 --- a/bin/generate_schema.py +++ b/bin/generate_schema.py @@ -42,9 +42,6 @@ before-all: description: Execute a shell command on the build system before any wheels are built. type: string_array - before-build: - description: Execute a shell command preparing each wheel's build. - type: string_array before-test: description: Execute a shell command before testing each wheel. type: string_array @@ -181,9 +178,12 @@ musllinux-x86_64-image: type: string description: Specify alternative manylinux / musllinux container images - repair-wheel-command: + before-build: + description: Binaries on the path that should be included in an isolated cross-build environment type: string_array + repair-wheel-command: description: Execute a shell command to repair each built wheel. + type: string_array skip: description: Choose the Python versions to skip. type: string_array @@ -273,6 +273,7 @@ properties: before-all: {"$ref": "#/$defs/inherit"} before-build: {"$ref": "#/$defs/inherit"} + xbuild-tools: {"$ref": "#/$defs/inherit"} before-test: {"$ref": "#/$defs/inherit"} config-settings: {"$ref": "#/$defs/inherit"} container-engine: {"$ref": "#/$defs/inherit"} diff --git a/cibuildwheel/platforms/ios.py b/cibuildwheel/platforms/ios.py index a99853d35..7c525a900 100644 --- a/cibuildwheel/platforms/ios.py +++ b/cibuildwheel/platforms/ios.py @@ -5,6 +5,7 @@ import shutil import subprocess import sys +import textwrap from collections.abc import Sequence, Set from dataclasses import dataclass from pathlib import Path @@ -221,16 +222,38 @@ def cross_virtualenv( # to get access to (for example) cmake for build purposes. xbuild_tools_path = venv_path / "cibw_xbuild_tools" xbuild_tools_path.mkdir() - for tool in xbuild_tools: - tool_path = shutil.which(tool) - if tool_path is None: - msg = f"Could not find a {tool!r} executable on the path." - raise errors.FatalError(msg) - - # Link the binary into the safe tools directory - original = Path(tool_path).resolve() - print(f"{tool!r} will be included in the cross-build environment (using {original})") - (xbuild_tools_path / tool).symlink_to(original) + # ["\u0000"] is a sentinel value used as a default, because TOML doesn't + # have an explicit NULL value. If xbuild-tools is set to the sentinel, it + # indicates that the user hasn't defined xbuild_tools *at all* (not even an + # `xbuild_tools = []` definition). + if xbuild_tools == ["\u0000"]: + log.warning( + textwrap.dedent( + """ + Your project configuration does not define any cross-build tools. + + iOS builds use an isolated build environment; if your build process requires any + third-party tools (such as cmake, ninja, or rustc), you must explicitly declare + that those tools are required using xbuild-tools/CIBW_XBUILD_TOOLS. This will + likely manifest as a "somebuildtool: command not found" error. + + If the build succeeds, you can silence this warning by setting adding + `xbuild-tools = []` to your pyproject.toml configuration, or exporting + CIBW_XBUILD_TOOLS as an empty string into your environment. + """ + ) + ) + else: + for tool in xbuild_tools: + tool_path = shutil.which(tool) + if tool_path is None: + msg = f"Could not find a {tool!r} executable on the path." + raise errors.FatalError(msg) + + # Link the binary into the safe tools directory + original = Path(tool_path).resolve() + print(f"{tool!r} will be included in the cross-build environment (using {original})") + (xbuild_tools_path / tool).symlink_to(original) env["PATH"] = os.pathsep.join( [ diff --git a/cibuildwheel/resources/defaults.toml b/cibuildwheel/resources/defaults.toml index 6c26b50f3..8ca424556 100644 --- a/cibuildwheel/resources/defaults.toml +++ b/cibuildwheel/resources/defaults.toml @@ -14,7 +14,8 @@ build-verbosity = 0 before-all = "" before-build = "" -xbuild-tools = [] +# TOML doesn't support explicit NULLs; use ["\u0000"] as a sentinel value. +xbuild-tools = ["\u0000"] repair-wheel-command = "" test-command = "" diff --git a/docs/options.md b/docs/options.md index 0113ec13f..baab338bc 100644 --- a/docs/options.md +++ b/docs/options.md @@ -1052,6 +1052,8 @@ If there are binaries present on the `PATH` when you invoke cibuildwheel, and th If you declare a tool as a cross-build tool, and that tool cannot be found in the runtime environment, an error will be raised. +If you do not define `CIBW_XBUILD_TOOLS`, and you build for a platform that uses a cross-platform environment, a warning will be raised. If your project does not require any cross-build tools, you can set `CIBW_XBUILD_TOOLS` to an empty list to silence this warning. + *Any* tool used by the build process must be included in the `CIBW_XBUILD_TOOLS` list, not just tools that cibuildwheel will invoke directly. For example, if your build invokes `cmake`, and the `cmake` script invokes `magick` to perform some image transformations, both `cmake` and `magick` must be included in your safe tools list. Platform-specific environment variables are also available on platforms that use cross-platform environment isolation:
@@ -1066,6 +1068,11 @@ Platform-specific environment variables are also available on platforms that use CIBW_XBUILD_TOOLS: cmake rustc ``` + ```yaml + # No cross-build tools are required + CIBW_XBUILD_TOOLS: + ``` + !!! tab examples "pyproject.toml" ```toml @@ -1074,6 +1081,12 @@ Platform-specific environment variables are also available on platforms that use xbuild-tools = ["cmake", "rustc"] ``` + ```toml + [tool.cibuildwheel] + # No cross-build tools are required + xbuild-tools = [] + ``` + ### `CIBW_REPAIR_WHEEL_COMMAND` {: #repair-wheel-command} > Execute a shell command to repair each built wheel diff --git a/docs/platforms/ios.md b/docs/platforms/ios.md index 72da6987a..0c93ba052 100644 --- a/docs/platforms/ios.md +++ b/docs/platforms/ios.md @@ -59,7 +59,7 @@ iOS builds support both the `pip` and `build` build frontends. In principle, sup The environment used to run builds does not inherit the full user environment - in particular, `PATH` is deliberately re-written. This is because UNIX C tooling doesn't do a great job differentiating between "macOS ARM64" and "iOS ARM64" binaries. If (for example) Homebrew is on the path when compilation commands are invoked, it's easy for a macOS version of a library to be linked into the iOS binary, rendering it unusable on iOS. To prevent this, iOS builds always force `PATH` to a "known minimal" path, that includes only the bare system utilities, and the iOS compiler toolchain. -If your project requires additional tools to build (such as `cmake`, `ninja`, or `rustc`), those tools must be explicitly declared as cross-build tools using [`CIBW_XBUILD_TOOLS`](../../options#xbuild-tools). *Any* tool used by the build process must be included in the `CIBW_XBUILD_TOOLS` list, not just tools that cibuildwheel will invoke directly. For example, if your build script invokes `cmake`, and the `cmake` script invokes `magick` to perform some image transformations, both `cmake` and `magick` must be included in your safe tools list. +If your project requires additional tools to build (such as `cmake`, `ninja`, or `rustc`), those tools must be explicitly declared as cross-build tools using [`CIBW_XBUILD_TOOLS`](../../options#xbuild-tools). *Any* tool used by the build process must be included in the `CIBW_XBUILD_TOOLS` list, not just tools that cibuildwheel will invoke directly. For example, if your build script invokes `cmake`, and the `cmake` script invokes `magick` to perform some image transformations, both `cmake` and `magick` must be included in your cross-build tools list. ## Tests diff --git a/test/test_ios.py b/test/test_ios.py index 00756b215..09cb1c931 100644 --- a/test/test_ios.py +++ b/test/test_ios.py @@ -37,7 +37,7 @@ def test_platform(self): {"CIBW_PLATFORM": "ios", "CIBW_BUILD_FRONTEND": "build"}, ], ) -def test_ios_platforms(tmp_path, build_config, monkeypatch): +def test_ios_platforms(tmp_path, build_config, monkeypatch, capfd): if utils.platform != "macos": pytest.skip("this test can only run on macOS") if utils.get_xcode_version() < (13, 0): @@ -60,9 +60,12 @@ def test_ios_platforms(tmp_path, build_config, monkeypatch): basic_project.files.update(basic_project_files) basic_project.generate(project_dir) - # Build the wheels. Mark the "does-exist" tool as a cross-build tool, and - # invoke it during a `before-build` step. It will also be invoked when - # `setup.py` is invoked. + # Build and test the wheels. Mark the "does-exist" tool as a cross-build + # tool, and invoke it during a `before-build` step. It will also be invoked + # when `setup.py` is invoked. + # + # Tests are only executed on simulator. The test suite passes if it's + # running on the same architecture as the current platform. actual_wheels = utils.cibuildwheel_run( project_dir, add_env={ @@ -76,11 +79,10 @@ def test_ios_platforms(tmp_path, build_config, monkeypatch): }, ) + # The expected wheels were produced. ios_version = os.getenv("IPHONEOS_DEPLOYMENT_TARGET", "13.0").replace(".", "_") platform_machine = platform.machine() - # Tests are only executed on simulator. The test suite passes if it's - # running on the same architecture as the current platform. if platform_machine == "x86_64": expected_wheels = { f"spam-0.1.0-cp313-cp313-ios_{ios_version}_x86_64_iphonesimulator.whl", @@ -94,9 +96,14 @@ def test_ios_platforms(tmp_path, build_config, monkeypatch): assert set(actual_wheels) == expected_wheels + # The user was notified that the cross-build tool was found. + captured = capfd.readouterr() + assert "'does-exist' will be included in the cross-build environment" in captured.out + @pytest.mark.serial def test_no_test_sources(tmp_path, capfd): + """Build will fail if test-sources isn't defined.""" if utils.platform != "macos": pytest.skip("this test can only run on macOS") if utils.get_xcode_version() < (13, 0): @@ -117,12 +124,14 @@ def test_no_test_sources(tmp_path, capfd): }, ) + # The error message indicates the configuration issue. captured = capfd.readouterr() assert "Testing on iOS requires a definition of test-sources." in captured.err @pytest.mark.serial def test_missing_xbuild_tool(tmp_path, capfd): + """Build will fail if xbuild-tools references a non-existent tool.""" if utils.platform != "macos": pytest.skip("this test can only run on macOS") if utils.get_xcode_version() < (13, 0): @@ -144,5 +153,99 @@ def test_missing_xbuild_tool(tmp_path, capfd): }, ) + # The error message indicates the problem tool. captured = capfd.readouterr() assert "Could not find a 'does-not-exist' executable on the path." in captured.err + + +@pytest.mark.serial +def test_no_xbuild_tool_definition(tmp_path, capfd): + """Build will succeed with a warning if there is no xbuild-tools definition.""" + if utils.platform != "macos": + pytest.skip("this test can only run on macOS") + if utils.get_xcode_version() < (13, 0): + pytest.skip("this test only works with Xcode 13.0 or greater") + + project_dir = tmp_path / "project" + basic_project = test_projects.new_c_project() + basic_project.files.update(basic_project_files) + basic_project.generate(project_dir) + + # Build, but don't test the wheels; we're only checking that the right + # warning was raised. + actual_wheels = utils.cibuildwheel_run( + project_dir, + add_env={ + "CIBW_PLATFORM": "ios", + "CIBW_BUILD": "cp313-*", + "CIBW_TEST_SKIP": "*", + }, + ) + + # The expected wheels were produced. + ios_version = os.getenv("IPHONEOS_DEPLOYMENT_TARGET", "13.0").replace(".", "_") + platform_machine = platform.machine() + + if platform_machine == "x86_64": + expected_wheels = { + f"spam-0.1.0-cp313-cp313-ios_{ios_version}_x86_64_iphonesimulator.whl", + } + + elif platform_machine == "arm64": + expected_wheels = { + f"spam-0.1.0-cp313-cp313-ios_{ios_version}_arm64_iphoneos.whl", + f"spam-0.1.0-cp313-cp313-ios_{ios_version}_arm64_iphonesimulator.whl", + } + + assert set(actual_wheels) == expected_wheels + + # The user was notified that there was no cross-build tool definition. + captured = capfd.readouterr() + assert "Your project configuration does not define any cross-build tools." in captured.err + + +@pytest.mark.serial +def test_empty_xbuild_tool_definition(tmp_path, capfd): + """Build will succeed with no warning if there is an empty xbuild-tools definition.""" + if utils.platform != "macos": + pytest.skip("this test can only run on macOS") + if utils.get_xcode_version() < (13, 0): + pytest.skip("this test only works with Xcode 13.0 or greater") + + project_dir = tmp_path / "project" + basic_project = test_projects.new_c_project() + basic_project.files.update(basic_project_files) + basic_project.generate(project_dir) + + # Build, but don't test the wheels; we're only checking that a warning + # wasn't raised. + actual_wheels = utils.cibuildwheel_run( + project_dir, + add_env={ + "CIBW_PLATFORM": "ios", + "CIBW_BUILD": "cp313-*", + "CIBW_TEST_SKIP": "*", + "CIBW_XBUILD_TOOLS": "", + }, + ) + + # The expected wheels were produced. + ios_version = os.getenv("IPHONEOS_DEPLOYMENT_TARGET", "13.0").replace(".", "_") + platform_machine = platform.machine() + + if platform_machine == "x86_64": + expected_wheels = { + f"spam-0.1.0-cp313-cp313-ios_{ios_version}_x86_64_iphonesimulator.whl", + } + + elif platform_machine == "arm64": + expected_wheels = { + f"spam-0.1.0-cp313-cp313-ios_{ios_version}_arm64_iphoneos.whl", + f"spam-0.1.0-cp313-cp313-ios_{ios_version}_arm64_iphonesimulator.whl", + } + + assert set(actual_wheels) == expected_wheels + + # The warnings about cross-build notifications were silenced. + captured = capfd.readouterr() + assert "Your project configuration does not define any cross-build tools." not in captured.err From 1d90d2efe2b14d171720d74cc72f3c6a1d8d6b42 Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Tue, 1 Apr 2025 14:18:42 +0800 Subject: [PATCH 09/12] Correct a bad copy-paste in the schema generator. --- bin/generate_schema.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bin/generate_schema.py b/bin/generate_schema.py index 0d471e4ff..ff75f40e4 100755 --- a/bin/generate_schema.py +++ b/bin/generate_schema.py @@ -42,6 +42,9 @@ before-all: description: Execute a shell command on the build system before any wheels are built. type: string_array + before-build: + description: Execute a shell command preparing each wheel's build. + type: string_array before-test: description: Execute a shell command before testing each wheel. type: string_array @@ -178,7 +181,7 @@ musllinux-x86_64-image: type: string description: Specify alternative manylinux / musllinux container images - before-build: + xbuild-tools: description: Binaries on the path that should be included in an isolated cross-build environment type: string_array repair-wheel-command: From ae60b241b531378fb00fac58f6e1ecd109949e50 Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Tue, 1 Apr 2025 14:19:18 +0800 Subject: [PATCH 10/12] .. and now fix the indentation. --- bin/generate_schema.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/generate_schema.py b/bin/generate_schema.py index ff75f40e4..8ca6a1a76 100755 --- a/bin/generate_schema.py +++ b/bin/generate_schema.py @@ -43,8 +43,8 @@ description: Execute a shell command on the build system before any wheels are built. type: string_array before-build: - description: Execute a shell command preparing each wheel's build. - type: string_array + description: Execute a shell command preparing each wheel's build. + type: string_array before-test: description: Execute a shell command before testing each wheel. type: string_array From 81094d3d82f17124918a33ad60ae5f5f6200e2d6 Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Fri, 4 Apr 2025 14:25:49 +0800 Subject: [PATCH 11/12] Move sentinel handling earlier into the parsing process. --- cibuildwheel/options.py | 11 +++++++++-- cibuildwheel/platforms/ios.py | 13 +++++-------- unit_test/options_test.py | 31 ++++++++++++++++++++++++++++--- unit_test/options_toml_test.py | 6 ++++++ 4 files changed, 48 insertions(+), 13 deletions(-) diff --git a/cibuildwheel/options.py b/cibuildwheel/options.py index a8230e7be..1e07a3f97 100644 --- a/cibuildwheel/options.py +++ b/cibuildwheel/options.py @@ -94,7 +94,7 @@ class BuildOptions: environment: ParsedEnvironment before_all: str before_build: str | None - xbuild_tools: list[str] + xbuild_tools: list[str] | None repair_command: str manylinux_images: dict[str, str] | None musllinux_images: dict[str, str] | None @@ -719,11 +719,18 @@ def _compute_build_options(self, identifier: str | None) -> BuildOptions: test_command = self.reader.get("test-command", option_format=ListFormat(sep=" && ")) before_test = self.reader.get("before-test", option_format=ListFormat(sep=" && ")) - xbuild_tools = shlex.split( + xbuild_tools: list[str] | None = shlex.split( self.reader.get( "xbuild-tools", option_format=ListFormat(sep=" ", quote=shlex.quote) ) ) + # ["\u0000"] is a sentinel value used as a default, because TOML + # doesn't have an explicit NULL value. If xbuild-tools is set to the + # sentinel, it indicates that the user hasn't defined xbuild-tools + # *at all* (not even an `xbuild-tools = []` definition). + if xbuild_tools == ["\u0000"]: + xbuild_tools = None + test_sources = shlex.split( self.reader.get( "test-sources", option_format=ListFormat(sep=" ", quote=shlex.quote) diff --git a/cibuildwheel/platforms/ios.py b/cibuildwheel/platforms/ios.py index 7c525a900..06fce61fa 100644 --- a/cibuildwheel/platforms/ios.py +++ b/cibuildwheel/platforms/ios.py @@ -152,7 +152,7 @@ def cross_virtualenv( build_python: Path, venv_path: Path, dependency_constraint_flags: Sequence[PathOrStr], - xbuild_tools: Sequence[str], + xbuild_tools: Sequence[str] | None, ) -> dict[str, str]: """Create a cross-compilation virtual environment. @@ -219,14 +219,11 @@ def cross_virtualenv( # build. Find their location on the path, and link the underlying binaries # (fully resolving symlinks) to a "safe" location that will *only* contain # those tools. This avoids needing to add *all* of Homebrew to the path just - # to get access to (for example) cmake for build purposes. + # to get access to (for example) cmake for build purposes. A value of None + # means the user hasn't provided a list of xbuild tools. xbuild_tools_path = venv_path / "cibw_xbuild_tools" xbuild_tools_path.mkdir() - # ["\u0000"] is a sentinel value used as a default, because TOML doesn't - # have an explicit NULL value. If xbuild-tools is set to the sentinel, it - # indicates that the user hasn't defined xbuild_tools *at all* (not even an - # `xbuild_tools = []` definition). - if xbuild_tools == ["\u0000"]: + if xbuild_tools is None: log.warning( textwrap.dedent( """ @@ -285,7 +282,7 @@ def setup_python( dependency_constraint_flags: Sequence[PathOrStr], environment: ParsedEnvironment, build_frontend: BuildFrontendName, - xbuild_tools: Sequence[str], + xbuild_tools: Sequence[str] | None, ) -> tuple[Path, dict[str, str]]: if build_frontend == "build[uv]": msg = "uv doesn't support iOS" diff --git a/unit_test/options_test.py b/unit_test/options_test.py index 8d438df7c..08c1dbfb1 100644 --- a/unit_test/options_test.py +++ b/unit_test/options_test.py @@ -22,7 +22,6 @@ build = ["cp38-*", "cp313-*"] skip = ["*musllinux*"] environment = {FOO="BAR"} -xbuild-tools = ["cmake", "rustc"] test-command = "pyproject" test-sources = ["test", "other dir"] @@ -74,14 +73,12 @@ def test_options_1(tmp_path, monkeypatch): pinned_x86_64_container_image = all_pinned_container_images["x86_64"] local = options.build_options("cp38-manylinux_x86_64") - assert local.xbuild_tools == ["cmake", "rustc"] assert local.manylinux_images is not None assert local.test_command == "pyproject" assert local.test_sources == ["test", "other dir"] assert local.manylinux_images["x86_64"] == pinned_x86_64_container_image["manylinux_2_28"] local = options.build_options("cp313-manylinux_x86_64") - assert local.xbuild_tools == ["cmake", "rustc"] assert local.manylinux_images is not None assert local.test_command == "pyproject-override" assert local.test_sources == ["test", "other dir"] @@ -573,3 +570,31 @@ def test_deprecated_image( assert f"{resolved_image!r}" in captured.err else: assert "Deprecated image" not in captured.err + + +@pytest.mark.parametrize( + ("definition", "expected"), + [ + ("", None), + ("xbuild-tools = []", []), + ('xbuild-tools = ["cmake", "rustc"]', ["cmake", "rustc"]), + ], +) +def test_xbuild_tools_handling(tmp_path: Path, definition: str, expected: list[str] | None) -> None: + args = CommandLineArguments.defaults() + args.package_dir = tmp_path + + pyproject_toml: Path = tmp_path / "pyproject.toml" + pyproject_toml.write_text( + textwrap.dedent( + f"""\ + [tool.cibuildwheel] + {definition} + """ + ) + ) + + options = Options(platform="ios", command_line_arguments=args, env={}) + + local = options.build_options("cp313-ios_13_0_arm64_iphoneos") + assert local.xbuild_tools == expected diff --git a/unit_test/options_toml_test.py b/unit_test/options_toml_test.py index 616a56725..6ec634152 100644 --- a/unit_test/options_toml_test.py +++ b/unit_test/options_toml_test.py @@ -17,6 +17,7 @@ [tool.cibuildwheel] build = "cp39*" environment = {THING = "OTHER", FOO="BAR"} +xbuild-tools = ["first"] test-command = "pyproject" test-requires = "something" @@ -274,6 +275,7 @@ def test_environment_override_empty(tmp_path): env={ "CIBW_MANYLINUX_I686_IMAGE": "", "CIBW_MANYLINUX_AARCH64_IMAGE": "manylinux1", + "CIBW_XBUILD_TOOLS": "", }, ) @@ -285,6 +287,10 @@ def test_environment_override_empty(tmp_path): assert options_reader.get("manylinux-i686-image", ignore_empty=True) == "manylinux1" assert options_reader.get("manylinux-aarch64-image", ignore_empty=True) == "manylinux1" + assert ( + options_reader.get("xbuild-tools", option_format=ListFormat(" ", quote=shlex.quote)) == "" + ) + @pytest.mark.parametrize("ignore_empty", [True, False], ids=["ignore_empty", "no_ignore_empty"]) def test_resolve_cascade(ignore_empty): From 7f8ab22246ac6a4d1e63769157f1a99e480b1506 Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Sun, 6 Apr 2025 18:20:28 -0700 Subject: [PATCH 12/12] Remove serialization from tests that won't start a test suite. --- test/test_ios.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/test/test_ios.py b/test/test_ios.py index 09cb1c931..c3cfa61af 100644 --- a/test/test_ios.py +++ b/test/test_ios.py @@ -101,7 +101,6 @@ def test_ios_platforms(tmp_path, build_config, monkeypatch, capfd): assert "'does-exist' will be included in the cross-build environment" in captured.out -@pytest.mark.serial def test_no_test_sources(tmp_path, capfd): """Build will fail if test-sources isn't defined.""" if utils.platform != "macos": @@ -129,7 +128,6 @@ def test_no_test_sources(tmp_path, capfd): assert "Testing on iOS requires a definition of test-sources." in captured.err -@pytest.mark.serial def test_missing_xbuild_tool(tmp_path, capfd): """Build will fail if xbuild-tools references a non-existent tool.""" if utils.platform != "macos": @@ -158,7 +156,6 @@ def test_missing_xbuild_tool(tmp_path, capfd): assert "Could not find a 'does-not-exist' executable on the path." in captured.err -@pytest.mark.serial def test_no_xbuild_tool_definition(tmp_path, capfd): """Build will succeed with a warning if there is no xbuild-tools definition.""" if utils.platform != "macos": @@ -204,7 +201,6 @@ def test_no_xbuild_tool_definition(tmp_path, capfd): assert "Your project configuration does not define any cross-build tools." in captured.err -@pytest.mark.serial def test_empty_xbuild_tool_definition(tmp_path, capfd): """Build will succeed with no warning if there is an empty xbuild-tools definition.""" if utils.platform != "macos":