From c341c7fdeba963917110d6bdafcb92728f8edfc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Sat, 25 May 2024 16:23:55 +0300 Subject: [PATCH] Added deprecation warnings to all modules not previously prefixed with an underscore This was done to discourage anyone from importing `wheel`. --- .pre-commit-config.yaml | 2 +- README.rst | 22 +- docs/news.rst | 1 + pyproject.toml | 6 +- src/wheel/__main__.py | 4 +- src/wheel/_bdist_wheel.py | 604 +++++++++++++++++ src/wheel/{cli => _cli}/__init__.py | 8 +- .../{cli/convert.py => _cli/_convert.py} | 4 +- src/wheel/{cli/pack.py => _cli/_pack.py} | 4 +- src/wheel/{cli/tags.py => _cli/_tags.py} | 2 +- src/wheel/{cli/unpack.py => _cli/_unpack.py} | 2 +- src/wheel/_macosx_libfile.py | 482 ++++++++++++++ src/wheel/_metadata.py | 183 ++++++ src/wheel/{util.py => _util.py} | 0 src/wheel/{vendored => _vendored}/__init__.py | 0 .../packaging/__init__.py | 0 .../packaging/_elffile.py | 0 .../packaging/_manylinux.py | 0 .../packaging/_musllinux.py | 0 .../packaging/_parser.py | 0 .../packaging/_structures.py | 0 .../packaging/_tokenizer.py | 0 .../packaging/markers.py | 0 .../packaging/requirements.py | 0 .../packaging/specifiers.py | 0 .../{vendored => _vendored}/packaging/tags.py | 0 .../packaging/utils.py | 0 .../packaging/version.py | 0 src/wheel/{vendored => _vendored}/vendor.txt | 0 src/wheel/_wheelfile.py | 227 +++++++ src/wheel/bdist_wheel.py | 611 +----------------- src/wheel/macosx_libfile.py | 495 +------------- src/wheel/metadata.py | 197 +----- src/wheel/wheelfile.py | 238 +------ tests/{cli => _cli}/eggnames.txt | 0 tests/{cli => _cli}/test_convert.py | 4 +- tests/{cli => _cli}/test_pack.py | 2 +- tests/{cli => _cli}/test_tags.py | 6 +- tests/{cli => _cli}/test_unpack.py | 4 +- tests/test_bdist_wheel.py | 8 +- tests/test_macosx_libfile.py | 4 +- tests/test_metadata.py | 2 +- tests/test_wheelfile.py | 4 +- 43 files changed, 1591 insertions(+), 1535 deletions(-) create mode 100644 src/wheel/_bdist_wheel.py rename src/wheel/{cli => _cli}/__init__.py (97%) rename src/wheel/{cli/convert.py => _cli/_convert.py} (99%) rename src/wheel/{cli/pack.py => _cli/_pack.py} (97%) rename src/wheel/{cli/tags.py => _cli/_tags.py} (99%) rename src/wheel/{cli/unpack.py => _cli/_unpack.py} (96%) create mode 100644 src/wheel/_macosx_libfile.py create mode 100644 src/wheel/_metadata.py rename src/wheel/{util.py => _util.py} (100%) rename src/wheel/{vendored => _vendored}/__init__.py (100%) rename src/wheel/{vendored => _vendored}/packaging/__init__.py (100%) rename src/wheel/{vendored => _vendored}/packaging/_elffile.py (100%) rename src/wheel/{vendored => _vendored}/packaging/_manylinux.py (100%) rename src/wheel/{vendored => _vendored}/packaging/_musllinux.py (100%) rename src/wheel/{vendored => _vendored}/packaging/_parser.py (100%) rename src/wheel/{vendored => _vendored}/packaging/_structures.py (100%) rename src/wheel/{vendored => _vendored}/packaging/_tokenizer.py (100%) rename src/wheel/{vendored => _vendored}/packaging/markers.py (100%) rename src/wheel/{vendored => _vendored}/packaging/requirements.py (100%) rename src/wheel/{vendored => _vendored}/packaging/specifiers.py (100%) rename src/wheel/{vendored => _vendored}/packaging/tags.py (100%) rename src/wheel/{vendored => _vendored}/packaging/utils.py (100%) rename src/wheel/{vendored => _vendored}/packaging/version.py (100%) rename src/wheel/{vendored => _vendored}/vendor.txt (100%) create mode 100644 src/wheel/_wheelfile.py rename tests/{cli => _cli}/eggnames.txt (100%) rename tests/{cli => _cli}/test_convert.py (89%) rename tests/{cli => _cli}/test_pack.py (98%) rename tests/{cli => _cli}/test_tags.py (98%) rename tests/{cli => _cli}/test_unpack.py (92%) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 84c68b377..22fef3105 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,4 +1,4 @@ -exclude: ^src/wheel/vendored +exclude: ^src/wheel/_vendored repos: - repo: https://github.com/pre-commit/pre-commit-hooks diff --git a/README.rst b/README.rst index 1cf194ec2..d7a796485 100644 --- a/README.rst +++ b/README.rst @@ -1,19 +1,25 @@ wheel ===== -This library is the reference implementation of the Python wheel packaging -standard, as defined in `PEP 427`_. +This is a command line tool for manipulating Python wheel files, as defined in +`PEP 427`_. It contains the following functionality: -It has two different roles: +* Convert ``.egg`` archives into ``.whl`` +* Unpack wheel archives +* Repack wheel archives +* Add or remove tags in existing wheel archives -#. A setuptools_ extension for building wheels that provides the - ``bdist_wheel`` setuptools command -#. A command line tool for working with wheel files +Historical note +--------------- -It should be noted that wheel is **not** intended to be used as a library, and -as such there is no stable, public API. +This library used to be the reference implementation of the Python wheel packaging +standard, and a setuptools_ extension containing the ``bdist_wheel`` command. The wheel +file processing functionality has since been moved to the packaging_ library, and the +``bdist_wheel`` command has been merged into setuptools itself, leaving this project to +only contain the command line interface. .. _PEP 427: https://www.python.org/dev/peps/pep-0427/ +.. _packaging: https://pypi.org/project/packaging/ .. _setuptools: https://pypi.org/project/setuptools/ Documentation diff --git a/docs/news.rst b/docs/news.rst index 75da05bac..7b03c9d2f 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -4,6 +4,7 @@ Release Notes **UNRELEASED** - Canonicalize requirements in METADATA file (PR by Wim Jeantine-Glenn) +- Added deprecation warnings to all modules not previously prefixed with an underscore **0.43.0 (2024-03-11)** diff --git a/pyproject.toml b/pyproject.toml index 1f7784eb7..936463a41 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,7 +32,7 @@ Changelog = "https://wheel.readthedocs.io/en/stable/news.html" Source = "https://github.com/pypa/wheel" [project.scripts] -wheel = "wheel.cli:main" +wheel = "wheel._cli:main" [project.entry-points."distutils.commands"] bdist_wheel = "wheel.bdist_wheel:bdist_wheel" @@ -70,7 +70,7 @@ exclude = [ [tool.black] extend-exclude = ''' -^/src/wheel/vendored/ +^/src/wheel/_vendored/ ''' [tool.pytest.ini_options] @@ -86,7 +86,7 @@ testpaths = ["test"] [tool.coverage.run] source = ["wheel"] -omit = ["*/vendored/*"] +omit = ["*/_vendored/*"] [tool.coverage.report] show_missing = true diff --git a/src/wheel/__main__.py b/src/wheel/__main__.py index 0be745374..8284ed86b 100644 --- a/src/wheel/__main__.py +++ b/src/wheel/__main__.py @@ -14,9 +14,9 @@ def main(): # needed for console script path = os.path.dirname(os.path.dirname(__file__)) sys.path[0:0] = [path] - import wheel.cli + from wheel import _cli - sys.exit(wheel.cli.main()) + sys.exit(_cli.main()) if __name__ == "__main__": diff --git a/src/wheel/_bdist_wheel.py b/src/wheel/_bdist_wheel.py new file mode 100644 index 000000000..f0601c1a0 --- /dev/null +++ b/src/wheel/_bdist_wheel.py @@ -0,0 +1,604 @@ +""" +Create a wheel (.whl) distribution. + +A wheel is a built archive format. +""" + +from __future__ import annotations + +import os +import re +import shutil +import stat +import struct +import sys +import sysconfig +import warnings +from email.generator import BytesGenerator, Generator +from email.policy import EmailPolicy +from glob import iglob +from shutil import rmtree +from typing import TYPE_CHECKING, Callable, Iterable, Literal, Sequence, cast +from zipfile import ZIP_DEFLATED, ZIP_STORED + +import setuptools +from setuptools import Command + +from . import __version__ as wheel_version +from ._metadata import pkginfo_to_metadata +from ._util import log +from ._vendored.packaging import tags +from ._vendored.packaging import version as _packaging_version +from ._wheelfile import WheelFile + +if TYPE_CHECKING: + import types + + +def safe_name(name: str) -> str: + """Convert an arbitrary string to a standard distribution name + Any runs of non-alphanumeric/. characters are replaced with a single '-'. + """ + return re.sub("[^A-Za-z0-9.]+", "-", name) + + +def safe_version(version: str) -> str: + """ + Convert an arbitrary string to a standard version string + """ + try: + # normalize the version + return str(_packaging_version.Version(version)) + except _packaging_version.InvalidVersion: + version = version.replace(" ", ".") + return re.sub("[^A-Za-z0-9.]+", "-", version) + + +setuptools_major_version = int(setuptools.__version__.split(".")[0]) + +PY_LIMITED_API_PATTERN = r"cp3\d" + + +def _is_32bit_interpreter() -> bool: + return struct.calcsize("P") == 4 + + +def python_tag() -> str: + return f"py{sys.version_info[0]}" + + +def get_platform(archive_root: str | None) -> str: + """Return our platform name 'win32', 'linux_x86_64'""" + result = sysconfig.get_platform() + if result.startswith("macosx") and archive_root is not None: + from ._macosx_libfile import calculate_macosx_platform_tag + + result = calculate_macosx_platform_tag(archive_root, result) + elif _is_32bit_interpreter(): + if result == "linux-x86_64": + # pip pull request #3497 + result = "linux-i686" + elif result == "linux-aarch64": + # packaging pull request #234 + # TODO armv8l, packaging pull request #690 => this did not land + # in pip/packaging yet + result = "linux-armv7l" + + return result.replace("-", "_") + + +def get_flag( + var: str, fallback: bool, expected: bool = True, warn: bool = True +) -> bool: + """Use a fallback value for determining SOABI flags if the needed config + var is unset or unavailable.""" + val = sysconfig.get_config_var(var) + if val is None: + if warn: + warnings.warn( + f"Config variable '{var}' is unset, Python ABI tag may be incorrect", + RuntimeWarning, + stacklevel=2, + ) + return fallback + return val == expected + + +def get_abi_tag() -> str | None: + """Return the ABI tag based on SOABI (if available) or emulate SOABI (PyPy2).""" + soabi: str = sysconfig.get_config_var("SOABI") + impl = tags.interpreter_name() + if not soabi and impl in ("cp", "pp") and hasattr(sys, "maxunicode"): + d = "" + m = "" + u = "" + if get_flag("Py_DEBUG", hasattr(sys, "gettotalrefcount"), warn=(impl == "cp")): + d = "d" + + if get_flag( + "WITH_PYMALLOC", + impl == "cp", + warn=(impl == "cp" and sys.version_info < (3, 8)), + ) and sys.version_info < (3, 8): + m = "m" + + abi = f"{impl}{tags.interpreter_version()}{d}{m}{u}" + elif soabi and impl == "cp" and soabi.startswith("cpython"): + # non-Windows + abi = "cp" + soabi.split("-")[1] + elif soabi and impl == "cp" and soabi.startswith("cp"): + # Windows + abi = soabi.split("-")[0] + elif soabi and impl == "pp": + # we want something like pypy36-pp73 + abi = "-".join(soabi.split("-")[:2]) + abi = abi.replace(".", "_").replace("-", "_") + elif soabi and impl == "graalpy": + abi = "-".join(soabi.split("-")[:3]) + abi = abi.replace(".", "_").replace("-", "_") + elif soabi: + abi = soabi.replace(".", "_").replace("-", "_") + else: + abi = None + + return abi + + +def safer_name(name: str) -> str: + return safe_name(name).replace("-", "_") + + +def safer_version(version: str) -> str: + return safe_version(version).replace("-", "_") + + +def remove_readonly( + func: Callable[..., object], + path: str, + excinfo: tuple[type[Exception], Exception, types.TracebackType], +) -> None: + remove_readonly_exc(func, path, excinfo[1]) + + +def remove_readonly_exc(func: Callable[..., object], path: str, exc: Exception) -> None: + os.chmod(path, stat.S_IWRITE) + func(path) + + +class bdist_wheel(Command): + description = "create a wheel distribution" + + supported_compressions = { + "stored": ZIP_STORED, + "deflated": ZIP_DEFLATED, + } + + user_options = [ + ("bdist-dir=", "b", "temporary directory for creating the distribution"), + ( + "plat-name=", + "p", + "platform name to embed in generated filenames " + f"(default: {get_platform(None)})", + ), + ( + "keep-temp", + "k", + "keep the pseudo-installation tree around after " + "creating the distribution archive", + ), + ("dist-dir=", "d", "directory to put final built distributions in"), + ("skip-build", None, "skip rebuilding everything (for testing/debugging)"), + ( + "relative", + None, + "build the archive using relative paths (default: false)", + ), + ( + "owner=", + "u", + "Owner name used when creating a tar file [default: current user]", + ), + ( + "group=", + "g", + "Group name used when creating a tar file [default: current group]", + ), + ("universal", None, "make a universal wheel (default: false)"), + ( + "compression=", + None, + "zipfile compression (one of: {}) (default: 'deflated')".format( + ", ".join(supported_compressions) + ), + ), + ( + "python-tag=", + None, + f"Python implementation compatibility tag (default: '{python_tag()}')", + ), + ( + "build-number=", + None, + "Build number for this particular version. " + "As specified in PEP-0427, this must start with a digit. " + "[default: None]", + ), + ( + "py-limited-api=", + None, + "Python tag (cp32|cp33|cpNN) for abi3 wheel tag (default: false)", + ), + ] + + boolean_options = ["keep-temp", "skip-build", "relative", "universal"] + + def initialize_options(self): + self.bdist_dir: str = None + self.data_dir = None + self.plat_name: str | None = None + self.plat_tag = None + self.format = "zip" + self.keep_temp = False + self.dist_dir: str | None = None + self.egginfo_dir = None + self.root_is_pure: bool | None = None + self.skip_build = None + self.relative = False + self.owner = None + self.group = None + self.universal: bool = False + self.compression: str | int = "deflated" + self.python_tag: str = python_tag() + self.build_number: str | None = None + self.py_limited_api: str | Literal[False] = False + self.plat_name_supplied = False + + def finalize_options(self): + if self.bdist_dir is None: + bdist_base = self.get_finalized_command("bdist").bdist_base + self.bdist_dir = os.path.join(bdist_base, "wheel") + + egg_info = self.distribution.get_command_obj("egg_info") + egg_info.ensure_finalized() # needed for correct `wheel_dist_name` + + self.data_dir = self.wheel_dist_name + ".data" + self.plat_name_supplied = self.plat_name is not None + + try: + self.compression = self.supported_compressions[self.compression] + except KeyError: + raise ValueError(f"Unsupported compression: {self.compression}") from None + + need_options = ("dist_dir", "plat_name", "skip_build") + + self.set_undefined_options("bdist", *zip(need_options, need_options)) + + self.root_is_pure = not ( + self.distribution.has_ext_modules() or self.distribution.has_c_libraries() + ) + + if self.py_limited_api and not re.match( + PY_LIMITED_API_PATTERN, self.py_limited_api + ): + raise ValueError(f"py-limited-api must match '{PY_LIMITED_API_PATTERN}'") + + # Support legacy [wheel] section for setting universal + wheel = self.distribution.get_option_dict("wheel") + if "universal" in wheel: + # please don't define this in your global configs + log.warning( + "The [wheel] section is deprecated. Use [bdist_wheel] instead.", + ) + val = wheel["universal"][1].strip() + if val.lower() in ("1", "true", "yes"): + self.universal = True + + if self.build_number is not None and not self.build_number[:1].isdigit(): + raise ValueError("Build tag (build-number) must start with a digit.") + + @property + def wheel_dist_name(self): + """Return distribution full name with - replaced with _""" + components = ( + safer_name(self.distribution.get_name()), + safer_version(self.distribution.get_version()), + ) + if self.build_number: + components += (self.build_number,) + return "-".join(components) + + def get_tag(self) -> tuple[str, str, str]: + # bdist sets self.plat_name if unset, we should only use it for purepy + # wheels if the user supplied it. + if self.plat_name_supplied: + plat_name = cast(str, self.plat_name) + elif self.root_is_pure: + plat_name = "any" + else: + # macosx contains system version in platform name so need special handle + if self.plat_name and not self.plat_name.startswith("macosx"): + plat_name = self.plat_name + else: + # on macosx always limit the platform name to comply with any + # c-extension modules in bdist_dir, since the user can specify + # a higher MACOSX_DEPLOYMENT_TARGET via tools like CMake + + # on other platforms, and on macosx if there are no c-extension + # modules, use the default platform name. + plat_name = get_platform(self.bdist_dir) + + if _is_32bit_interpreter(): + if plat_name in ("linux-x86_64", "linux_x86_64"): + plat_name = "linux_i686" + if plat_name in ("linux-aarch64", "linux_aarch64"): + # TODO armv8l, packaging pull request #690 => this did not land + # in pip/packaging yet + plat_name = "linux_armv7l" + + plat_name = ( + plat_name.lower().replace("-", "_").replace(".", "_").replace(" ", "_") + ) + + if self.root_is_pure: + if self.universal: + impl = "py2.py3" + else: + impl = self.python_tag + tag = (impl, "none", plat_name) + else: + impl_name = tags.interpreter_name() + impl_ver = tags.interpreter_version() + impl = impl_name + impl_ver + # We don't work on CPython 3.1, 3.0. + if self.py_limited_api and (impl_name + impl_ver).startswith("cp3"): + impl = self.py_limited_api + abi_tag = "abi3" + else: + abi_tag = str(get_abi_tag()).lower() + tag = (impl, abi_tag, plat_name) + # issue gh-374: allow overriding plat_name + supported_tags = [ + (t.interpreter, t.abi, plat_name) for t in tags.sys_tags() + ] + assert ( + tag in supported_tags + ), f"would build wheel with unsupported tag {tag}" + return tag + + def run(self): + build_scripts = self.reinitialize_command("build_scripts") + build_scripts.executable = "python" + build_scripts.force = True + + build_ext = self.reinitialize_command("build_ext") + build_ext.inplace = False + + if not self.skip_build: + self.run_command("build") + + install = self.reinitialize_command("install", reinit_subcommands=True) + install.root = self.bdist_dir + install.compile = False + install.skip_build = self.skip_build + install.warn_dir = False + + # A wheel without setuptools scripts is more cross-platform. + # Use the (undocumented) `no_ep` option to setuptools' + # install_scripts command to avoid creating entry point scripts. + install_scripts = self.reinitialize_command("install_scripts") + install_scripts.no_ep = True + + # Use a custom scheme for the archive, because we have to decide + # at installation time which scheme to use. + for key in ("headers", "scripts", "data", "purelib", "platlib"): + setattr(install, "install_" + key, os.path.join(self.data_dir, key)) + + basedir_observed = "" + + if os.name == "nt": + # win32 barfs if any of these are ''; could be '.'? + # (distutils.command.install:change_roots bug) + basedir_observed = os.path.normpath(os.path.join(self.data_dir, "..")) + self.install_libbase = self.install_lib = basedir_observed + + setattr( + install, + "install_purelib" if self.root_is_pure else "install_platlib", + basedir_observed, + ) + + log.info(f"installing to {self.bdist_dir}") + + self.run_command("install") + + impl_tag, abi_tag, plat_tag = self.get_tag() + archive_basename = f"{self.wheel_dist_name}-{impl_tag}-{abi_tag}-{plat_tag}" + if not self.relative: + archive_root = self.bdist_dir + else: + archive_root = os.path.join( + self.bdist_dir, self._ensure_relative(install.install_base) + ) + + self.set_undefined_options("install_egg_info", ("target", "egginfo_dir")) + distinfo_dirname = ( + f"{safer_name(self.distribution.get_name())}-" + f"{safer_version(self.distribution.get_version())}.dist-info" + ) + distinfo_dir = os.path.join(self.bdist_dir, distinfo_dirname) + self.egg2dist(self.egginfo_dir, distinfo_dir) + + self.write_wheelfile(distinfo_dir) + + # Make the archive + if not os.path.exists(self.dist_dir): + os.makedirs(self.dist_dir) + + wheel_path = os.path.join(self.dist_dir, archive_basename + ".whl") + with WheelFile(wheel_path, "w", self.compression) as wf: + wf.write_files(archive_root) + + # Add to 'Distribution.dist_files' so that the "upload" command works + getattr(self.distribution, "dist_files", []).append( + ( + "bdist_wheel", + "{}.{}".format(*sys.version_info[:2]), # like 3.7 + wheel_path, + ) + ) + + if not self.keep_temp: + log.info(f"removing {self.bdist_dir}") + if not self.dry_run: + if sys.version_info < (3, 12): + rmtree(self.bdist_dir, onerror=remove_readonly) + else: + rmtree(self.bdist_dir, onexc=remove_readonly_exc) + + def write_wheelfile( + self, wheelfile_base: str, generator: str = f"bdist_wheel ({wheel_version})" + ): + from email.message import Message + + msg = Message() + msg["Wheel-Version"] = "1.0" # of the spec + msg["Generator"] = generator + msg["Root-Is-Purelib"] = str(self.root_is_pure).lower() + if self.build_number is not None: + msg["Build"] = self.build_number + + # Doesn't work for bdist_wininst + impl_tag, abi_tag, plat_tag = self.get_tag() + for impl in impl_tag.split("."): + for abi in abi_tag.split("."): + for plat in plat_tag.split("."): + msg["Tag"] = "-".join((impl, abi, plat)) + + wheelfile_path = os.path.join(wheelfile_base, "WHEEL") + log.info(f"creating {wheelfile_path}") + with open(wheelfile_path, "wb") as f: + BytesGenerator(f, maxheaderlen=0).flatten(msg) + + def _ensure_relative(self, path: str) -> str: + # copied from dir_util, deleted + drive, path = os.path.splitdrive(path) + if path[0:1] == os.sep: + path = drive + path[1:] + return path + + @property + def license_paths(self) -> Iterable[str]: + if setuptools_major_version >= 57: + # Setuptools has resolved any patterns to actual file names + return self.distribution.metadata.license_files or () + + files: set[str] = set() + metadata = self.distribution.get_option_dict("metadata") + if setuptools_major_version >= 42: + # Setuptools recognizes the license_files option but does not do globbing + patterns = cast(Sequence[str], self.distribution.metadata.license_files) + else: + # Prior to those, wheel is entirely responsible for handling license files + if "license_files" in metadata: + patterns = metadata["license_files"][1].split() + else: + patterns = () + + if "license_file" in metadata: + warnings.warn( + 'The "license_file" option is deprecated. Use "license_files" instead.', + DeprecationWarning, + stacklevel=2, + ) + files.add(metadata["license_file"][1]) + + if not files and not patterns and not isinstance(patterns, list): + patterns = ("LICEN[CS]E*", "COPYING*", "NOTICE*", "AUTHORS*") + + for pattern in patterns: + for path in iglob(pattern): + if path.endswith("~"): + log.debug( + f'ignoring license file "{path}" as it looks like a backup' + ) + continue + + if path not in files and os.path.isfile(path): + log.info( + f'adding license file "{path}" (matched pattern "{pattern}")' + ) + files.add(path) + + return files + + def egg2dist(self, egginfo_path: str, distinfo_path: str): + """Convert an .egg-info directory into a .dist-info directory""" + + def adios(p: str) -> None: + """Appropriately delete directory, file or link.""" + if os.path.exists(p) and not os.path.islink(p) and os.path.isdir(p): + shutil.rmtree(p) + elif os.path.exists(p): + os.unlink(p) + + adios(distinfo_path) + + if not os.path.exists(egginfo_path): + # There is no egg-info. This is probably because the egg-info + # file/directory is not named matching the distribution name used + # to name the archive file. Check for this case and report + # accordingly. + import glob + + pat = os.path.join(os.path.dirname(egginfo_path), "*.egg-info") + possible = glob.glob(pat) + err = f"Egg metadata expected at {egginfo_path} but not found" + if possible: + alt = os.path.basename(possible[0]) + err += f" ({alt} found - possible misnamed archive file?)" + + raise ValueError(err) + + if os.path.isfile(egginfo_path): + # .egg-info is a single file + pkg_info = pkginfo_to_metadata(egginfo_path, egginfo_path) + os.mkdir(distinfo_path) + else: + # .egg-info is a directory + pkginfo_path = os.path.join(egginfo_path, "PKG-INFO") + pkg_info = pkginfo_to_metadata(egginfo_path, pkginfo_path) + + # ignore common egg metadata that is useless to wheel + shutil.copytree( + egginfo_path, + distinfo_path, + ignore=lambda x, y: { + "PKG-INFO", + "requires.txt", + "SOURCES.txt", + "not-zip-safe", + }, + ) + + # delete dependency_links if it is only whitespace + dependency_links_path = os.path.join(distinfo_path, "dependency_links.txt") + with open(dependency_links_path, encoding="utf-8") as dependency_links_file: + dependency_links = dependency_links_file.read().strip() + if not dependency_links: + adios(dependency_links_path) + + pkg_info_path = os.path.join(distinfo_path, "METADATA") + serialization_policy = EmailPolicy( + utf8=True, + mangle_from_=False, + max_line_length=0, + ) + with open(pkg_info_path, "w", encoding="utf-8") as out: + Generator(out, policy=serialization_policy).flatten(pkg_info) + + for license_path in self.license_paths: + filename = os.path.basename(license_path) + shutil.copy(license_path, os.path.join(distinfo_path, filename)) + + adios(egginfo_path) diff --git a/src/wheel/cli/__init__.py b/src/wheel/_cli/__init__.py similarity index 97% rename from src/wheel/cli/__init__.py rename to src/wheel/_cli/__init__.py index 6ba1217f5..ef051a808 100644 --- a/src/wheel/cli/__init__.py +++ b/src/wheel/_cli/__init__.py @@ -15,25 +15,25 @@ class WheelError(Exception): def unpack_f(args: argparse.Namespace) -> None: - from .unpack import unpack + from ._unpack import unpack unpack(args.wheelfile, args.dest) def pack_f(args: argparse.Namespace) -> None: - from .pack import pack + from ._pack import pack pack(args.directory, args.dest_dir, args.build_number) def convert_f(args: argparse.Namespace) -> None: - from .convert import convert + from ._convert import convert convert(args.files, args.dest_dir, args.verbose) def tags_f(args: argparse.Namespace) -> None: - from .tags import tags + from ._tags import tags names = ( tags( diff --git a/src/wheel/cli/convert.py b/src/wheel/_cli/_convert.py similarity index 99% rename from src/wheel/cli/convert.py rename to src/wheel/_cli/_convert.py index 46fdf1eb3..3ec294c1a 100644 --- a/src/wheel/cli/convert.py +++ b/src/wheel/_cli/_convert.py @@ -7,8 +7,8 @@ import zipfile from glob import iglob -from ..bdist_wheel import bdist_wheel -from ..wheelfile import WheelFile +from .._bdist_wheel import bdist_wheel +from .._wheelfile import WheelFile from . import WheelError try: diff --git a/src/wheel/cli/pack.py b/src/wheel/_cli/_pack.py similarity index 97% rename from src/wheel/cli/pack.py rename to src/wheel/_cli/_pack.py index 64469c0c7..e75f6e03d 100644 --- a/src/wheel/cli/pack.py +++ b/src/wheel/_cli/_pack.py @@ -6,8 +6,8 @@ from email.generator import BytesGenerator from email.parser import BytesParser -from wheel.cli import WheelError -from wheel.wheelfile import WheelFile +from .._wheelfile import WheelFile +from . import WheelError DIST_INFO_RE = re.compile(r"^(?P(?P.+?)-(?P\d.*?))\.dist-info$") diff --git a/src/wheel/cli/tags.py b/src/wheel/_cli/_tags.py similarity index 99% rename from src/wheel/cli/tags.py rename to src/wheel/_cli/_tags.py index 88da72e9e..aba1955cb 100644 --- a/src/wheel/cli/tags.py +++ b/src/wheel/_cli/_tags.py @@ -6,7 +6,7 @@ from collections.abc import Iterable from email.parser import BytesParser -from ..wheelfile import WheelFile +from .._wheelfile import WheelFile def _compute_tags(original_tags: Iterable[str], new_tags: str | None) -> set[str]: diff --git a/src/wheel/cli/unpack.py b/src/wheel/_cli/_unpack.py similarity index 96% rename from src/wheel/cli/unpack.py rename to src/wheel/_cli/_unpack.py index d48840e6e..117550abb 100644 --- a/src/wheel/cli/unpack.py +++ b/src/wheel/_cli/_unpack.py @@ -2,7 +2,7 @@ from pathlib import Path -from ..wheelfile import WheelFile +from .._wheelfile import WheelFile def unpack(path: str, dest: str = ".") -> None: diff --git a/src/wheel/_macosx_libfile.py b/src/wheel/_macosx_libfile.py new file mode 100644 index 000000000..abdfc9eda --- /dev/null +++ b/src/wheel/_macosx_libfile.py @@ -0,0 +1,482 @@ +""" +This module contains function to analyse dynamic library +headers to extract system information + +Currently only for MacOSX + +Library file on macosx system starts with Mach-O or Fat field. +This can be distinguish by first 32 bites and it is called magic number. +Proper value of magic number is with suffix _MAGIC. Suffix _CIGAM means +reversed bytes order. +Both fields can occur in two types: 32 and 64 bytes. + +FAT field inform that this library contains few version of library +(typically for different types version). It contains +information where Mach-O headers starts. + +Each section started with Mach-O header contains one library +(So if file starts with this field it contains only one version). + +After filed Mach-O there are section fields. +Each of them starts with two fields: +cmd - magic number for this command +cmdsize - total size occupied by this section information. + +In this case only sections LC_VERSION_MIN_MACOSX (for macosx 10.13 and earlier) +and LC_BUILD_VERSION (for macosx 10.14 and newer) are interesting, +because them contains information about minimal system version. + +Important remarks: +- For fat files this implementation looks for maximum number version. + It not check if it is 32 or 64 and do not compare it with currently built package. + So it is possible to false report higher version that needed. +- All structures signatures are taken form macosx header files. +- I think that binary format will be more stable than `otool` output. + and if apple introduce some changes both implementation will need to be updated. +- The system compile will set the deployment target no lower than + 11.0 for arm64 builds. For "Universal 2" builds use the x86_64 deployment + target when the arm64 target is 11.0. +""" + +from __future__ import annotations + +import ctypes +import os +import sys +from io import BufferedIOBase +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Union + + StrPath = Union[str, os.PathLike[str]] + +"""here the needed const and struct from mach-o header files""" + +FAT_MAGIC = 0xCAFEBABE +FAT_CIGAM = 0xBEBAFECA +FAT_MAGIC_64 = 0xCAFEBABF +FAT_CIGAM_64 = 0xBFBAFECA +MH_MAGIC = 0xFEEDFACE +MH_CIGAM = 0xCEFAEDFE +MH_MAGIC_64 = 0xFEEDFACF +MH_CIGAM_64 = 0xCFFAEDFE + +LC_VERSION_MIN_MACOSX = 0x24 +LC_BUILD_VERSION = 0x32 + +CPU_TYPE_ARM64 = 0x0100000C + +mach_header_fields = [ + ("magic", ctypes.c_uint32), + ("cputype", ctypes.c_int), + ("cpusubtype", ctypes.c_int), + ("filetype", ctypes.c_uint32), + ("ncmds", ctypes.c_uint32), + ("sizeofcmds", ctypes.c_uint32), + ("flags", ctypes.c_uint32), +] +""" +struct mach_header { + uint32_t magic; /* mach magic number identifier */ + cpu_type_t cputype; /* cpu specifier */ + cpu_subtype_t cpusubtype; /* machine specifier */ + uint32_t filetype; /* type of file */ + uint32_t ncmds; /* number of load commands */ + uint32_t sizeofcmds; /* the size of all the load commands */ + uint32_t flags; /* flags */ +}; +typedef integer_t cpu_type_t; +typedef integer_t cpu_subtype_t; +""" + +mach_header_fields_64 = mach_header_fields + [("reserved", ctypes.c_uint32)] +""" +struct mach_header_64 { + uint32_t magic; /* mach magic number identifier */ + cpu_type_t cputype; /* cpu specifier */ + cpu_subtype_t cpusubtype; /* machine specifier */ + uint32_t filetype; /* type of file */ + uint32_t ncmds; /* number of load commands */ + uint32_t sizeofcmds; /* the size of all the load commands */ + uint32_t flags; /* flags */ + uint32_t reserved; /* reserved */ +}; +""" + +fat_header_fields = [("magic", ctypes.c_uint32), ("nfat_arch", ctypes.c_uint32)] +""" +struct fat_header { + uint32_t magic; /* FAT_MAGIC or FAT_MAGIC_64 */ + uint32_t nfat_arch; /* number of structs that follow */ +}; +""" + +fat_arch_fields = [ + ("cputype", ctypes.c_int), + ("cpusubtype", ctypes.c_int), + ("offset", ctypes.c_uint32), + ("size", ctypes.c_uint32), + ("align", ctypes.c_uint32), +] +""" +struct fat_arch { + cpu_type_t cputype; /* cpu specifier (int) */ + cpu_subtype_t cpusubtype; /* machine specifier (int) */ + uint32_t offset; /* file offset to this object file */ + uint32_t size; /* size of this object file */ + uint32_t align; /* alignment as a power of 2 */ +}; +""" + +fat_arch_64_fields = [ + ("cputype", ctypes.c_int), + ("cpusubtype", ctypes.c_int), + ("offset", ctypes.c_uint64), + ("size", ctypes.c_uint64), + ("align", ctypes.c_uint32), + ("reserved", ctypes.c_uint32), +] +""" +struct fat_arch_64 { + cpu_type_t cputype; /* cpu specifier (int) */ + cpu_subtype_t cpusubtype; /* machine specifier (int) */ + uint64_t offset; /* file offset to this object file */ + uint64_t size; /* size of this object file */ + uint32_t align; /* alignment as a power of 2 */ + uint32_t reserved; /* reserved */ +}; +""" + +segment_base_fields = [("cmd", ctypes.c_uint32), ("cmdsize", ctypes.c_uint32)] +"""base for reading segment info""" + +segment_command_fields = [ + ("cmd", ctypes.c_uint32), + ("cmdsize", ctypes.c_uint32), + ("segname", ctypes.c_char * 16), + ("vmaddr", ctypes.c_uint32), + ("vmsize", ctypes.c_uint32), + ("fileoff", ctypes.c_uint32), + ("filesize", ctypes.c_uint32), + ("maxprot", ctypes.c_int), + ("initprot", ctypes.c_int), + ("nsects", ctypes.c_uint32), + ("flags", ctypes.c_uint32), +] +""" +struct segment_command { /* for 32-bit architectures */ + uint32_t cmd; /* LC_SEGMENT */ + uint32_t cmdsize; /* includes sizeof section structs */ + char segname[16]; /* segment name */ + uint32_t vmaddr; /* memory address of this segment */ + uint32_t vmsize; /* memory size of this segment */ + uint32_t fileoff; /* file offset of this segment */ + uint32_t filesize; /* amount to map from the file */ + vm_prot_t maxprot; /* maximum VM protection */ + vm_prot_t initprot; /* initial VM protection */ + uint32_t nsects; /* number of sections in segment */ + uint32_t flags; /* flags */ +}; +typedef int vm_prot_t; +""" + +segment_command_fields_64 = [ + ("cmd", ctypes.c_uint32), + ("cmdsize", ctypes.c_uint32), + ("segname", ctypes.c_char * 16), + ("vmaddr", ctypes.c_uint64), + ("vmsize", ctypes.c_uint64), + ("fileoff", ctypes.c_uint64), + ("filesize", ctypes.c_uint64), + ("maxprot", ctypes.c_int), + ("initprot", ctypes.c_int), + ("nsects", ctypes.c_uint32), + ("flags", ctypes.c_uint32), +] +""" +struct segment_command_64 { /* for 64-bit architectures */ + uint32_t cmd; /* LC_SEGMENT_64 */ + uint32_t cmdsize; /* includes sizeof section_64 structs */ + char segname[16]; /* segment name */ + uint64_t vmaddr; /* memory address of this segment */ + uint64_t vmsize; /* memory size of this segment */ + uint64_t fileoff; /* file offset of this segment */ + uint64_t filesize; /* amount to map from the file */ + vm_prot_t maxprot; /* maximum VM protection */ + vm_prot_t initprot; /* initial VM protection */ + uint32_t nsects; /* number of sections in segment */ + uint32_t flags; /* flags */ +}; +""" + +version_min_command_fields = segment_base_fields + [ + ("version", ctypes.c_uint32), + ("sdk", ctypes.c_uint32), +] +""" +struct version_min_command { + uint32_t cmd; /* LC_VERSION_MIN_MACOSX or + LC_VERSION_MIN_IPHONEOS or + LC_VERSION_MIN_WATCHOS or + LC_VERSION_MIN_TVOS */ + uint32_t cmdsize; /* sizeof(struct min_version_command) */ + uint32_t version; /* X.Y.Z is encoded in nibbles xxxx.yy.zz */ + uint32_t sdk; /* X.Y.Z is encoded in nibbles xxxx.yy.zz */ +}; +""" + +build_version_command_fields = segment_base_fields + [ + ("platform", ctypes.c_uint32), + ("minos", ctypes.c_uint32), + ("sdk", ctypes.c_uint32), + ("ntools", ctypes.c_uint32), +] +""" +struct build_version_command { + uint32_t cmd; /* LC_BUILD_VERSION */ + uint32_t cmdsize; /* sizeof(struct build_version_command) plus */ + /* ntools * sizeof(struct build_tool_version) */ + uint32_t platform; /* platform */ + uint32_t minos; /* X.Y.Z is encoded in nibbles xxxx.yy.zz */ + uint32_t sdk; /* X.Y.Z is encoded in nibbles xxxx.yy.zz */ + uint32_t ntools; /* number of tool entries following this */ +}; +""" + + +def swap32(x: int) -> int: + return ( + ((x << 24) & 0xFF000000) + | ((x << 8) & 0x00FF0000) + | ((x >> 8) & 0x0000FF00) + | ((x >> 24) & 0x000000FF) + ) + + +def get_base_class_and_magic_number( + lib_file: BufferedIOBase, + seek: int | None = None, +) -> tuple[type[ctypes.Structure], int]: + if seek is None: + seek = lib_file.tell() + else: + lib_file.seek(seek) + magic_number = ctypes.c_uint32.from_buffer_copy( + lib_file.read(ctypes.sizeof(ctypes.c_uint32)) + ).value + + # Handle wrong byte order + if magic_number in [FAT_CIGAM, FAT_CIGAM_64, MH_CIGAM, MH_CIGAM_64]: + if sys.byteorder == "little": + BaseClass = ctypes.BigEndianStructure + else: + BaseClass = ctypes.LittleEndianStructure + + magic_number = swap32(magic_number) + else: + BaseClass = ctypes.Structure + + lib_file.seek(seek) + return BaseClass, magic_number + + +def read_data(struct_class: type[ctypes.Structure], lib_file: BufferedIOBase): + return struct_class.from_buffer_copy(lib_file.read(ctypes.sizeof(struct_class))) + + +def extract_macosx_min_system_version(path_to_lib: str): + with open(path_to_lib, "rb") as lib_file: + BaseClass, magic_number = get_base_class_and_magic_number(lib_file, 0) + if magic_number not in [FAT_MAGIC, FAT_MAGIC_64, MH_MAGIC, MH_MAGIC_64]: + return + + if magic_number in [FAT_MAGIC, FAT_CIGAM_64]: + + class FatHeader(BaseClass): + _fields_ = fat_header_fields + + fat_header = read_data(FatHeader, lib_file) + if magic_number == FAT_MAGIC: + + class FatArch(BaseClass): + _fields_ = fat_arch_fields + + else: + + class FatArch(BaseClass): + _fields_ = fat_arch_64_fields + + fat_arch_list = [ + read_data(FatArch, lib_file) for _ in range(fat_header.nfat_arch) + ] + + versions_list: list[tuple[int, int, int]] = [] + for el in fat_arch_list: + try: + version = read_mach_header(lib_file, el.offset) + if version is not None: + if el.cputype == CPU_TYPE_ARM64 and len(fat_arch_list) != 1: + # Xcode will not set the deployment target below 11.0.0 + # for the arm64 architecture. Ignore the arm64 deployment + # in fat binaries when the target is 11.0.0, that way + # the other architectures can select a lower deployment + # target. + # This is safe because there is no arm64 variant for + # macOS 10.15 or earlier. + if version == (11, 0, 0): + continue + versions_list.append(version) + except ValueError: + pass + + if len(versions_list) > 0: + return max(versions_list) + else: + return None + + else: + try: + return read_mach_header(lib_file, 0) + except ValueError: + """when some error during read library files""" + return None + + +def read_mach_header( + lib_file: BufferedIOBase, + seek: int | None = None, +) -> tuple[int, int, int] | None: + """ + This function parses a Mach-O header and extracts + information about the minimal macOS version. + + :param lib_file: reference to opened library file with pointer + """ + base_class, magic_number = get_base_class_and_magic_number(lib_file, seek) + arch = "32" if magic_number == MH_MAGIC else "64" + + class SegmentBase(base_class): + _fields_ = segment_base_fields + + if arch == "32": + + class MachHeader(base_class): + _fields_ = mach_header_fields + + else: + + class MachHeader(base_class): + _fields_ = mach_header_fields_64 + + mach_header = read_data(MachHeader, lib_file) + for _i in range(mach_header.ncmds): + pos = lib_file.tell() + segment_base = read_data(SegmentBase, lib_file) + lib_file.seek(pos) + if segment_base.cmd == LC_VERSION_MIN_MACOSX: + + class VersionMinCommand(base_class): + _fields_ = version_min_command_fields + + version_info = read_data(VersionMinCommand, lib_file) + return parse_version(version_info.version) + elif segment_base.cmd == LC_BUILD_VERSION: + + class VersionBuild(base_class): + _fields_ = build_version_command_fields + + version_info = read_data(VersionBuild, lib_file) + return parse_version(version_info.minos) + else: + lib_file.seek(pos + segment_base.cmdsize) + continue + + +def parse_version(version: int) -> tuple[int, int, int]: + x = (version & 0xFFFF0000) >> 16 + y = (version & 0x0000FF00) >> 8 + z = version & 0x000000FF + return x, y, z + + +def calculate_macosx_platform_tag(archive_root: StrPath, platform_tag: str) -> str: + """ + Calculate proper macosx platform tag basing on files which are included to wheel + + Example platform tag `macosx-10.14-x86_64` + """ + prefix, base_version, suffix = platform_tag.split("-") + base_version = tuple(int(x) for x in base_version.split(".")) + base_version = base_version[:2] + if base_version[0] > 10: + base_version = (base_version[0], 0) + assert len(base_version) == 2 + if "MACOSX_DEPLOYMENT_TARGET" in os.environ: + deploy_target = tuple( + int(x) for x in os.environ["MACOSX_DEPLOYMENT_TARGET"].split(".") + ) + deploy_target = deploy_target[:2] + if deploy_target[0] > 10: + deploy_target = (deploy_target[0], 0) + if deploy_target < base_version: + sys.stderr.write( + "[WARNING] MACOSX_DEPLOYMENT_TARGET is set to a lower value ({}) than " + "the version on which the Python interpreter was compiled ({}), and " + "will be ignored.\n".format( + ".".join(str(x) for x in deploy_target), + ".".join(str(x) for x in base_version), + ) + ) + else: + base_version = deploy_target + + assert len(base_version) == 2 + start_version = base_version + versions_dict: dict[str, tuple[int, int]] = {} + for dirpath, _dirnames, filenames in os.walk(archive_root): + for filename in filenames: + if filename.endswith(".dylib") or filename.endswith(".so"): + lib_path = os.path.join(dirpath, filename) + min_ver = extract_macosx_min_system_version(lib_path) + if min_ver is not None: + min_ver = min_ver[0:2] + if min_ver[0] > 10: + min_ver = (min_ver[0], 0) + versions_dict[lib_path] = min_ver + + if len(versions_dict) > 0: + base_version = max(base_version, max(versions_dict.values())) + + # macosx platform tag do not support minor bugfix release + fin_base_version = "_".join([str(x) for x in base_version]) + if start_version < base_version: + problematic_files = [k for k, v in versions_dict.items() if v > start_version] + problematic_files = "\n".join(problematic_files) + if len(problematic_files) == 1: + files_form = "this file" + else: + files_form = "these files" + error_message = ( + "[WARNING] This wheel needs a higher macOS version than {} " + "To silence this warning, set MACOSX_DEPLOYMENT_TARGET to at least " + + fin_base_version + + " or recreate " + + files_form + + " with lower " + "MACOSX_DEPLOYMENT_TARGET: \n" + problematic_files + ) + + if "MACOSX_DEPLOYMENT_TARGET" in os.environ: + error_message = error_message.format( + "is set in MACOSX_DEPLOYMENT_TARGET variable." + ) + else: + error_message = error_message.format( + "the version your Python interpreter is compiled against." + ) + + sys.stderr.write(error_message) + + platform_tag = prefix + "_" + fin_base_version + "_" + suffix + return platform_tag diff --git a/src/wheel/_metadata.py b/src/wheel/_metadata.py new file mode 100644 index 000000000..3a171755a --- /dev/null +++ b/src/wheel/_metadata.py @@ -0,0 +1,183 @@ +""" +Tools for converting old- to new-style metadata. +""" + +from __future__ import annotations + +import functools +import itertools +import os.path +import re +import textwrap +from email.message import Message +from email.parser import Parser +from typing import Generator, Iterable, Iterator, Literal + +from packaging.requirements import Requirement + + +def _nonblank(str: str) -> bool | Literal[""]: + return str and not str.startswith("#") + + +@functools.singledispatch +def yield_lines(iterable: Iterable[str]) -> Iterator[str]: + r""" + Yield valid lines of a string or iterable. + >>> list(yield_lines('')) + [] + >>> list(yield_lines(['foo', 'bar'])) + ['foo', 'bar'] + >>> list(yield_lines('foo\nbar')) + ['foo', 'bar'] + >>> list(yield_lines('\nfoo\n#bar\nbaz #comment')) + ['foo', 'baz #comment'] + >>> list(yield_lines(['foo\nbar', 'baz', 'bing\n\n\n'])) + ['foo', 'bar', 'baz', 'bing'] + """ + return itertools.chain.from_iterable(map(yield_lines, iterable)) + + +@yield_lines.register(str) +def _(text: str) -> Iterator[str]: + return filter(_nonblank, map(str.strip, text.splitlines())) + + +def split_sections( + s: str | Iterator[str], +) -> Generator[tuple[str | None, list[str]], None, None]: + """Split a string or iterable thereof into (section, content) pairs + Each ``section`` is a stripped version of the section header ("[section]") + and each ``content`` is a list of stripped lines excluding blank lines and + comment-only lines. If there are any such lines before the first section + header, they're returned in a first ``section`` of ``None``. + """ + section = None + content: list[str] = [] + for line in yield_lines(s): + if line.startswith("["): + if line.endswith("]"): + if section or content: + yield section, content + section = line[1:-1].strip() + content = [] + else: + raise ValueError("Invalid section heading", line) + else: + content.append(line) + + # wrap up last segment + yield section, content + + +def safe_extra(extra: str) -> str: + """Convert an arbitrary string to a standard 'extra' name + Any runs of non-alphanumeric characters are replaced with a single '_', + and the result is always lowercased. + """ + return re.sub("[^A-Za-z0-9.-]+", "_", extra).lower() + + +def safe_name(name: str) -> str: + """Convert an arbitrary string to a standard distribution name + Any runs of non-alphanumeric/. characters are replaced with a single '-'. + """ + return re.sub("[^A-Za-z0-9.]+", "-", name) + + +def requires_to_requires_dist(requirement: Requirement) -> str: + """Return the version specifier for a requirement in PEP 345/566 fashion.""" + if requirement.url: + return " @ " + requirement.url + + requires_dist: list[str] = [] + for spec in requirement.specifier: + requires_dist.append(spec.operator + spec.version) + + if requires_dist: + return " " + ",".join(sorted(requires_dist)) + else: + return "" + + +def convert_requirements(requirements: list[str]) -> Iterator[str]: + """Yield Requires-Dist: strings for parsed requirements strings.""" + for req in requirements: + parsed_requirement = Requirement(req) + spec = requires_to_requires_dist(parsed_requirement) + extras = ",".join(sorted(safe_extra(e) for e in parsed_requirement.extras)) + if extras: + extras = f"[{extras}]" + + yield safe_name(parsed_requirement.name) + extras + spec + + +def generate_requirements( + extras_require: dict[str | None, list[str]], +) -> Iterator[tuple[str, str]]: + """ + Convert requirements from a setup()-style dictionary to + ('Requires-Dist', 'requirement') and ('Provides-Extra', 'extra') tuples. + + extras_require is a dictionary of {extra: [requirements]} as passed to setup(), + using the empty extra {'': [requirements]} to hold install_requires. + """ + for extra, depends in extras_require.items(): + condition = "" + extra = extra or "" + if ":" in extra: # setuptools extra:condition syntax + extra, condition = extra.split(":", 1) + + extra = safe_extra(extra) + if extra: + yield "Provides-Extra", extra + if condition: + condition = "(" + condition + ") and " + condition += f"extra == '{extra}'" + + if condition: + condition = " ; " + condition + + for new_req in convert_requirements(depends): + canonical_req = str(Requirement(new_req + condition)) + yield "Requires-Dist", canonical_req + + +def pkginfo_to_metadata(egg_info_path: str, pkginfo_path: str) -> Message: + """ + Convert .egg-info directory with PKG-INFO to the Metadata 2.1 format + """ + with open(pkginfo_path, encoding="utf-8") as headers: + pkg_info = Parser().parse(headers) + + pkg_info.replace_header("Metadata-Version", "2.1") + # Those will be regenerated from `requires.txt`. + del pkg_info["Provides-Extra"] + del pkg_info["Requires-Dist"] + requires_path = os.path.join(egg_info_path, "requires.txt") + if os.path.exists(requires_path): + with open(requires_path, encoding="utf-8") as requires_file: + requires = requires_file.read() + + parsed_requirements = sorted(split_sections(requires), key=lambda x: x[0] or "") + for extra, reqs in parsed_requirements: + for key, value in generate_requirements({extra: reqs}): + if (key, value) not in pkg_info.items(): + pkg_info[key] = value + + description = pkg_info["Description"] + if description: + description_lines = pkg_info["Description"].splitlines() + dedented_description = "\n".join( + # if the first line of long_description is blank, + # the first line here will be indented. + ( + description_lines[0].lstrip(), + textwrap.dedent("\n".join(description_lines[1:])), + "\n", + ) + ) + pkg_info.set_payload(dedented_description) + del pkg_info["Description"] + + return pkg_info diff --git a/src/wheel/util.py b/src/wheel/_util.py similarity index 100% rename from src/wheel/util.py rename to src/wheel/_util.py diff --git a/src/wheel/vendored/__init__.py b/src/wheel/_vendored/__init__.py similarity index 100% rename from src/wheel/vendored/__init__.py rename to src/wheel/_vendored/__init__.py diff --git a/src/wheel/vendored/packaging/__init__.py b/src/wheel/_vendored/packaging/__init__.py similarity index 100% rename from src/wheel/vendored/packaging/__init__.py rename to src/wheel/_vendored/packaging/__init__.py diff --git a/src/wheel/vendored/packaging/_elffile.py b/src/wheel/_vendored/packaging/_elffile.py similarity index 100% rename from src/wheel/vendored/packaging/_elffile.py rename to src/wheel/_vendored/packaging/_elffile.py diff --git a/src/wheel/vendored/packaging/_manylinux.py b/src/wheel/_vendored/packaging/_manylinux.py similarity index 100% rename from src/wheel/vendored/packaging/_manylinux.py rename to src/wheel/_vendored/packaging/_manylinux.py diff --git a/src/wheel/vendored/packaging/_musllinux.py b/src/wheel/_vendored/packaging/_musllinux.py similarity index 100% rename from src/wheel/vendored/packaging/_musllinux.py rename to src/wheel/_vendored/packaging/_musllinux.py diff --git a/src/wheel/vendored/packaging/_parser.py b/src/wheel/_vendored/packaging/_parser.py similarity index 100% rename from src/wheel/vendored/packaging/_parser.py rename to src/wheel/_vendored/packaging/_parser.py diff --git a/src/wheel/vendored/packaging/_structures.py b/src/wheel/_vendored/packaging/_structures.py similarity index 100% rename from src/wheel/vendored/packaging/_structures.py rename to src/wheel/_vendored/packaging/_structures.py diff --git a/src/wheel/vendored/packaging/_tokenizer.py b/src/wheel/_vendored/packaging/_tokenizer.py similarity index 100% rename from src/wheel/vendored/packaging/_tokenizer.py rename to src/wheel/_vendored/packaging/_tokenizer.py diff --git a/src/wheel/vendored/packaging/markers.py b/src/wheel/_vendored/packaging/markers.py similarity index 100% rename from src/wheel/vendored/packaging/markers.py rename to src/wheel/_vendored/packaging/markers.py diff --git a/src/wheel/vendored/packaging/requirements.py b/src/wheel/_vendored/packaging/requirements.py similarity index 100% rename from src/wheel/vendored/packaging/requirements.py rename to src/wheel/_vendored/packaging/requirements.py diff --git a/src/wheel/vendored/packaging/specifiers.py b/src/wheel/_vendored/packaging/specifiers.py similarity index 100% rename from src/wheel/vendored/packaging/specifiers.py rename to src/wheel/_vendored/packaging/specifiers.py diff --git a/src/wheel/vendored/packaging/tags.py b/src/wheel/_vendored/packaging/tags.py similarity index 100% rename from src/wheel/vendored/packaging/tags.py rename to src/wheel/_vendored/packaging/tags.py diff --git a/src/wheel/vendored/packaging/utils.py b/src/wheel/_vendored/packaging/utils.py similarity index 100% rename from src/wheel/vendored/packaging/utils.py rename to src/wheel/_vendored/packaging/utils.py diff --git a/src/wheel/vendored/packaging/version.py b/src/wheel/_vendored/packaging/version.py similarity index 100% rename from src/wheel/vendored/packaging/version.py rename to src/wheel/_vendored/packaging/version.py diff --git a/src/wheel/vendored/vendor.txt b/src/wheel/_vendored/vendor.txt similarity index 100% rename from src/wheel/vendored/vendor.txt rename to src/wheel/_vendored/vendor.txt diff --git a/src/wheel/_wheelfile.py b/src/wheel/_wheelfile.py new file mode 100644 index 000000000..18de37430 --- /dev/null +++ b/src/wheel/_wheelfile.py @@ -0,0 +1,227 @@ +from __future__ import annotations + +import csv +import hashlib +import os.path +import re +import stat +import time +from io import StringIO, TextIOWrapper +from typing import IO, TYPE_CHECKING, Literal +from zipfile import ZIP_DEFLATED, ZipFile, ZipInfo + +from ._cli import WheelError +from ._util import log, urlsafe_b64decode, urlsafe_b64encode + +if TYPE_CHECKING: + from typing import Protocol, Sized, Union + + from typing_extensions import Buffer + + StrPath = Union[str, os.PathLike[str]] + + class SizedBuffer(Sized, Buffer, Protocol): ... + + +# Non-greedy matching of an optional build number may be too clever (more +# invalid wheel filenames will match). Separate regex for .dist-info? +WHEEL_INFO_RE = re.compile( + r"""^(?P(?P[^\s-]+?)-(?P[^\s-]+?))(-(?P\d[^\s-]*))? + -(?P[^\s-]+?)-(?P[^\s-]+?)-(?P\S+)\.whl$""", + re.VERBOSE, +) +MINIMUM_TIMESTAMP = 315532800 # 1980-01-01 00:00:00 UTC + + +def get_zipinfo_datetime(timestamp: float | None = None): + # Some applications need reproducible .whl files, but they can't do this without + # forcing the timestamp of the individual ZipInfo objects. See issue #143. + timestamp = int(os.environ.get("SOURCE_DATE_EPOCH", timestamp or time.time())) + timestamp = max(timestamp, MINIMUM_TIMESTAMP) + return time.gmtime(timestamp)[0:6] + + +class WheelFile(ZipFile): + """A ZipFile derivative class that also reads SHA-256 hashes from + .dist-info/RECORD and checks any read files against those. + """ + + _default_algorithm = hashlib.sha256 + + def __init__( + self, + file: StrPath, + mode: Literal["r", "w", "x", "a"] = "r", + compression: int = ZIP_DEFLATED, + ): + basename = os.path.basename(file) + self.parsed_filename = WHEEL_INFO_RE.match(basename) + if not basename.endswith(".whl") or self.parsed_filename is None: + raise WheelError(f"Bad wheel filename {basename!r}") + + ZipFile.__init__(self, file, mode, compression=compression, allowZip64=True) + + self.dist_info_path = "{}.dist-info".format( + self.parsed_filename.group("namever") + ) + self.record_path = self.dist_info_path + "/RECORD" + self._file_hashes: dict[str, tuple[None, None] | tuple[int, bytes]] = {} + self._file_sizes = {} + if mode == "r": + # Ignore RECORD and any embedded wheel signatures + self._file_hashes[self.record_path] = None, None + self._file_hashes[self.record_path + ".jws"] = None, None + self._file_hashes[self.record_path + ".p7s"] = None, None + + # Fill in the expected hashes by reading them from RECORD + try: + record = self.open(self.record_path) + except KeyError: + raise WheelError(f"Missing {self.record_path} file") from None + + with record: + for line in csv.reader( + TextIOWrapper(record, newline="", encoding="utf-8") + ): + path, hash_sum, size = line + if not hash_sum: + continue + + algorithm, hash_sum = hash_sum.split("=") + try: + hashlib.new(algorithm) + except ValueError: + raise WheelError( + f"Unsupported hash algorithm: {algorithm}" + ) from None + + if algorithm.lower() in {"md5", "sha1"}: + raise WheelError( + f"Weak hash algorithm ({algorithm}) is not permitted by " + f"PEP 427" + ) + + self._file_hashes[path] = ( + algorithm, + urlsafe_b64decode(hash_sum.encode("ascii")), + ) + + def open( + self, + name_or_info: str | ZipInfo, + mode: Literal["r", "w"] = "r", + pwd: bytes | None = None, + ) -> IO[bytes]: + def _update_crc(newdata: bytes) -> None: + eof = ef._eof + update_crc_orig(newdata) + running_hash.update(newdata) + if eof and running_hash.digest() != expected_hash: + raise WheelError(f"Hash mismatch for file '{ef_name}'") + + ef_name = ( + name_or_info.filename if isinstance(name_or_info, ZipInfo) else name_or_info + ) + if ( + mode == "r" + and not ef_name.endswith("/") + and ef_name not in self._file_hashes + ): + raise WheelError(f"No hash found for file '{ef_name}'") + + ef = ZipFile.open(self, name_or_info, mode, pwd) + if mode == "r" and not ef_name.endswith("/"): + algorithm, expected_hash = self._file_hashes[ef_name] + if expected_hash is not None: + # Monkey patch the _update_crc method to also check for the hash from + # RECORD + running_hash = hashlib.new(algorithm) + update_crc_orig, ef._update_crc = ef._update_crc, _update_crc + + return ef + + def write_files(self, base_dir: str): + log.info(f"creating '{self.filename}' and adding '{base_dir}' to it") + deferred: list[tuple[str, str]] = [] + for root, dirnames, filenames in os.walk(base_dir): + # Sort the directory names so that `os.walk` will walk them in a + # defined order on the next iteration. + dirnames.sort() + for name in sorted(filenames): + path = os.path.normpath(os.path.join(root, name)) + if os.path.isfile(path): + arcname = os.path.relpath(path, base_dir).replace(os.path.sep, "/") + if arcname == self.record_path: + pass + elif root.endswith(".dist-info"): + deferred.append((path, arcname)) + else: + self.write(path, arcname) + + deferred.sort() + for path, arcname in deferred: + self.write(path, arcname) + + def write( + self, + filename: str, + arcname: str | None = None, + compress_type: int | None = None, + ) -> None: + with open(filename, "rb") as f: + st = os.fstat(f.fileno()) + data = f.read() + + zinfo = ZipInfo( + arcname or filename, date_time=get_zipinfo_datetime(st.st_mtime) + ) + zinfo.external_attr = (stat.S_IMODE(st.st_mode) | stat.S_IFMT(st.st_mode)) << 16 + zinfo.compress_type = compress_type or self.compression + self.writestr(zinfo, data, compress_type) + + def writestr( + self, + zinfo_or_arcname: str | ZipInfo, + data: SizedBuffer | str, + compress_type: int | None = None, + ): + if isinstance(zinfo_or_arcname, str): + zinfo_or_arcname = ZipInfo( + zinfo_or_arcname, date_time=get_zipinfo_datetime() + ) + zinfo_or_arcname.compress_type = self.compression + zinfo_or_arcname.external_attr = (0o664 | stat.S_IFREG) << 16 + + if isinstance(data, str): + data = data.encode("utf-8") + + ZipFile.writestr(self, zinfo_or_arcname, data, compress_type) + fname = ( + zinfo_or_arcname.filename + if isinstance(zinfo_or_arcname, ZipInfo) + else zinfo_or_arcname + ) + log.info(f"adding '{fname}'") + if fname != self.record_path: + hash_ = self._default_algorithm(data) + self._file_hashes[fname] = ( + hash_.name, + urlsafe_b64encode(hash_.digest()).decode("ascii"), + ) + self._file_sizes[fname] = len(data) + + def close(self): + # Write RECORD + if self.fp is not None and self.mode == "w" and self._file_hashes: + data = StringIO() + writer = csv.writer(data, delimiter=",", quotechar='"', lineterminator="\n") + writer.writerows( + ( + (fname, algorithm + "=" + hash_, self._file_sizes[fname]) + for fname, (algorithm, hash_) in self._file_hashes.items() + ) + ) + writer.writerow((format(self.record_path), "", "")) + self.writestr(self.record_path, data.getvalue()) + + ZipFile.close(self) diff --git a/src/wheel/bdist_wheel.py b/src/wheel/bdist_wheel.py index 3cc7165e4..05d93eee8 100644 --- a/src/wheel/bdist_wheel.py +++ b/src/wheel/bdist_wheel.py @@ -1,604 +1,11 @@ -""" -Create a wheel (.whl) distribution. +from warnings import warn -A wheel is a built archive format. -""" +from ._bdist_wheel import bdist_wheel as bdist_wheel -from __future__ import annotations - -import os -import re -import shutil -import stat -import struct -import sys -import sysconfig -import warnings -from email.generator import BytesGenerator, Generator -from email.policy import EmailPolicy -from glob import iglob -from shutil import rmtree -from typing import TYPE_CHECKING, Callable, Iterable, Literal, Sequence, cast -from zipfile import ZIP_DEFLATED, ZIP_STORED - -import setuptools -from setuptools import Command - -from . import __version__ as wheel_version -from .metadata import pkginfo_to_metadata -from .util import log -from .vendored.packaging import tags -from .vendored.packaging import version as _packaging_version -from .wheelfile import WheelFile - -if TYPE_CHECKING: - import types - - -def safe_name(name: str) -> str: - """Convert an arbitrary string to a standard distribution name - Any runs of non-alphanumeric/. characters are replaced with a single '-'. - """ - return re.sub("[^A-Za-z0-9.]+", "-", name) - - -def safe_version(version: str) -> str: - """ - Convert an arbitrary string to a standard version string - """ - try: - # normalize the version - return str(_packaging_version.Version(version)) - except _packaging_version.InvalidVersion: - version = version.replace(" ", ".") - return re.sub("[^A-Za-z0-9.]+", "-", version) - - -setuptools_major_version = int(setuptools.__version__.split(".")[0]) - -PY_LIMITED_API_PATTERN = r"cp3\d" - - -def _is_32bit_interpreter() -> bool: - return struct.calcsize("P") == 4 - - -def python_tag() -> str: - return f"py{sys.version_info[0]}" - - -def get_platform(archive_root: str | None) -> str: - """Return our platform name 'win32', 'linux_x86_64'""" - result = sysconfig.get_platform() - if result.startswith("macosx") and archive_root is not None: - from .macosx_libfile import calculate_macosx_platform_tag - - result = calculate_macosx_platform_tag(archive_root, result) - elif _is_32bit_interpreter(): - if result == "linux-x86_64": - # pip pull request #3497 - result = "linux-i686" - elif result == "linux-aarch64": - # packaging pull request #234 - # TODO armv8l, packaging pull request #690 => this did not land - # in pip/packaging yet - result = "linux-armv7l" - - return result.replace("-", "_") - - -def get_flag( - var: str, fallback: bool, expected: bool = True, warn: bool = True -) -> bool: - """Use a fallback value for determining SOABI flags if the needed config - var is unset or unavailable.""" - val = sysconfig.get_config_var(var) - if val is None: - if warn: - warnings.warn( - f"Config variable '{var}' is unset, Python ABI tag may be incorrect", - RuntimeWarning, - stacklevel=2, - ) - return fallback - return val == expected - - -def get_abi_tag() -> str | None: - """Return the ABI tag based on SOABI (if available) or emulate SOABI (PyPy2).""" - soabi: str = sysconfig.get_config_var("SOABI") - impl = tags.interpreter_name() - if not soabi and impl in ("cp", "pp") and hasattr(sys, "maxunicode"): - d = "" - m = "" - u = "" - if get_flag("Py_DEBUG", hasattr(sys, "gettotalrefcount"), warn=(impl == "cp")): - d = "d" - - if get_flag( - "WITH_PYMALLOC", - impl == "cp", - warn=(impl == "cp" and sys.version_info < (3, 8)), - ) and sys.version_info < (3, 8): - m = "m" - - abi = f"{impl}{tags.interpreter_version()}{d}{m}{u}" - elif soabi and impl == "cp" and soabi.startswith("cpython"): - # non-Windows - abi = "cp" + soabi.split("-")[1] - elif soabi and impl == "cp" and soabi.startswith("cp"): - # Windows - abi = soabi.split("-")[0] - elif soabi and impl == "pp": - # we want something like pypy36-pp73 - abi = "-".join(soabi.split("-")[:2]) - abi = abi.replace(".", "_").replace("-", "_") - elif soabi and impl == "graalpy": - abi = "-".join(soabi.split("-")[:3]) - abi = abi.replace(".", "_").replace("-", "_") - elif soabi: - abi = soabi.replace(".", "_").replace("-", "_") - else: - abi = None - - return abi - - -def safer_name(name: str) -> str: - return safe_name(name).replace("-", "_") - - -def safer_version(version: str) -> str: - return safe_version(version).replace("-", "_") - - -def remove_readonly( - func: Callable[..., object], - path: str, - excinfo: tuple[type[Exception], Exception, types.TracebackType], -) -> None: - remove_readonly_exc(func, path, excinfo[1]) - - -def remove_readonly_exc(func: Callable[..., object], path: str, exc: Exception) -> None: - os.chmod(path, stat.S_IWRITE) - func(path) - - -class bdist_wheel(Command): - description = "create a wheel distribution" - - supported_compressions = { - "stored": ZIP_STORED, - "deflated": ZIP_DEFLATED, - } - - user_options = [ - ("bdist-dir=", "b", "temporary directory for creating the distribution"), - ( - "plat-name=", - "p", - "platform name to embed in generated filenames " - f"(default: {get_platform(None)})", - ), - ( - "keep-temp", - "k", - "keep the pseudo-installation tree around after " - "creating the distribution archive", - ), - ("dist-dir=", "d", "directory to put final built distributions in"), - ("skip-build", None, "skip rebuilding everything (for testing/debugging)"), - ( - "relative", - None, - "build the archive using relative paths (default: false)", - ), - ( - "owner=", - "u", - "Owner name used when creating a tar file [default: current user]", - ), - ( - "group=", - "g", - "Group name used when creating a tar file [default: current group]", - ), - ("universal", None, "make a universal wheel (default: false)"), - ( - "compression=", - None, - "zipfile compression (one of: {}) (default: 'deflated')".format( - ", ".join(supported_compressions) - ), - ), - ( - "python-tag=", - None, - f"Python implementation compatibility tag (default: '{python_tag()}')", - ), - ( - "build-number=", - None, - "Build number for this particular version. " - "As specified in PEP-0427, this must start with a digit. " - "[default: None]", - ), - ( - "py-limited-api=", - None, - "Python tag (cp32|cp33|cpNN) for abi3 wheel tag (default: false)", - ), - ] - - boolean_options = ["keep-temp", "skip-build", "relative", "universal"] - - def initialize_options(self): - self.bdist_dir: str = None - self.data_dir = None - self.plat_name: str | None = None - self.plat_tag = None - self.format = "zip" - self.keep_temp = False - self.dist_dir: str | None = None - self.egginfo_dir = None - self.root_is_pure: bool | None = None - self.skip_build = None - self.relative = False - self.owner = None - self.group = None - self.universal: bool = False - self.compression: str | int = "deflated" - self.python_tag: str = python_tag() - self.build_number: str | None = None - self.py_limited_api: str | Literal[False] = False - self.plat_name_supplied = False - - def finalize_options(self): - if self.bdist_dir is None: - bdist_base = self.get_finalized_command("bdist").bdist_base - self.bdist_dir = os.path.join(bdist_base, "wheel") - - egg_info = self.distribution.get_command_obj("egg_info") - egg_info.ensure_finalized() # needed for correct `wheel_dist_name` - - self.data_dir = self.wheel_dist_name + ".data" - self.plat_name_supplied = self.plat_name is not None - - try: - self.compression = self.supported_compressions[self.compression] - except KeyError: - raise ValueError(f"Unsupported compression: {self.compression}") from None - - need_options = ("dist_dir", "plat_name", "skip_build") - - self.set_undefined_options("bdist", *zip(need_options, need_options)) - - self.root_is_pure = not ( - self.distribution.has_ext_modules() or self.distribution.has_c_libraries() - ) - - if self.py_limited_api and not re.match( - PY_LIMITED_API_PATTERN, self.py_limited_api - ): - raise ValueError(f"py-limited-api must match '{PY_LIMITED_API_PATTERN}'") - - # Support legacy [wheel] section for setting universal - wheel = self.distribution.get_option_dict("wheel") - if "universal" in wheel: - # please don't define this in your global configs - log.warning( - "The [wheel] section is deprecated. Use [bdist_wheel] instead.", - ) - val = wheel["universal"][1].strip() - if val.lower() in ("1", "true", "yes"): - self.universal = True - - if self.build_number is not None and not self.build_number[:1].isdigit(): - raise ValueError("Build tag (build-number) must start with a digit.") - - @property - def wheel_dist_name(self): - """Return distribution full name with - replaced with _""" - components = ( - safer_name(self.distribution.get_name()), - safer_version(self.distribution.get_version()), - ) - if self.build_number: - components += (self.build_number,) - return "-".join(components) - - def get_tag(self) -> tuple[str, str, str]: - # bdist sets self.plat_name if unset, we should only use it for purepy - # wheels if the user supplied it. - if self.plat_name_supplied: - plat_name = cast(str, self.plat_name) - elif self.root_is_pure: - plat_name = "any" - else: - # macosx contains system version in platform name so need special handle - if self.plat_name and not self.plat_name.startswith("macosx"): - plat_name = self.plat_name - else: - # on macosx always limit the platform name to comply with any - # c-extension modules in bdist_dir, since the user can specify - # a higher MACOSX_DEPLOYMENT_TARGET via tools like CMake - - # on other platforms, and on macosx if there are no c-extension - # modules, use the default platform name. - plat_name = get_platform(self.bdist_dir) - - if _is_32bit_interpreter(): - if plat_name in ("linux-x86_64", "linux_x86_64"): - plat_name = "linux_i686" - if plat_name in ("linux-aarch64", "linux_aarch64"): - # TODO armv8l, packaging pull request #690 => this did not land - # in pip/packaging yet - plat_name = "linux_armv7l" - - plat_name = ( - plat_name.lower().replace("-", "_").replace(".", "_").replace(" ", "_") - ) - - if self.root_is_pure: - if self.universal: - impl = "py2.py3" - else: - impl = self.python_tag - tag = (impl, "none", plat_name) - else: - impl_name = tags.interpreter_name() - impl_ver = tags.interpreter_version() - impl = impl_name + impl_ver - # We don't work on CPython 3.1, 3.0. - if self.py_limited_api and (impl_name + impl_ver).startswith("cp3"): - impl = self.py_limited_api - abi_tag = "abi3" - else: - abi_tag = str(get_abi_tag()).lower() - tag = (impl, abi_tag, plat_name) - # issue gh-374: allow overriding plat_name - supported_tags = [ - (t.interpreter, t.abi, plat_name) for t in tags.sys_tags() - ] - assert ( - tag in supported_tags - ), f"would build wheel with unsupported tag {tag}" - return tag - - def run(self): - build_scripts = self.reinitialize_command("build_scripts") - build_scripts.executable = "python" - build_scripts.force = True - - build_ext = self.reinitialize_command("build_ext") - build_ext.inplace = False - - if not self.skip_build: - self.run_command("build") - - install = self.reinitialize_command("install", reinit_subcommands=True) - install.root = self.bdist_dir - install.compile = False - install.skip_build = self.skip_build - install.warn_dir = False - - # A wheel without setuptools scripts is more cross-platform. - # Use the (undocumented) `no_ep` option to setuptools' - # install_scripts command to avoid creating entry point scripts. - install_scripts = self.reinitialize_command("install_scripts") - install_scripts.no_ep = True - - # Use a custom scheme for the archive, because we have to decide - # at installation time which scheme to use. - for key in ("headers", "scripts", "data", "purelib", "platlib"): - setattr(install, "install_" + key, os.path.join(self.data_dir, key)) - - basedir_observed = "" - - if os.name == "nt": - # win32 barfs if any of these are ''; could be '.'? - # (distutils.command.install:change_roots bug) - basedir_observed = os.path.normpath(os.path.join(self.data_dir, "..")) - self.install_libbase = self.install_lib = basedir_observed - - setattr( - install, - "install_purelib" if self.root_is_pure else "install_platlib", - basedir_observed, - ) - - log.info(f"installing to {self.bdist_dir}") - - self.run_command("install") - - impl_tag, abi_tag, plat_tag = self.get_tag() - archive_basename = f"{self.wheel_dist_name}-{impl_tag}-{abi_tag}-{plat_tag}" - if not self.relative: - archive_root = self.bdist_dir - else: - archive_root = os.path.join( - self.bdist_dir, self._ensure_relative(install.install_base) - ) - - self.set_undefined_options("install_egg_info", ("target", "egginfo_dir")) - distinfo_dirname = ( - f"{safer_name(self.distribution.get_name())}-" - f"{safer_version(self.distribution.get_version())}.dist-info" - ) - distinfo_dir = os.path.join(self.bdist_dir, distinfo_dirname) - self.egg2dist(self.egginfo_dir, distinfo_dir) - - self.write_wheelfile(distinfo_dir) - - # Make the archive - if not os.path.exists(self.dist_dir): - os.makedirs(self.dist_dir) - - wheel_path = os.path.join(self.dist_dir, archive_basename + ".whl") - with WheelFile(wheel_path, "w", self.compression) as wf: - wf.write_files(archive_root) - - # Add to 'Distribution.dist_files' so that the "upload" command works - getattr(self.distribution, "dist_files", []).append( - ( - "bdist_wheel", - "{}.{}".format(*sys.version_info[:2]), # like 3.7 - wheel_path, - ) - ) - - if not self.keep_temp: - log.info(f"removing {self.bdist_dir}") - if not self.dry_run: - if sys.version_info < (3, 12): - rmtree(self.bdist_dir, onerror=remove_readonly) - else: - rmtree(self.bdist_dir, onexc=remove_readonly_exc) - - def write_wheelfile( - self, wheelfile_base: str, generator: str = f"bdist_wheel ({wheel_version})" - ): - from email.message import Message - - msg = Message() - msg["Wheel-Version"] = "1.0" # of the spec - msg["Generator"] = generator - msg["Root-Is-Purelib"] = str(self.root_is_pure).lower() - if self.build_number is not None: - msg["Build"] = self.build_number - - # Doesn't work for bdist_wininst - impl_tag, abi_tag, plat_tag = self.get_tag() - for impl in impl_tag.split("."): - for abi in abi_tag.split("."): - for plat in plat_tag.split("."): - msg["Tag"] = "-".join((impl, abi, plat)) - - wheelfile_path = os.path.join(wheelfile_base, "WHEEL") - log.info(f"creating {wheelfile_path}") - with open(wheelfile_path, "wb") as f: - BytesGenerator(f, maxheaderlen=0).flatten(msg) - - def _ensure_relative(self, path: str) -> str: - # copied from dir_util, deleted - drive, path = os.path.splitdrive(path) - if path[0:1] == os.sep: - path = drive + path[1:] - return path - - @property - def license_paths(self) -> Iterable[str]: - if setuptools_major_version >= 57: - # Setuptools has resolved any patterns to actual file names - return self.distribution.metadata.license_files or () - - files: set[str] = set() - metadata = self.distribution.get_option_dict("metadata") - if setuptools_major_version >= 42: - # Setuptools recognizes the license_files option but does not do globbing - patterns = cast(Sequence[str], self.distribution.metadata.license_files) - else: - # Prior to those, wheel is entirely responsible for handling license files - if "license_files" in metadata: - patterns = metadata["license_files"][1].split() - else: - patterns = () - - if "license_file" in metadata: - warnings.warn( - 'The "license_file" option is deprecated. Use "license_files" instead.', - DeprecationWarning, - stacklevel=2, - ) - files.add(metadata["license_file"][1]) - - if not files and not patterns and not isinstance(patterns, list): - patterns = ("LICEN[CS]E*", "COPYING*", "NOTICE*", "AUTHORS*") - - for pattern in patterns: - for path in iglob(pattern): - if path.endswith("~"): - log.debug( - f'ignoring license file "{path}" as it looks like a backup' - ) - continue - - if path not in files and os.path.isfile(path): - log.info( - f'adding license file "{path}" (matched pattern "{pattern}")' - ) - files.add(path) - - return files - - def egg2dist(self, egginfo_path: str, distinfo_path: str): - """Convert an .egg-info directory into a .dist-info directory""" - - def adios(p: str) -> None: - """Appropriately delete directory, file or link.""" - if os.path.exists(p) and not os.path.islink(p) and os.path.isdir(p): - shutil.rmtree(p) - elif os.path.exists(p): - os.unlink(p) - - adios(distinfo_path) - - if not os.path.exists(egginfo_path): - # There is no egg-info. This is probably because the egg-info - # file/directory is not named matching the distribution name used - # to name the archive file. Check for this case and report - # accordingly. - import glob - - pat = os.path.join(os.path.dirname(egginfo_path), "*.egg-info") - possible = glob.glob(pat) - err = f"Egg metadata expected at {egginfo_path} but not found" - if possible: - alt = os.path.basename(possible[0]) - err += f" ({alt} found - possible misnamed archive file?)" - - raise ValueError(err) - - if os.path.isfile(egginfo_path): - # .egg-info is a single file - pkg_info = pkginfo_to_metadata(egginfo_path, egginfo_path) - os.mkdir(distinfo_path) - else: - # .egg-info is a directory - pkginfo_path = os.path.join(egginfo_path, "PKG-INFO") - pkg_info = pkginfo_to_metadata(egginfo_path, pkginfo_path) - - # ignore common egg metadata that is useless to wheel - shutil.copytree( - egginfo_path, - distinfo_path, - ignore=lambda x, y: { - "PKG-INFO", - "requires.txt", - "SOURCES.txt", - "not-zip-safe", - }, - ) - - # delete dependency_links if it is only whitespace - dependency_links_path = os.path.join(distinfo_path, "dependency_links.txt") - with open(dependency_links_path, encoding="utf-8") as dependency_links_file: - dependency_links = dependency_links_file.read().strip() - if not dependency_links: - adios(dependency_links_path) - - pkg_info_path = os.path.join(distinfo_path, "METADATA") - serialization_policy = EmailPolicy( - utf8=True, - mangle_from_=False, - max_line_length=0, - ) - with open(pkg_info_path, "w", encoding="utf-8") as out: - Generator(out, policy=serialization_policy).flatten(pkg_info) - - for license_path in self.license_paths: - filename = os.path.basename(license_path) - shutil.copy(license_path, os.path.join(distinfo_path, filename)) - - adios(egginfo_path) +warn( + "The 'wheel' package is no longer the canonical location of the 'bdist_wheel' " + "command, and will be removed in a future release. Please update to setuptools " + "v70.1 or later which contains an integrated version of this command.", + DeprecationWarning, + stacklevel=1, +) diff --git a/src/wheel/macosx_libfile.py b/src/wheel/macosx_libfile.py index abdfc9eda..92fa4a6fa 100644 --- a/src/wheel/macosx_libfile.py +++ b/src/wheel/macosx_libfile.py @@ -1,482 +1,13 @@ -""" -This module contains function to analyse dynamic library -headers to extract system information - -Currently only for MacOSX - -Library file on macosx system starts with Mach-O or Fat field. -This can be distinguish by first 32 bites and it is called magic number. -Proper value of magic number is with suffix _MAGIC. Suffix _CIGAM means -reversed bytes order. -Both fields can occur in two types: 32 and 64 bytes. - -FAT field inform that this library contains few version of library -(typically for different types version). It contains -information where Mach-O headers starts. - -Each section started with Mach-O header contains one library -(So if file starts with this field it contains only one version). - -After filed Mach-O there are section fields. -Each of them starts with two fields: -cmd - magic number for this command -cmdsize - total size occupied by this section information. - -In this case only sections LC_VERSION_MIN_MACOSX (for macosx 10.13 and earlier) -and LC_BUILD_VERSION (for macosx 10.14 and newer) are interesting, -because them contains information about minimal system version. - -Important remarks: -- For fat files this implementation looks for maximum number version. - It not check if it is 32 or 64 and do not compare it with currently built package. - So it is possible to false report higher version that needed. -- All structures signatures are taken form macosx header files. -- I think that binary format will be more stable than `otool` output. - and if apple introduce some changes both implementation will need to be updated. -- The system compile will set the deployment target no lower than - 11.0 for arm64 builds. For "Universal 2" builds use the x86_64 deployment - target when the arm64 target is 11.0. -""" - -from __future__ import annotations - -import ctypes -import os -import sys -from io import BufferedIOBase -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from typing import Union - - StrPath = Union[str, os.PathLike[str]] - -"""here the needed const and struct from mach-o header files""" - -FAT_MAGIC = 0xCAFEBABE -FAT_CIGAM = 0xBEBAFECA -FAT_MAGIC_64 = 0xCAFEBABF -FAT_CIGAM_64 = 0xBFBAFECA -MH_MAGIC = 0xFEEDFACE -MH_CIGAM = 0xCEFAEDFE -MH_MAGIC_64 = 0xFEEDFACF -MH_CIGAM_64 = 0xCFFAEDFE - -LC_VERSION_MIN_MACOSX = 0x24 -LC_BUILD_VERSION = 0x32 - -CPU_TYPE_ARM64 = 0x0100000C - -mach_header_fields = [ - ("magic", ctypes.c_uint32), - ("cputype", ctypes.c_int), - ("cpusubtype", ctypes.c_int), - ("filetype", ctypes.c_uint32), - ("ncmds", ctypes.c_uint32), - ("sizeofcmds", ctypes.c_uint32), - ("flags", ctypes.c_uint32), -] -""" -struct mach_header { - uint32_t magic; /* mach magic number identifier */ - cpu_type_t cputype; /* cpu specifier */ - cpu_subtype_t cpusubtype; /* machine specifier */ - uint32_t filetype; /* type of file */ - uint32_t ncmds; /* number of load commands */ - uint32_t sizeofcmds; /* the size of all the load commands */ - uint32_t flags; /* flags */ -}; -typedef integer_t cpu_type_t; -typedef integer_t cpu_subtype_t; -""" - -mach_header_fields_64 = mach_header_fields + [("reserved", ctypes.c_uint32)] -""" -struct mach_header_64 { - uint32_t magic; /* mach magic number identifier */ - cpu_type_t cputype; /* cpu specifier */ - cpu_subtype_t cpusubtype; /* machine specifier */ - uint32_t filetype; /* type of file */ - uint32_t ncmds; /* number of load commands */ - uint32_t sizeofcmds; /* the size of all the load commands */ - uint32_t flags; /* flags */ - uint32_t reserved; /* reserved */ -}; -""" - -fat_header_fields = [("magic", ctypes.c_uint32), ("nfat_arch", ctypes.c_uint32)] -""" -struct fat_header { - uint32_t magic; /* FAT_MAGIC or FAT_MAGIC_64 */ - uint32_t nfat_arch; /* number of structs that follow */ -}; -""" - -fat_arch_fields = [ - ("cputype", ctypes.c_int), - ("cpusubtype", ctypes.c_int), - ("offset", ctypes.c_uint32), - ("size", ctypes.c_uint32), - ("align", ctypes.c_uint32), -] -""" -struct fat_arch { - cpu_type_t cputype; /* cpu specifier (int) */ - cpu_subtype_t cpusubtype; /* machine specifier (int) */ - uint32_t offset; /* file offset to this object file */ - uint32_t size; /* size of this object file */ - uint32_t align; /* alignment as a power of 2 */ -}; -""" - -fat_arch_64_fields = [ - ("cputype", ctypes.c_int), - ("cpusubtype", ctypes.c_int), - ("offset", ctypes.c_uint64), - ("size", ctypes.c_uint64), - ("align", ctypes.c_uint32), - ("reserved", ctypes.c_uint32), -] -""" -struct fat_arch_64 { - cpu_type_t cputype; /* cpu specifier (int) */ - cpu_subtype_t cpusubtype; /* machine specifier (int) */ - uint64_t offset; /* file offset to this object file */ - uint64_t size; /* size of this object file */ - uint32_t align; /* alignment as a power of 2 */ - uint32_t reserved; /* reserved */ -}; -""" - -segment_base_fields = [("cmd", ctypes.c_uint32), ("cmdsize", ctypes.c_uint32)] -"""base for reading segment info""" - -segment_command_fields = [ - ("cmd", ctypes.c_uint32), - ("cmdsize", ctypes.c_uint32), - ("segname", ctypes.c_char * 16), - ("vmaddr", ctypes.c_uint32), - ("vmsize", ctypes.c_uint32), - ("fileoff", ctypes.c_uint32), - ("filesize", ctypes.c_uint32), - ("maxprot", ctypes.c_int), - ("initprot", ctypes.c_int), - ("nsects", ctypes.c_uint32), - ("flags", ctypes.c_uint32), -] -""" -struct segment_command { /* for 32-bit architectures */ - uint32_t cmd; /* LC_SEGMENT */ - uint32_t cmdsize; /* includes sizeof section structs */ - char segname[16]; /* segment name */ - uint32_t vmaddr; /* memory address of this segment */ - uint32_t vmsize; /* memory size of this segment */ - uint32_t fileoff; /* file offset of this segment */ - uint32_t filesize; /* amount to map from the file */ - vm_prot_t maxprot; /* maximum VM protection */ - vm_prot_t initprot; /* initial VM protection */ - uint32_t nsects; /* number of sections in segment */ - uint32_t flags; /* flags */ -}; -typedef int vm_prot_t; -""" - -segment_command_fields_64 = [ - ("cmd", ctypes.c_uint32), - ("cmdsize", ctypes.c_uint32), - ("segname", ctypes.c_char * 16), - ("vmaddr", ctypes.c_uint64), - ("vmsize", ctypes.c_uint64), - ("fileoff", ctypes.c_uint64), - ("filesize", ctypes.c_uint64), - ("maxprot", ctypes.c_int), - ("initprot", ctypes.c_int), - ("nsects", ctypes.c_uint32), - ("flags", ctypes.c_uint32), -] -""" -struct segment_command_64 { /* for 64-bit architectures */ - uint32_t cmd; /* LC_SEGMENT_64 */ - uint32_t cmdsize; /* includes sizeof section_64 structs */ - char segname[16]; /* segment name */ - uint64_t vmaddr; /* memory address of this segment */ - uint64_t vmsize; /* memory size of this segment */ - uint64_t fileoff; /* file offset of this segment */ - uint64_t filesize; /* amount to map from the file */ - vm_prot_t maxprot; /* maximum VM protection */ - vm_prot_t initprot; /* initial VM protection */ - uint32_t nsects; /* number of sections in segment */ - uint32_t flags; /* flags */ -}; -""" - -version_min_command_fields = segment_base_fields + [ - ("version", ctypes.c_uint32), - ("sdk", ctypes.c_uint32), -] -""" -struct version_min_command { - uint32_t cmd; /* LC_VERSION_MIN_MACOSX or - LC_VERSION_MIN_IPHONEOS or - LC_VERSION_MIN_WATCHOS or - LC_VERSION_MIN_TVOS */ - uint32_t cmdsize; /* sizeof(struct min_version_command) */ - uint32_t version; /* X.Y.Z is encoded in nibbles xxxx.yy.zz */ - uint32_t sdk; /* X.Y.Z is encoded in nibbles xxxx.yy.zz */ -}; -""" - -build_version_command_fields = segment_base_fields + [ - ("platform", ctypes.c_uint32), - ("minos", ctypes.c_uint32), - ("sdk", ctypes.c_uint32), - ("ntools", ctypes.c_uint32), -] -""" -struct build_version_command { - uint32_t cmd; /* LC_BUILD_VERSION */ - uint32_t cmdsize; /* sizeof(struct build_version_command) plus */ - /* ntools * sizeof(struct build_tool_version) */ - uint32_t platform; /* platform */ - uint32_t minos; /* X.Y.Z is encoded in nibbles xxxx.yy.zz */ - uint32_t sdk; /* X.Y.Z is encoded in nibbles xxxx.yy.zz */ - uint32_t ntools; /* number of tool entries following this */ -}; -""" - - -def swap32(x: int) -> int: - return ( - ((x << 24) & 0xFF000000) - | ((x << 8) & 0x00FF0000) - | ((x >> 8) & 0x0000FF00) - | ((x >> 24) & 0x000000FF) - ) - - -def get_base_class_and_magic_number( - lib_file: BufferedIOBase, - seek: int | None = None, -) -> tuple[type[ctypes.Structure], int]: - if seek is None: - seek = lib_file.tell() - else: - lib_file.seek(seek) - magic_number = ctypes.c_uint32.from_buffer_copy( - lib_file.read(ctypes.sizeof(ctypes.c_uint32)) - ).value - - # Handle wrong byte order - if magic_number in [FAT_CIGAM, FAT_CIGAM_64, MH_CIGAM, MH_CIGAM_64]: - if sys.byteorder == "little": - BaseClass = ctypes.BigEndianStructure - else: - BaseClass = ctypes.LittleEndianStructure - - magic_number = swap32(magic_number) - else: - BaseClass = ctypes.Structure - - lib_file.seek(seek) - return BaseClass, magic_number - - -def read_data(struct_class: type[ctypes.Structure], lib_file: BufferedIOBase): - return struct_class.from_buffer_copy(lib_file.read(ctypes.sizeof(struct_class))) - - -def extract_macosx_min_system_version(path_to_lib: str): - with open(path_to_lib, "rb") as lib_file: - BaseClass, magic_number = get_base_class_and_magic_number(lib_file, 0) - if magic_number not in [FAT_MAGIC, FAT_MAGIC_64, MH_MAGIC, MH_MAGIC_64]: - return - - if magic_number in [FAT_MAGIC, FAT_CIGAM_64]: - - class FatHeader(BaseClass): - _fields_ = fat_header_fields - - fat_header = read_data(FatHeader, lib_file) - if magic_number == FAT_MAGIC: - - class FatArch(BaseClass): - _fields_ = fat_arch_fields - - else: - - class FatArch(BaseClass): - _fields_ = fat_arch_64_fields - - fat_arch_list = [ - read_data(FatArch, lib_file) for _ in range(fat_header.nfat_arch) - ] - - versions_list: list[tuple[int, int, int]] = [] - for el in fat_arch_list: - try: - version = read_mach_header(lib_file, el.offset) - if version is not None: - if el.cputype == CPU_TYPE_ARM64 and len(fat_arch_list) != 1: - # Xcode will not set the deployment target below 11.0.0 - # for the arm64 architecture. Ignore the arm64 deployment - # in fat binaries when the target is 11.0.0, that way - # the other architectures can select a lower deployment - # target. - # This is safe because there is no arm64 variant for - # macOS 10.15 or earlier. - if version == (11, 0, 0): - continue - versions_list.append(version) - except ValueError: - pass - - if len(versions_list) > 0: - return max(versions_list) - else: - return None - - else: - try: - return read_mach_header(lib_file, 0) - except ValueError: - """when some error during read library files""" - return None - - -def read_mach_header( - lib_file: BufferedIOBase, - seek: int | None = None, -) -> tuple[int, int, int] | None: - """ - This function parses a Mach-O header and extracts - information about the minimal macOS version. - - :param lib_file: reference to opened library file with pointer - """ - base_class, magic_number = get_base_class_and_magic_number(lib_file, seek) - arch = "32" if magic_number == MH_MAGIC else "64" - - class SegmentBase(base_class): - _fields_ = segment_base_fields - - if arch == "32": - - class MachHeader(base_class): - _fields_ = mach_header_fields - - else: - - class MachHeader(base_class): - _fields_ = mach_header_fields_64 - - mach_header = read_data(MachHeader, lib_file) - for _i in range(mach_header.ncmds): - pos = lib_file.tell() - segment_base = read_data(SegmentBase, lib_file) - lib_file.seek(pos) - if segment_base.cmd == LC_VERSION_MIN_MACOSX: - - class VersionMinCommand(base_class): - _fields_ = version_min_command_fields - - version_info = read_data(VersionMinCommand, lib_file) - return parse_version(version_info.version) - elif segment_base.cmd == LC_BUILD_VERSION: - - class VersionBuild(base_class): - _fields_ = build_version_command_fields - - version_info = read_data(VersionBuild, lib_file) - return parse_version(version_info.minos) - else: - lib_file.seek(pos + segment_base.cmdsize) - continue - - -def parse_version(version: int) -> tuple[int, int, int]: - x = (version & 0xFFFF0000) >> 16 - y = (version & 0x0000FF00) >> 8 - z = version & 0x000000FF - return x, y, z - - -def calculate_macosx_platform_tag(archive_root: StrPath, platform_tag: str) -> str: - """ - Calculate proper macosx platform tag basing on files which are included to wheel - - Example platform tag `macosx-10.14-x86_64` - """ - prefix, base_version, suffix = platform_tag.split("-") - base_version = tuple(int(x) for x in base_version.split(".")) - base_version = base_version[:2] - if base_version[0] > 10: - base_version = (base_version[0], 0) - assert len(base_version) == 2 - if "MACOSX_DEPLOYMENT_TARGET" in os.environ: - deploy_target = tuple( - int(x) for x in os.environ["MACOSX_DEPLOYMENT_TARGET"].split(".") - ) - deploy_target = deploy_target[:2] - if deploy_target[0] > 10: - deploy_target = (deploy_target[0], 0) - if deploy_target < base_version: - sys.stderr.write( - "[WARNING] MACOSX_DEPLOYMENT_TARGET is set to a lower value ({}) than " - "the version on which the Python interpreter was compiled ({}), and " - "will be ignored.\n".format( - ".".join(str(x) for x in deploy_target), - ".".join(str(x) for x in base_version), - ) - ) - else: - base_version = deploy_target - - assert len(base_version) == 2 - start_version = base_version - versions_dict: dict[str, tuple[int, int]] = {} - for dirpath, _dirnames, filenames in os.walk(archive_root): - for filename in filenames: - if filename.endswith(".dylib") or filename.endswith(".so"): - lib_path = os.path.join(dirpath, filename) - min_ver = extract_macosx_min_system_version(lib_path) - if min_ver is not None: - min_ver = min_ver[0:2] - if min_ver[0] > 10: - min_ver = (min_ver[0], 0) - versions_dict[lib_path] = min_ver - - if len(versions_dict) > 0: - base_version = max(base_version, max(versions_dict.values())) - - # macosx platform tag do not support minor bugfix release - fin_base_version = "_".join([str(x) for x in base_version]) - if start_version < base_version: - problematic_files = [k for k, v in versions_dict.items() if v > start_version] - problematic_files = "\n".join(problematic_files) - if len(problematic_files) == 1: - files_form = "this file" - else: - files_form = "these files" - error_message = ( - "[WARNING] This wheel needs a higher macOS version than {} " - "To silence this warning, set MACOSX_DEPLOYMENT_TARGET to at least " - + fin_base_version - + " or recreate " - + files_form - + " with lower " - "MACOSX_DEPLOYMENT_TARGET: \n" + problematic_files - ) - - if "MACOSX_DEPLOYMENT_TARGET" in os.environ: - error_message = error_message.format( - "is set in MACOSX_DEPLOYMENT_TARGET variable." - ) - else: - error_message = error_message.format( - "the version your Python interpreter is compiled against." - ) - - sys.stderr.write(error_message) - - platform_tag = prefix + "_" + fin_base_version + "_" + suffix - return platform_tag +from warnings import warn + +from ._macosx_libfile import ( + calculate_macosx_platform_tag as calculate_macosx_platform_tag, +) + +warn( + f"The {__package__}.{__name__} module has been deprecated and will be removed in a " + f"future release of 'wheel'. Please use the appropriate APIs from 'packaging' " + f"instead.", + DeprecationWarning, + stacklevel=1, +) diff --git a/src/wheel/metadata.py b/src/wheel/metadata.py index b8098fa85..f64bf6040 100644 --- a/src/wheel/metadata.py +++ b/src/wheel/metadata.py @@ -1,183 +1,14 @@ -""" -Tools for converting old- to new-style metadata. -""" - -from __future__ import annotations - -import functools -import itertools -import os.path -import re -import textwrap -from email.message import Message -from email.parser import Parser -from typing import Generator, Iterable, Iterator, Literal - -from .vendored.packaging.requirements import Requirement - - -def _nonblank(str: str) -> bool | Literal[""]: - return str and not str.startswith("#") - - -@functools.singledispatch -def yield_lines(iterable: Iterable[str]) -> Iterator[str]: - r""" - Yield valid lines of a string or iterable. - >>> list(yield_lines('')) - [] - >>> list(yield_lines(['foo', 'bar'])) - ['foo', 'bar'] - >>> list(yield_lines('foo\nbar')) - ['foo', 'bar'] - >>> list(yield_lines('\nfoo\n#bar\nbaz #comment')) - ['foo', 'baz #comment'] - >>> list(yield_lines(['foo\nbar', 'baz', 'bing\n\n\n'])) - ['foo', 'bar', 'baz', 'bing'] - """ - return itertools.chain.from_iterable(map(yield_lines, iterable)) - - -@yield_lines.register(str) -def _(text: str) -> Iterator[str]: - return filter(_nonblank, map(str.strip, text.splitlines())) - - -def split_sections( - s: str | Iterator[str], -) -> Generator[tuple[str | None, list[str]], None, None]: - """Split a string or iterable thereof into (section, content) pairs - Each ``section`` is a stripped version of the section header ("[section]") - and each ``content`` is a list of stripped lines excluding blank lines and - comment-only lines. If there are any such lines before the first section - header, they're returned in a first ``section`` of ``None``. - """ - section = None - content: list[str] = [] - for line in yield_lines(s): - if line.startswith("["): - if line.endswith("]"): - if section or content: - yield section, content - section = line[1:-1].strip() - content = [] - else: - raise ValueError("Invalid section heading", line) - else: - content.append(line) - - # wrap up last segment - yield section, content - - -def safe_extra(extra: str) -> str: - """Convert an arbitrary string to a standard 'extra' name - Any runs of non-alphanumeric characters are replaced with a single '_', - and the result is always lowercased. - """ - return re.sub("[^A-Za-z0-9.-]+", "_", extra).lower() - - -def safe_name(name: str) -> str: - """Convert an arbitrary string to a standard distribution name - Any runs of non-alphanumeric/. characters are replaced with a single '-'. - """ - return re.sub("[^A-Za-z0-9.]+", "-", name) - - -def requires_to_requires_dist(requirement: Requirement) -> str: - """Return the version specifier for a requirement in PEP 345/566 fashion.""" - if requirement.url: - return " @ " + requirement.url - - requires_dist: list[str] = [] - for spec in requirement.specifier: - requires_dist.append(spec.operator + spec.version) - - if requires_dist: - return " " + ",".join(sorted(requires_dist)) - else: - return "" - - -def convert_requirements(requirements: list[str]) -> Iterator[str]: - """Yield Requires-Dist: strings for parsed requirements strings.""" - for req in requirements: - parsed_requirement = Requirement(req) - spec = requires_to_requires_dist(parsed_requirement) - extras = ",".join(sorted(safe_extra(e) for e in parsed_requirement.extras)) - if extras: - extras = f"[{extras}]" - - yield safe_name(parsed_requirement.name) + extras + spec - - -def generate_requirements( - extras_require: dict[str | None, list[str]], -) -> Iterator[tuple[str, str]]: - """ - Convert requirements from a setup()-style dictionary to - ('Requires-Dist', 'requirement') and ('Provides-Extra', 'extra') tuples. - - extras_require is a dictionary of {extra: [requirements]} as passed to setup(), - using the empty extra {'': [requirements]} to hold install_requires. - """ - for extra, depends in extras_require.items(): - condition = "" - extra = extra or "" - if ":" in extra: # setuptools extra:condition syntax - extra, condition = extra.split(":", 1) - - extra = safe_extra(extra) - if extra: - yield "Provides-Extra", extra - if condition: - condition = "(" + condition + ") and " - condition += f"extra == '{extra}'" - - if condition: - condition = " ; " + condition - - for new_req in convert_requirements(depends): - canonical_req = str(Requirement(new_req + condition)) - yield "Requires-Dist", canonical_req - - -def pkginfo_to_metadata(egg_info_path: str, pkginfo_path: str) -> Message: - """ - Convert .egg-info directory with PKG-INFO to the Metadata 2.1 format - """ - with open(pkginfo_path, encoding="utf-8") as headers: - pkg_info = Parser().parse(headers) - - pkg_info.replace_header("Metadata-Version", "2.1") - # Those will be regenerated from `requires.txt`. - del pkg_info["Provides-Extra"] - del pkg_info["Requires-Dist"] - requires_path = os.path.join(egg_info_path, "requires.txt") - if os.path.exists(requires_path): - with open(requires_path, encoding="utf-8") as requires_file: - requires = requires_file.read() - - parsed_requirements = sorted(split_sections(requires), key=lambda x: x[0] or "") - for extra, reqs in parsed_requirements: - for key, value in generate_requirements({extra: reqs}): - if (key, value) not in pkg_info.items(): - pkg_info[key] = value - - description = pkg_info["Description"] - if description: - description_lines = pkg_info["Description"].splitlines() - dedented_description = "\n".join( - # if the first line of long_description is blank, - # the first line here will be indented. - ( - description_lines[0].lstrip(), - textwrap.dedent("\n".join(description_lines[1:])), - "\n", - ) - ) - pkg_info.set_payload(dedented_description) - del pkg_info["Description"] - - return pkg_info +from warnings import warn + +from ._metadata import convert_requirements as convert_requirements +from ._metadata import generate_requirements as generate_requirements +from ._metadata import pkginfo_to_metadata as pkginfo_to_metadata +from ._metadata import requires_to_requires_dist as requires_to_requires_dist + +warn( + f"The {__package__}.{__name__} module has been deprecated and will be removed in a " + f"future release of 'wheel'. Please use the appropriate APIs from 'packaging' " + f"instead.", + DeprecationWarning, + stacklevel=1, +) diff --git a/src/wheel/wheelfile.py b/src/wheel/wheelfile.py index 0a0f4596c..852cba424 100644 --- a/src/wheel/wheelfile.py +++ b/src/wheel/wheelfile.py @@ -1,227 +1,13 @@ -from __future__ import annotations - -import csv -import hashlib -import os.path -import re -import stat -import time -from io import StringIO, TextIOWrapper -from typing import IO, TYPE_CHECKING, Literal -from zipfile import ZIP_DEFLATED, ZipFile, ZipInfo - -from wheel.cli import WheelError -from wheel.util import log, urlsafe_b64decode, urlsafe_b64encode - -if TYPE_CHECKING: - from typing import Protocol, Sized, Union - - from typing_extensions import Buffer - - StrPath = Union[str, os.PathLike[str]] - - class SizedBuffer(Sized, Buffer, Protocol): ... - - -# Non-greedy matching of an optional build number may be too clever (more -# invalid wheel filenames will match). Separate regex for .dist-info? -WHEEL_INFO_RE = re.compile( - r"""^(?P(?P[^\s-]+?)-(?P[^\s-]+?))(-(?P\d[^\s-]*))? - -(?P[^\s-]+?)-(?P[^\s-]+?)-(?P\S+)\.whl$""", - re.VERBOSE, +from warnings import warn + +from ._wheelfile import WHEEL_INFO_RE as WHEEL_INFO_RE +from ._wheelfile import WheelFile as WheelFile +from ._wheelfile import get_zipinfo_datetime as get_zipinfo_datetime + +warn( + f"The {__package__}.{__name__} module has been deprecated and will be removed in a " + f"future release of 'wheel'. Please use the appropriate APIs from 'packaging' " + f"instead.", + DeprecationWarning, + stacklevel=1, ) -MINIMUM_TIMESTAMP = 315532800 # 1980-01-01 00:00:00 UTC - - -def get_zipinfo_datetime(timestamp: float | None = None): - # Some applications need reproducible .whl files, but they can't do this without - # forcing the timestamp of the individual ZipInfo objects. See issue #143. - timestamp = int(os.environ.get("SOURCE_DATE_EPOCH", timestamp or time.time())) - timestamp = max(timestamp, MINIMUM_TIMESTAMP) - return time.gmtime(timestamp)[0:6] - - -class WheelFile(ZipFile): - """A ZipFile derivative class that also reads SHA-256 hashes from - .dist-info/RECORD and checks any read files against those. - """ - - _default_algorithm = hashlib.sha256 - - def __init__( - self, - file: StrPath, - mode: Literal["r", "w", "x", "a"] = "r", - compression: int = ZIP_DEFLATED, - ): - basename = os.path.basename(file) - self.parsed_filename = WHEEL_INFO_RE.match(basename) - if not basename.endswith(".whl") or self.parsed_filename is None: - raise WheelError(f"Bad wheel filename {basename!r}") - - ZipFile.__init__(self, file, mode, compression=compression, allowZip64=True) - - self.dist_info_path = "{}.dist-info".format( - self.parsed_filename.group("namever") - ) - self.record_path = self.dist_info_path + "/RECORD" - self._file_hashes: dict[str, tuple[None, None] | tuple[int, bytes]] = {} - self._file_sizes = {} - if mode == "r": - # Ignore RECORD and any embedded wheel signatures - self._file_hashes[self.record_path] = None, None - self._file_hashes[self.record_path + ".jws"] = None, None - self._file_hashes[self.record_path + ".p7s"] = None, None - - # Fill in the expected hashes by reading them from RECORD - try: - record = self.open(self.record_path) - except KeyError: - raise WheelError(f"Missing {self.record_path} file") from None - - with record: - for line in csv.reader( - TextIOWrapper(record, newline="", encoding="utf-8") - ): - path, hash_sum, size = line - if not hash_sum: - continue - - algorithm, hash_sum = hash_sum.split("=") - try: - hashlib.new(algorithm) - except ValueError: - raise WheelError( - f"Unsupported hash algorithm: {algorithm}" - ) from None - - if algorithm.lower() in {"md5", "sha1"}: - raise WheelError( - f"Weak hash algorithm ({algorithm}) is not permitted by " - f"PEP 427" - ) - - self._file_hashes[path] = ( - algorithm, - urlsafe_b64decode(hash_sum.encode("ascii")), - ) - - def open( - self, - name_or_info: str | ZipInfo, - mode: Literal["r", "w"] = "r", - pwd: bytes | None = None, - ) -> IO[bytes]: - def _update_crc(newdata: bytes) -> None: - eof = ef._eof - update_crc_orig(newdata) - running_hash.update(newdata) - if eof and running_hash.digest() != expected_hash: - raise WheelError(f"Hash mismatch for file '{ef_name}'") - - ef_name = ( - name_or_info.filename if isinstance(name_or_info, ZipInfo) else name_or_info - ) - if ( - mode == "r" - and not ef_name.endswith("/") - and ef_name not in self._file_hashes - ): - raise WheelError(f"No hash found for file '{ef_name}'") - - ef = ZipFile.open(self, name_or_info, mode, pwd) - if mode == "r" and not ef_name.endswith("/"): - algorithm, expected_hash = self._file_hashes[ef_name] - if expected_hash is not None: - # Monkey patch the _update_crc method to also check for the hash from - # RECORD - running_hash = hashlib.new(algorithm) - update_crc_orig, ef._update_crc = ef._update_crc, _update_crc - - return ef - - def write_files(self, base_dir: str): - log.info(f"creating '{self.filename}' and adding '{base_dir}' to it") - deferred: list[tuple[str, str]] = [] - for root, dirnames, filenames in os.walk(base_dir): - # Sort the directory names so that `os.walk` will walk them in a - # defined order on the next iteration. - dirnames.sort() - for name in sorted(filenames): - path = os.path.normpath(os.path.join(root, name)) - if os.path.isfile(path): - arcname = os.path.relpath(path, base_dir).replace(os.path.sep, "/") - if arcname == self.record_path: - pass - elif root.endswith(".dist-info"): - deferred.append((path, arcname)) - else: - self.write(path, arcname) - - deferred.sort() - for path, arcname in deferred: - self.write(path, arcname) - - def write( - self, - filename: str, - arcname: str | None = None, - compress_type: int | None = None, - ) -> None: - with open(filename, "rb") as f: - st = os.fstat(f.fileno()) - data = f.read() - - zinfo = ZipInfo( - arcname or filename, date_time=get_zipinfo_datetime(st.st_mtime) - ) - zinfo.external_attr = (stat.S_IMODE(st.st_mode) | stat.S_IFMT(st.st_mode)) << 16 - zinfo.compress_type = compress_type or self.compression - self.writestr(zinfo, data, compress_type) - - def writestr( - self, - zinfo_or_arcname: str | ZipInfo, - data: SizedBuffer | str, - compress_type: int | None = None, - ): - if isinstance(zinfo_or_arcname, str): - zinfo_or_arcname = ZipInfo( - zinfo_or_arcname, date_time=get_zipinfo_datetime() - ) - zinfo_or_arcname.compress_type = self.compression - zinfo_or_arcname.external_attr = (0o664 | stat.S_IFREG) << 16 - - if isinstance(data, str): - data = data.encode("utf-8") - - ZipFile.writestr(self, zinfo_or_arcname, data, compress_type) - fname = ( - zinfo_or_arcname.filename - if isinstance(zinfo_or_arcname, ZipInfo) - else zinfo_or_arcname - ) - log.info(f"adding '{fname}'") - if fname != self.record_path: - hash_ = self._default_algorithm(data) - self._file_hashes[fname] = ( - hash_.name, - urlsafe_b64encode(hash_.digest()).decode("ascii"), - ) - self._file_sizes[fname] = len(data) - - def close(self): - # Write RECORD - if self.fp is not None and self.mode == "w" and self._file_hashes: - data = StringIO() - writer = csv.writer(data, delimiter=",", quotechar='"', lineterminator="\n") - writer.writerows( - ( - (fname, algorithm + "=" + hash_, self._file_sizes[fname]) - for fname, (algorithm, hash_) in self._file_hashes.items() - ) - ) - writer.writerow((format(self.record_path), "", "")) - self.writestr(self.record_path, data.getvalue()) - - ZipFile.close(self) diff --git a/tests/cli/eggnames.txt b/tests/_cli/eggnames.txt similarity index 100% rename from tests/cli/eggnames.txt rename to tests/_cli/eggnames.txt diff --git a/tests/cli/test_convert.py b/tests/_cli/test_convert.py similarity index 89% rename from tests/cli/test_convert.py rename to tests/_cli/test_convert.py index 4f26b2377..a5041743d 100644 --- a/tests/cli/test_convert.py +++ b/tests/_cli/test_convert.py @@ -3,8 +3,8 @@ import os.path import re -from wheel.cli.convert import convert, egg_info_re -from wheel.wheelfile import WHEEL_INFO_RE +from wheel._cli._convert import convert, egg_info_re +from wheel._wheelfile import WHEEL_INFO_RE def test_egg_re(): diff --git a/tests/cli/test_pack.py b/tests/_cli/test_pack.py similarity index 98% rename from tests/cli/test_pack.py rename to tests/_cli/test_pack.py index f0363bcf6..ae5794c67 100644 --- a/tests/cli/test_pack.py +++ b/tests/_cli/test_pack.py @@ -7,7 +7,7 @@ from zipfile import ZipFile import pytest -from wheel.cli.pack import pack +from wheel._cli._pack import pack THISDIR = os.path.dirname(__file__) TESTWHEEL_NAME = "test-1.0-py2.py3-none-any.whl" diff --git a/tests/cli/test_tags.py b/tests/_cli/test_tags.py similarity index 98% rename from tests/cli/test_tags.py rename to tests/_cli/test_tags.py index 6940b10ab..35637ab94 100644 --- a/tests/cli/test_tags.py +++ b/tests/_cli/test_tags.py @@ -6,9 +6,9 @@ from zipfile import ZipFile import pytest -from wheel.cli import main, parser -from wheel.cli.tags import tags -from wheel.wheelfile import WheelFile +from wheel._cli import main, parser +from wheel._cli._tags import tags +from wheel._wheelfile import WheelFile TESTDIR = Path(__file__).parent.parent TESTWHEEL_NAME = "test-1.0-py2.py3-none-any.whl" diff --git a/tests/cli/test_unpack.py b/tests/_cli/test_unpack.py similarity index 92% rename from tests/cli/test_unpack.py rename to tests/_cli/test_unpack.py index be93a3b2c..c232a22c2 100644 --- a/tests/cli/test_unpack.py +++ b/tests/_cli/test_unpack.py @@ -4,8 +4,8 @@ import stat import pytest -from wheel.cli.unpack import unpack -from wheel.wheelfile import WheelFile +from wheel._cli._unpack import unpack +from wheel._wheelfile import WheelFile def test_unpack(wheel_paths, tmp_path): diff --git a/tests/test_bdist_wheel.py b/tests/test_bdist_wheel.py index 84ead4514..1bc0b3331 100644 --- a/tests/test_bdist_wheel.py +++ b/tests/test_bdist_wheel.py @@ -16,14 +16,14 @@ import pytest import setuptools -from wheel.bdist_wheel import ( +from wheel._bdist_wheel import ( bdist_wheel, get_abi_tag, remove_readonly, remove_readonly_exc, ) -from wheel.vendored.packaging import tags -from wheel.wheelfile import WheelFile +from wheel._vendored.packaging import tags +from wheel._wheelfile import WheelFile DEFAULT_FILES = { "dummy_dist-1.0.dist-info/top_level.txt", @@ -439,6 +439,4 @@ def _fake_import(name: str, *args, **kwargs): if module.startswith("wheel"): monkeypatch.delitem(sys.modules, module) - from wheel import bdist_wheel - assert bdist_wheel diff --git a/tests/test_macosx_libfile.py b/tests/test_macosx_libfile.py index 762d83621..7cf37c269 100644 --- a/tests/test_macosx_libfile.py +++ b/tests/test_macosx_libfile.py @@ -5,8 +5,8 @@ import sysconfig import pytest -from wheel.bdist_wheel import get_platform -from wheel.macosx_libfile import extract_macosx_min_system_version +from wheel._bdist_wheel import get_platform +from wheel._macosx_libfile import extract_macosx_min_system_version def test_read_from_dylib(): diff --git a/tests/test_metadata.py b/tests/test_metadata.py index ff7e7ccd7..204c3fe5a 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -1,6 +1,6 @@ from __future__ import annotations -from wheel.metadata import pkginfo_to_metadata +from wheel._metadata import pkginfo_to_metadata def test_pkginfo_to_metadata(tmp_path): diff --git a/tests/test_wheelfile.py b/tests/test_wheelfile.py index cbc8e802e..b4a007bfa 100644 --- a/tests/test_wheelfile.py +++ b/tests/test_wheelfile.py @@ -5,8 +5,8 @@ from zipfile import ZIP_DEFLATED, ZipFile import pytest -from wheel.cli import WheelError -from wheel.wheelfile import WheelFile +from wheel._cli import WheelError +from wheel._wheelfile import WheelFile @pytest.fixture