From 643c70e11783dfe244032d83067dd7d61a3cab75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Tue, 6 May 2025 22:49:49 +0200 Subject: [PATCH 01/24] Add pylock --- src/packaging/pylock.py | 598 ++++++++++++++++++++++++++++++++++++++++ tests/requirements.txt | 2 + tests/test_pylock.py | 484 ++++++++++++++++++++++++++++++++ 3 files changed, 1084 insertions(+) create mode 100644 src/packaging/pylock.py create mode 100644 tests/test_pylock.py diff --git a/src/packaging/pylock.py b/src/packaging/pylock.py new file mode 100644 index 000000000..365a2d56b --- /dev/null +++ b/src/packaging/pylock.py @@ -0,0 +1,598 @@ +from __future__ import annotations + +import dataclasses +import logging +import re +import sys +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path +from typing import ( + TYPE_CHECKING, + Any, + Protocol, + TypeVar, +) + +from .markers import Marker +from .specifiers import SpecifierSet +from .utils import NormalizedName, is_normalized_name +from .version import Version + +if TYPE_CHECKING: # pragma: no cover + if sys.version_info >= (3, 11): + from typing import Self + else: + from typing_extensions import Self + +__all__ = [ + "Package", + "PackageArchive", + "PackageDirectory", + "PackageSdist", + "PackageVcs", + "PackageWheel", + "Pylock", + "PylockUnsupportedVersionError", + "PylockValidationError", + "is_valid_pylock_path", +] + +T = TypeVar("T") + + +class FromMappingProtocol(Protocol): # pragma: no cover + @classmethod + def _from_dict(cls, d: Mapping[str, Any]) -> Self: ... + + +FromMappingProtocolT = TypeVar("FromMappingProtocolT", bound=FromMappingProtocol) + + +class SingleArgConstructor(Protocol): # pragma: no cover + def __init__(self, value: Any) -> None: ... + + +SingleArgConstructorT = TypeVar("SingleArgConstructorT", bound=SingleArgConstructor) + +PYLOCK_FILE_NAME_RE = re.compile(r"^pylock\.([^.]+)\.toml$") + + +def is_valid_pylock_path(path: Path) -> bool: + return path.name == "pylock.toml" or bool(PYLOCK_FILE_NAME_RE.match(path.name)) + + +def _toml_key(key: str) -> str: + return key.replace("_", "-") + + +def _toml_value(key: str, value: Any) -> Any: + if isinstance(value, (Version, Marker, SpecifierSet)): + return str(value) + if isinstance(value, Sequence) and key == "environments": + return [str(v) for v in value] + return value + + +def _toml_dict_factory(data: list[tuple[str, Any]]) -> dict[str, Any]: + return { + _toml_key(key): _toml_value(key, value) + for key, value in data + if value is not None + } + + +def _get(d: Mapping[str, Any], expected_type: type[T], key: str) -> T | None: + """Get value from dictionary and verify expected type.""" + value = d.get(key) + if value is None: + return None + if not isinstance(value, expected_type): + raise PylockValidationError( + f"{key!r} has unexpected type {type(value).__name__} " + f"(expected {expected_type.__name__})" + ) + return value + + +def _get_required(d: Mapping[str, Any], expected_type: type[T], key: str) -> T: + """Get required value from dictionary and verify expected type.""" + value = _get(d, expected_type, key) + if value is None: + raise PylockRequiredKeyError(key) + return value + + +def _get_sequence( + d: Mapping[str, Any], expected_item_type: type[T], key: str +) -> Sequence[T] | None: + """Get list value from dictionary and verify expected items type.""" + value = _get(d, Sequence, key) # type: ignore[type-abstract] + if value is None: + return None + for i, item in enumerate(value): + if not isinstance(item, expected_item_type): + raise PylockValidationError( + f"Item {i} of {key!r} has unexpected type {type(item).__name__} " + f"(expected {expected_item_type.__name__})" + ) + return value + + +def _get_as( + d: Mapping[str, Any], + expected_type: type[T], + target_type: type[SingleArgConstructorT], + key: str, +) -> SingleArgConstructorT | None: + """Get value from dictionary, verify expected type, convert to target type. + + This assumes the target_type constructor accepts the value. + """ + value = _get(d, expected_type, key) + if value is None: + return None + try: + return target_type(value) + except Exception as e: + raise PylockValidationError(f"Error in {key!r}: {e}") from e + + +def _get_required_as( + d: Mapping[str, Any], + expected_type: type[T], + target_type: type[SingleArgConstructorT], + key: str, +) -> SingleArgConstructorT: + """Get required value from dictionary, verify expected type, + convert to target type.""" + value = _get_as(d, expected_type, target_type, key) + if value is None: + raise PylockRequiredKeyError(key) + return value + + +def _get_sequence_as( + d: Mapping[str, Any], + expected_item_type: type[T], + target_item_type: type[SingleArgConstructorT], + key: str, +) -> Sequence[SingleArgConstructorT] | None: + """Get list value from dictionary and verify expected items type.""" + value = _get_sequence(d, expected_item_type, key) + if value is None: + return None + result = [] + for i, item in enumerate(value): + try: + result.append(target_item_type(item)) + except Exception as e: + raise PylockValidationError(f"Error in item {i} of {key!r}: {e}") from e + return result + + +def _get_object( + d: Mapping[str, Any], target_type: type[FromMappingProtocolT], key: str +) -> FromMappingProtocolT | None: + """Get dictionary value from dictionary and convert to dataclass.""" + value = _get(d, Mapping, key) # type: ignore[type-abstract] + if value is None: + return None + try: + return target_type._from_dict(value) + except Exception as e: + raise PylockValidationError(f"Error in {key!r}: {e}") from e + + +def _get_sequence_of_objects( + d: Mapping[str, Any], target_item_type: type[FromMappingProtocolT], key: str +) -> Sequence[FromMappingProtocolT] | None: + """Get list value from dictionary and convert items to dataclass.""" + value = _get(d, Sequence, key) # type: ignore[type-abstract] + if value is None: + return None + result = [] + for i, item in enumerate(value): + if not isinstance(item, Mapping): + raise PylockValidationError(f"Item {i} of {key!r} is not a table") + try: + result.append(target_item_type._from_dict(item)) + except Exception as e: + raise PylockValidationError(f"Error in item {i} of {key!r}: {e}") from e + return result + + +def _get_required_list_of_objects( + d: Mapping[str, Any], target_type: type[FromMappingProtocolT], key: str +) -> Sequence[FromMappingProtocolT]: + """Get required list value from dictionary and convert items to dataclass.""" + result = _get_sequence_of_objects(d, target_type, key) + if result is None: + raise PylockRequiredKeyError(key) + return result + + +def _exactly_one(iterable: Iterable[object]) -> bool: + found = False + for item in iterable: + if item: + if found: + return False + found = True + return found + + +def _validate_path_url(path: str | None, url: str | None) -> None: + if not path and not url: + raise PylockValidationError("path or url must be provided") + + +def _validate_hashes(hashes: Mapping[str, Any]) -> None: + if not hashes: + raise PylockValidationError("At least one hash must be provided") + if not all(isinstance(hash, str) for hash in hashes.values()): + raise PylockValidationError("Hash values must be strings") + + +class PylockValidationError(Exception): + pass + + +class PylockRequiredKeyError(PylockValidationError): + def __init__(self, key: str) -> None: + super().__init__(f"Missing required field {key!r}") + + +class PylockUnsupportedVersionError(PylockValidationError): + pass + + +@dataclass(frozen=True, init=False) +class PackageVcs: + type: str + url: str | None # = None + path: str | None # = None + requested_revision: str | None # = None + commit_id: str + subdirectory: str | None = None + + def __init__( + self, + *, + type: str, + url: str | None = None, + path: str | None = None, + requested_revision: str | None = None, + commit_id: str, + subdirectory: str | None = None, + ) -> None: + # In Python 3.10+ make dataclass kw_only=True and remove __init__ + object.__setattr__(self, "type", type) + object.__setattr__(self, "url", url) + object.__setattr__(self, "path", path) + object.__setattr__(self, "requested_revision", requested_revision) + object.__setattr__(self, "commit_id", commit_id) + object.__setattr__(self, "subdirectory", subdirectory) + # __post_init__ in Python 3.10+ + _validate_path_url(self.path, self.url) + + @classmethod + def _from_dict(cls, d: Mapping[str, Any]) -> Self: + return cls( + type=_get_required(d, str, "type"), + url=_get(d, str, "url"), + path=_get(d, str, "path"), + requested_revision=_get(d, str, "requested-revision"), + commit_id=_get_required(d, str, "commit-id"), + subdirectory=_get(d, str, "subdirectory"), + ) + + +@dataclass(frozen=True, init=False) +class PackageDirectory: + path: str + editable: bool | None = None + subdirectory: str | None = None + + def __init__( + self, + *, + path: str, + editable: bool | None = None, + subdirectory: str | None = None, + ) -> None: + # In Python 3.10+ make dataclass kw_only=True and remove __init__ + object.__setattr__(self, "path", path) + object.__setattr__(self, "editable", editable) + object.__setattr__(self, "subdirectory", subdirectory) + + @classmethod + def _from_dict(cls, d: Mapping[str, Any]) -> Self: + return cls( + path=_get_required(d, str, "path"), + editable=_get(d, bool, "editable"), + subdirectory=_get(d, str, "subdirectory"), + ) + + +@dataclass(frozen=True, init=False) +class PackageArchive: + url: str | None # = None + path: str | None # = None + size: int | None # = None + upload_time: datetime | None # = None + hashes: Mapping[str, str] + subdirectory: str | None = None + + def __init__( + self, + *, + hashes: Mapping[str, str], + url: str | None = None, + path: str | None = None, + size: int | None = None, + upload_time: datetime | None = None, + subdirectory: str | None = None, + ) -> None: + # In Python 3.10+ make dataclass kw_only=True and remove __init__ + object.__setattr__(self, "url", url) + object.__setattr__(self, "path", path) + object.__setattr__(self, "size", size) + object.__setattr__(self, "upload_time", upload_time) + object.__setattr__(self, "hashes", hashes) + object.__setattr__(self, "subdirectory", subdirectory) + # __post_init__ in Python 3.10+ + _validate_path_url(self.path, self.url) + _validate_hashes(self.hashes) + + @classmethod + def _from_dict(cls, d: Mapping[str, Any]) -> Self: + return cls( + url=_get(d, str, "url"), + path=_get(d, str, "path"), + size=_get(d, int, "size"), + upload_time=_get(d, datetime, "upload-time"), + hashes=_get_required(d, Mapping, "hashes"), # type: ignore[type-abstract] + subdirectory=_get(d, str, "subdirectory"), + ) + + +@dataclass(frozen=True, init=False) +class PackageSdist: + name: str | None # = None + upload_time: datetime | None # = None + url: str | None # = None + path: str | None # = None + size: int | None # = None + hashes: Mapping[str, str] + + def __init__( + self, + *, + hashes: Mapping[str, str], + name: str | None = None, + upload_time: datetime | None = None, + url: str | None = None, + path: str | None = None, + size: int | None = None, + ) -> None: + # In Python 3.10+ make dataclass kw_only=True and remove __init__ + object.__setattr__(self, "name", name) + object.__setattr__(self, "upload_time", upload_time) + object.__setattr__(self, "url", url) + object.__setattr__(self, "path", path) + object.__setattr__(self, "size", size) + object.__setattr__(self, "hashes", hashes) + # __post_init__ in Python 3.10+ + _validate_path_url(self.path, self.url) + _validate_hashes(self.hashes) + + @classmethod + def _from_dict(cls, d: Mapping[str, Any]) -> Self: + return cls( + name=_get(d, str, "name"), + upload_time=_get(d, datetime, "upload-time"), + url=_get(d, str, "url"), + path=_get(d, str, "path"), + size=_get(d, int, "size"), + hashes=_get_required(d, Mapping, "hashes"), # type: ignore[type-abstract] + ) + + +@dataclass(frozen=True, init=False) +class PackageWheel: + name: str # | None + upload_time: datetime | None # = None + url: str | None # = None + path: str | None # = None + size: int | None # = None + hashes: Mapping[str, str] + + def __init__( + self, + *, + hashes: Mapping[str, str], + name: str | None = None, + upload_time: datetime | None = None, + url: str | None = None, + path: str | None = None, + size: int | None = None, + ) -> None: + # In Python 3.10+ make dataclass kw_only=True and remove __init__ + object.__setattr__(self, "name", name) + object.__setattr__(self, "upload_time", upload_time) + object.__setattr__(self, "url", url) + object.__setattr__(self, "path", path) + object.__setattr__(self, "size", size) + object.__setattr__(self, "hashes", hashes) + # __post_init__ in Python 3.10+ + _validate_path_url(self.path, self.url) + _validate_hashes(self.hashes) + + @classmethod + def _from_dict(cls, d: Mapping[str, Any]) -> Self: + return cls( + name=_get(d, str, "name"), + upload_time=_get(d, datetime, "upload-time"), + url=_get(d, str, "url"), + path=_get(d, str, "path"), + size=_get(d, int, "size"), + hashes=_get_required(d, Mapping, "hashes"), # type: ignore[type-abstract] + ) + + +@dataclass(frozen=True, init=False) +class Package: + name: NormalizedName + version: Version | None = None + marker: Marker | None = None + requires_python: SpecifierSet | None = None + dependencies: Sequence[Mapping[str, Any]] | None = None + vcs: PackageVcs | None = None + directory: PackageDirectory | None = None + archive: PackageArchive | None = None + index: str | None = None + sdist: PackageSdist | None = None + wheels: Sequence[PackageWheel] | None = None + attestation_identities: Sequence[Mapping[str, Any]] | None = None + tool: Mapping[str, Any] | None = None + + def __init__( + self, + *, + name: str, + version: Version | None = None, + marker: Marker | None = None, + requires_python: SpecifierSet | None = None, + dependencies: Sequence[Mapping[str, Any]] | None = None, + vcs: PackageVcs | None = None, + directory: PackageDirectory | None = None, + archive: PackageArchive | None = None, + index: str | None = None, + sdist: PackageSdist | None = None, + wheels: Sequence[PackageWheel] | None = None, + attestation_identities: Sequence[Mapping[str, Any]] | None = None, + tool: Mapping[str, Any] | None = None, + ) -> None: + # In Python 3.10+ make dataclass kw_only=True and remove __init__ + object.__setattr__(self, "name", name) + object.__setattr__(self, "version", version) + object.__setattr__(self, "marker", marker) + object.__setattr__(self, "requires_python", requires_python) + object.__setattr__(self, "dependencies", dependencies) + object.__setattr__(self, "vcs", vcs) + object.__setattr__(self, "directory", directory) + object.__setattr__(self, "archive", archive) + object.__setattr__(self, "index", index) + object.__setattr__(self, "sdist", sdist) + object.__setattr__(self, "wheels", wheels) + object.__setattr__(self, "attestation_identities", attestation_identities) + object.__setattr__(self, "tool", tool) + # __post_init__ in Python 3.10+ + if not is_normalized_name(self.name): + raise PylockValidationError(f"Package name {self.name!r} is not normalized") + if self.sdist or self.wheels: + if any([self.vcs, self.directory, self.archive]): + raise PylockValidationError( + "None of vcs, directory, archive " + "must be set if sdist or wheels are set" + ) + else: + # no sdist nor wheels + if not _exactly_one([self.vcs, self.directory, self.archive]): + raise PylockValidationError( + "Exactly one of vcs, directory, archive must be set " + "if sdist and wheels are not set" + ) + + @classmethod + def _from_dict(cls, d: Mapping[str, Any]) -> Self: + package = cls( + name=_get_required(d, str, "name"), + version=_get_as(d, str, Version, "version"), + requires_python=_get_as(d, str, SpecifierSet, "requires-python"), + dependencies=_get_sequence(d, Mapping, "dependencies"), # type: ignore[type-abstract] + marker=_get_as(d, str, Marker, "marker"), + vcs=_get_object(d, PackageVcs, "vcs"), + directory=_get_object(d, PackageDirectory, "directory"), + archive=_get_object(d, PackageArchive, "archive"), + index=_get(d, str, "index"), + sdist=_get_object(d, PackageSdist, "sdist"), + wheels=_get_sequence_of_objects(d, PackageWheel, "wheels"), + attestation_identities=_get_sequence(d, Mapping, "attestation-identities"), # type: ignore[type-abstract] + tool=_get(d, Mapping, "tool"), # type: ignore[type-abstract] + ) + return package + + @property + def is_direct(self) -> bool: + return not (self.sdist or self.wheels) + + +@dataclass(frozen=True, init=False) +class Pylock: + lock_version: Version + environments: Sequence[Marker] | None # = None + requires_python: SpecifierSet | None # = None + extras: Sequence[str] | None # = None + dependency_groups: Sequence[str] | None # = None + default_groups: Sequence[str] | None # = None + created_by: str + packages: Sequence[Package] + tool: Mapping[str, Any] | None = None + + def __init__( + self, + *, + lock_version: Version, + created_by: str, + environments: Sequence[Marker] | None = None, + requires_python: SpecifierSet | None = None, + extras: Sequence[str] | None = None, + dependency_groups: Sequence[str] | None = None, + default_groups: Sequence[str] | None = None, + packages: Sequence[Package], + tool: Mapping[str, Any] | None = None, + ) -> None: + # In Python 3.10+ make dataclass kw_only=True and remove __init__ + object.__setattr__(self, "lock_version", lock_version) + object.__setattr__(self, "environments", environments) + object.__setattr__(self, "requires_python", requires_python) + object.__setattr__(self, "extras", extras) + object.__setattr__(self, "dependency_groups", dependency_groups) + object.__setattr__(self, "default_groups", default_groups) + object.__setattr__(self, "created_by", created_by) + object.__setattr__(self, "packages", packages) + object.__setattr__(self, "tool", tool) + # __post_init__ in Python 3.10+ + if self.lock_version < Version("1") or self.lock_version >= Version("2"): + raise PylockUnsupportedVersionError( + f"pylock version {self.lock_version} is not supported" + ) + if self.lock_version > Version("1.0"): + logging.warning( + "pylock minor version %s is not supported", self.lock_version + ) + + def to_dict(self) -> Mapping[str, Any]: + return dataclasses.asdict(self, dict_factory=_toml_dict_factory) + + @classmethod + def _from_dict(cls, d: Mapping[str, Any]) -> Self: + return cls( + lock_version=_get_required_as(d, str, Version, "lock-version"), + environments=_get_sequence_as(d, str, Marker, "environments"), + extras=_get_sequence(d, str, "extras"), + dependency_groups=_get_sequence(d, str, "dependency-groups"), + default_groups=_get_sequence(d, str, "default-groups"), + created_by=_get_required(d, str, "created-by"), + requires_python=_get_as(d, str, SpecifierSet, "requires-python"), + packages=_get_required_list_of_objects(d, Package, "packages"), + tool=_get(d, Mapping, "tool"), # type: ignore[type-abstract] + ) + + +def from_dict(d: Mapping[str, Any]) -> Pylock: + """Create a Pylock object from a dictionary.""" + return Pylock._from_dict(d) diff --git a/tests/requirements.txt b/tests/requirements.txt index 6dacfd1b1..efe3f4f76 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -2,3 +2,5 @@ coverage[toml]>=5.0.0 pip>=21.1 pretend pytest>=6.2.0 +tomli; python_version<'3.11' +tomli_w diff --git a/tests/test_pylock.py b/tests/test_pylock.py new file mode 100644 index 000000000..13ba8b92a --- /dev/null +++ b/tests/test_pylock.py @@ -0,0 +1,484 @@ +from __future__ import annotations + +import sys +from datetime import datetime +from pathlib import Path +from textwrap import dedent +from typing import Any + +import pytest +import tomli_w + +from packaging.markers import Marker +from packaging.pylock import ( + Package, + PackageDirectory, + PackageVcs, + PackageWheel, + PylockRequiredKeyError, + PylockUnsupportedVersionError, + PylockValidationError, + _exactly_one, + from_dict, + is_valid_pylock_path, +) +from packaging.specifiers import SpecifierSet +from packaging.version import Version + +if sys.version_info >= (3, 11): + import tomllib +else: + import tomli as tomllib + + +@pytest.mark.parametrize( + "file_name,valid", + [ + ("pylock.toml", True), + ("pylock.spam.toml", True), + ("pylock.json", False), + ("pylock..toml", False), + ], +) +def test_pylock_file_name(file_name: str, valid: bool) -> None: + assert is_valid_pylock_path(Path(file_name)) is valid + + +def test_exactly_one() -> None: + assert not _exactly_one([]) + assert not _exactly_one([False]) + assert not _exactly_one([False, False]) + assert not _exactly_one([True, True]) + assert _exactly_one([True]) + assert _exactly_one([True, False]) + + +# This is the PEP 751 example, with the following differences: +# - a minor modification to the 'environments' field to use double quotes +# instead of single quotes, since that is what 'packaging' does when +# serializing markers; +# - added an index field, which was not demonstrated in the PEP 751 example. + +PEP751_EXAMPLE = dedent( + """\ + lock-version = '1.0' + environments = ["sys_platform == \\"win32\\"", "sys_platform == \\"linux\\""] + requires-python = '==3.12' + created-by = 'mousebender' + + [[packages]] + name = 'attrs' + version = '25.1.0' + requires-python = '>=3.8' + wheels = [ + {name = 'attrs-25.1.0-py3-none-any.whl', upload-time = 2025-01-25T11:30:10.164985+00:00, url = 'https://files.pythonhosted.org/packages/fc/30/d4986a882011f9df997a55e6becd864812ccfcd821d64aac8570ee39f719/attrs-25.1.0-py3-none-any.whl', size = 63152, hashes = {sha256 = 'c75a69e28a550a7e93789579c22aa26b0f5b83b75dc4e08fe092980051e1090a'}}, + ] + [[packages.attestation-identities]] + environment = 'release-pypi' + kind = 'GitHub' + repository = 'python-attrs/attrs' + workflow = 'pypi-package.yml' + + [[packages]] + name = 'cattrs' + version = '24.1.2' + requires-python = '>=3.8' + dependencies = [ + {name = 'attrs'}, + ] + index = 'https://pypi.org/simple' + wheels = [ + {name = 'cattrs-24.1.2-py3-none-any.whl', upload-time = 2024-09-22T14:58:34.812643+00:00, url = 'https://files.pythonhosted.org/packages/c8/d5/867e75361fc45f6de75fe277dd085627a9db5ebb511a87f27dc1396b5351/cattrs-24.1.2-py3-none-any.whl', size = 66446, hashes = {sha256 = '67c7495b760168d931a10233f979b28dc04daf853b30752246f4f8471c6d68d0'}}, + ] + + [[packages]] + name = 'numpy' + version = '2.2.3' + requires-python = '>=3.10' + wheels = [ + {name = 'numpy-2.2.3-cp312-cp312-win_amd64.whl', upload-time = 2025-02-13T16:51:21.821880+00:00, url = 'https://files.pythonhosted.org/packages/42/6e/55580a538116d16ae7c9aa17d4edd56e83f42126cb1dfe7a684da7925d2c/numpy-2.2.3-cp312-cp312-win_amd64.whl', size = 12626357, hashes = {sha256 = '83807d445817326b4bcdaaaf8e8e9f1753da04341eceec705c001ff342002e5d'}}, + {name = 'numpy-2.2.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl', upload-time = 2025-02-13T16:50:00.079662+00:00, url = 'https://files.pythonhosted.org/packages/39/04/78d2e7402fb479d893953fb78fa7045f7deb635ec095b6b4f0260223091a/numpy-2.2.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl', size = 16116679, hashes = {sha256 = '3b787adbf04b0db1967798dba8da1af07e387908ed1553a0d6e74c084d1ceafe'}}, + ] + + [tool.mousebender] + command = ['.', 'lock', '--platform', 'cpython3.12-windows-x64', '--platform', 'cpython3.12-manylinux2014-x64', 'cattrs', 'numpy'] + run-on = 2025-03-06T12:28:57.760769 + """ # noqa: E501 +) + + +def test_toml_roundtrip() -> None: + pylock_dict = tomllib.loads(PEP751_EXAMPLE) + pylock = from_dict(pylock_dict) + # Check that the roundrip via Pylock dataclasses produces the same toml + # output, modulo TOML serialization differences. + assert tomli_w.dumps(pylock.to_dict()) == tomli_w.dumps(pylock_dict) + + +@pytest.mark.parametrize("version", ["1.0", "1.1"]) +def test_pylock_version(version: str) -> None: + data = { + "lock-version": version, + "created-by": "pip", + "packages": [], + } + from_dict(data) + + +def test_pylock_unsupported_version() -> None: + data = { + "lock-version": "2.0", + "created-by": "pip", + "packages": [], + } + with pytest.raises(PylockUnsupportedVersionError): + from_dict(data) + + +def test_pylock_invalid_version() -> None: + data = { + "lock-version": "2.x", + "created-by": "pip", + "packages": [], + } + with pytest.raises(PylockValidationError) as exc_info: + from_dict(data) + assert str(exc_info.value) == "Error in 'lock-version': Invalid version: '2.x'" + + +def test_pylock_unexpected_type() -> None: + data = { + "lock-version": 1.0, + "created-by": "pip", + "packages": [], + } + with pytest.raises(PylockValidationError) as exc_info: + from_dict(data) + assert str(exc_info.value) == ( + "'lock-version' has unexpected type float (expected str)" + ) + + +def test_pylock_missing_version() -> None: + data = { + "created-by": "pip", + "packages": [], + } + with pytest.raises(PylockRequiredKeyError) as exc_info: + from_dict(data) + assert str(exc_info.value) == "Missing required field 'lock-version'" + + +def test_pylock_missing_created_by() -> None: + data = { + "lock-version": "1.0", + "packages": [], + } + with pytest.raises(PylockRequiredKeyError) as exc_info: + from_dict(data) + assert str(exc_info.value) == "Missing required field 'created-by'" + + +def test_pylock_missing_packages() -> None: + data = { + "lock-version": "1.0", + "created-by": "uv", + } + with pytest.raises(PylockRequiredKeyError) as exc_info: + from_dict(data) + assert str(exc_info.value) == "Missing required field 'packages'" + + +def test_pylock_packages_without_dist() -> None: + data = { + "lock-version": "1.0", + "created-by": "pip", + "packages": [{"name": "example", "version": "1.0"}], + } + with pytest.raises(PylockValidationError) as exc_info: + from_dict(data) + assert str(exc_info.value) == ( + "Error in item 0 of 'packages': " + "Exactly one of vcs, directory, archive must be set " + "if sdist and wheels are not set" + ) + + +def test_pylock_packages_with_dist_and_archive() -> None: + data = { + "lock-version": "1.0", + "created-by": "pip", + "packages": [ + { + "name": "example", + "version": "1.0", + "archive": { + "path": "example.tar.gz", + "hashes": {"sha256": "f" * 40}, + }, + "sdist": { + "path": "example.tar.gz", + "hashes": {"sha256": "f" * 40}, + }, + } + ], + } + with pytest.raises(PylockValidationError) as exc_info: + from_dict(data) + assert str(exc_info.value) == ( + "Error in item 0 of 'packages': " + "None of vcs, directory, archive must be set " + "if sdist or wheels are set" + ) + + +def test_pylock_basic_package() -> None: + data = { + "lock-version": "1.0", + "created-by": "pip", + "requires-python": ">=3.10", + "environments": ['os_name == "posix"'], + "packages": [ + { + "name": "example", + "version": "1.0", + "marker": 'os_name == "posix"', + "requires-python": "!=3.10.1,>=3.10", + "directory": { + "path": ".", + "editable": False, + }, + } + ], + } + pylock = from_dict(data) + assert pylock.environments == [Marker('os_name == "posix"')] + package = pylock.packages[0] + assert package.version == Version("1.0") + assert package.marker == Marker('os_name == "posix"') + assert package.requires_python == SpecifierSet(">=3.10, !=3.10.1") + assert pylock.to_dict() == data + + +def test_pylock_vcs_package() -> None: + data = { + "lock-version": "1.0", + "created-by": "pip", + "packages": [ + { + "name": "packaging", + "vcs": { + "type": "git", + "url": "https://githhub/pypa/packaging", + "commit-id": "...", + }, + } + ], + } + pylock = from_dict(data) + assert pylock.to_dict() == data + + +def test_pylock_invalid_archive() -> None: + data = { + "lock-version": "1.0", + "created-by": "pip", + "requires-python": ">=3.10", + "environments": ['os_name == "posix"'], + "packages": [ + { + "name": "example", + "archive": { + # "path": "example.tar.gz", + "hashes": {"sha256": "f" * 40}, + }, + } + ], + } + with pytest.raises(PylockValidationError) as exc_info: + from_dict(data) + assert str(exc_info.value) == ( + "Error in item 0 of 'packages': " + "Error in 'archive': " + "path or url must be provided" + ) + + +def test_pylock_invalid_vcs() -> None: + with pytest.raises(PylockValidationError) as exc_info: + PackageVcs(type="git", url=None, path=None, commit_id="f" * 40) + assert str(exc_info.value) == "path or url must be provided" + + +def test_pylock_invalid_wheel() -> None: + data = { + "lock-version": "1.0", + "created-by": "pip", + "requires-python": ">=3.10", + "environments": ['os_name == "posix"'], + "packages": [ + { + "name": "example", + "wheels": [ + { + "name": "example-1.0-py3-none-any.whl", + "path": "./example-1.0-py3-none-any.whl", + # "hashes": {"sha256": "f" * 40}, + } + ], + } + ], + } + with pytest.raises(PylockValidationError) as exc_info: + from_dict(data) + assert str(exc_info.value) == ( + "Error in item 0 of 'packages': " + "Error in item 0 of 'wheels': " + "Missing required field 'hashes'" + ) + + +def test_pylock_invalid_environments() -> None: + data = { + "lock-version": "1.0", + "created-by": "pip", + "environments": [ + 'os_name == "posix"', + 'invalid_marker == "..."', + ], + "packages": [], + } + with pytest.raises(PylockValidationError) as exc_info: + from_dict(data) + assert str(exc_info.value) == ( + "Error in item 1 of 'environments': " + "Expected a marker variable or quoted string\n" + ' invalid_marker == "..."\n' + " ^" + ) + + +def test_pylock_invalid_environments_type() -> None: + data = { + "lock-version": "1.0", + "created-by": "pip", + "environments": [ + 'os_name == "posix"', + 1, + ], + "packages": [], + } + with pytest.raises(PylockValidationError) as exc_info: + from_dict(data) + assert str(exc_info.value) == ( + "Item 1 of 'environments' has unexpected type int (expected str)" + ) + + +def test_pylock_extras_and_groups() -> None: + data = { + "lock-version": "1.0", + "created-by": "pip", + "extras": ["feat1", "feat2"], + "dependency-groups": ["dev", "docs"], + "default-groups": ["dev"], + "packages": [], + } + pylock = from_dict(data) + assert pylock.extras == ["feat1", "feat2"] + assert pylock.dependency_groups == ["dev", "docs"] + assert pylock.default_groups == ["dev"] + + +def test_pylock_tool() -> None: + data = { + "lock-version": "1.0", + "created-by": "pip", + "packages": [ + { + "name": "example", + "sdist": { + "name": "example-1.0.tar.gz", + "path": "./example-1.0.tar.gz", + "upload-time": datetime(2023, 10, 1, 0, 0), + "hashes": {"sha256": "f" * 40}, + }, + "tool": {"pip": {"foo": "bar"}}, + } + ], + "tool": {"pip": {"version": "25.2"}}, + } + pylock = from_dict(data) + assert pylock.tool == {"pip": {"version": "25.2"}} + package = pylock.packages[0] + assert package.tool == {"pip": {"foo": "bar"}} + + +def test_pylock_package_not_a_table() -> None: + data = { + "lock-version": "1.0", + "created-by": "pip", + "packages": ["example"], + } + with pytest.raises(PylockValidationError) as exc_info: + from_dict(data) + assert str(exc_info.value) == "Item 0 of 'packages' is not a table" + + +@pytest.mark.parametrize( + "hashes,expected_error", + [ + ( + { + "sha256": "f" * 40, + "md5": 1, + }, + "Hash values must be strings", + ), + ( + {}, + "At least one hash must be provided", + ), + ( + "sha256:...", + "'hashes' has unexpected type str (expected Mapping)", + ), + ], +) +def test_hash_validation(hashes: dict[str, Any], expected_error: str) -> None: + with pytest.raises(PylockValidationError) as exc_info: + PackageWheel._from_dict( + dict( + name="example-1.0-py3-none-any.whl", + upload_time=None, + url="https://example.com/example-1.0-py3-none-any.whl", + path=None, + size=None, + hashes=hashes, + ) + ) + assert str(exc_info.value) == expected_error + + +def test_package_name_validation() -> None: + with pytest.raises(PylockValidationError) as exc_info: + Package(name="Example") + assert str(exc_info.value) == "Package name 'Example' is not normalized" + + +def test_is_direct() -> None: + direct_package = Package( + name="example", + directory=PackageDirectory(path="."), + ) + assert direct_package.is_direct + wheel_package = Package( + name="example", + wheels=[ + PackageWheel( + url="https://example.com/example-1.0-py3-none-any.whl", + hashes={"sha256": "f" * 40}, + ) + ], + ) + assert not wheel_package.is_direct From a28379f960a7a551efde009acd124e79c0163502 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Wed, 7 May 2025 11:26:11 +0200 Subject: [PATCH 02/24] pylock: clarify error messages --- src/packaging/pylock.py | 51 ++++++++++++++++++++++++++++++++--------- tests/test_pylock.py | 38 +++++++++++++++--------------- 2 files changed, 58 insertions(+), 31 deletions(-) diff --git a/src/packaging/pylock.py b/src/packaging/pylock.py index 365a2d56b..7b3370b1e 100644 --- a/src/packaging/pylock.py +++ b/src/packaging/pylock.py @@ -90,8 +90,9 @@ def _get(d: Mapping[str, Any], expected_type: type[T], key: str) -> T | None: return None if not isinstance(value, expected_type): raise PylockValidationError( - f"{key!r} has unexpected type {type(value).__name__} " - f"(expected {expected_type.__name__})" + f"Unexpected type {type(value).__name__} " + f"(expected {expected_type.__name__})", + context=key, ) return value @@ -114,8 +115,9 @@ def _get_sequence( for i, item in enumerate(value): if not isinstance(item, expected_item_type): raise PylockValidationError( - f"Item {i} of {key!r} has unexpected type {type(item).__name__} " - f"(expected {expected_item_type.__name__})" + f"Unexpected type {type(item).__name__} " + f"(expected {expected_item_type.__name__})", + context=f"{key}[{i}]", ) return value @@ -136,7 +138,7 @@ def _get_as( try: return target_type(value) except Exception as e: - raise PylockValidationError(f"Error in {key!r}: {e}") from e + raise PylockValidationError(e, context=key) from e def _get_required_as( @@ -168,7 +170,7 @@ def _get_sequence_as( try: result.append(target_item_type(item)) except Exception as e: - raise PylockValidationError(f"Error in item {i} of {key!r}: {e}") from e + raise PylockValidationError(e, context=f"{key}[{i}]") from e return result @@ -182,7 +184,7 @@ def _get_object( try: return target_type._from_dict(value) except Exception as e: - raise PylockValidationError(f"Error in {key!r}: {e}") from e + raise PylockValidationError(e, context=key) from e def _get_sequence_of_objects( @@ -195,11 +197,14 @@ def _get_sequence_of_objects( result = [] for i, item in enumerate(value): if not isinstance(item, Mapping): - raise PylockValidationError(f"Item {i} of {key!r} is not a table") + raise PylockValidationError( + f"Unexpected type {type(item).__name__} (expected Mapping)", + context=f"{key}[{i}]", + ) try: result.append(target_item_type._from_dict(item)) except Exception as e: - raise PylockValidationError(f"Error in item {i} of {key!r}: {e}") from e + raise PylockValidationError(e, context=f"{key}[{i}]") from e return result @@ -236,12 +241,36 @@ def _validate_hashes(hashes: Mapping[str, Any]) -> None: class PylockValidationError(Exception): - pass + context: str | None = None + message: str + + def __init__( + self, + cause: str | Exception, + *, + context: str | None = None, + ) -> None: + if isinstance(cause, PylockValidationError): + if cause.context: + self.context = ( + f"{context}.{cause.context}" if context else cause.context + ) + else: + self.context = context + self.message = cause.message + else: + self.context = context + self.message = str(cause) + + def __str__(self) -> str: + if self.context: + return f"{self.message} in '{self.context}'" + return self.message class PylockRequiredKeyError(PylockValidationError): def __init__(self, key: str) -> None: - super().__init__(f"Missing required field {key!r}") + super().__init__("Missing required value", context=key) class PylockUnsupportedVersionError(PylockValidationError): diff --git a/tests/test_pylock.py b/tests/test_pylock.py index 13ba8b92a..a136810bb 100644 --- a/tests/test_pylock.py +++ b/tests/test_pylock.py @@ -143,7 +143,7 @@ def test_pylock_invalid_version() -> None: } with pytest.raises(PylockValidationError) as exc_info: from_dict(data) - assert str(exc_info.value) == "Error in 'lock-version': Invalid version: '2.x'" + assert str(exc_info.value) == "Invalid version: '2.x' in 'lock-version'" def test_pylock_unexpected_type() -> None: @@ -155,7 +155,7 @@ def test_pylock_unexpected_type() -> None: with pytest.raises(PylockValidationError) as exc_info: from_dict(data) assert str(exc_info.value) == ( - "'lock-version' has unexpected type float (expected str)" + "Unexpected type float (expected str) in 'lock-version'" ) @@ -166,7 +166,7 @@ def test_pylock_missing_version() -> None: } with pytest.raises(PylockRequiredKeyError) as exc_info: from_dict(data) - assert str(exc_info.value) == "Missing required field 'lock-version'" + assert str(exc_info.value) == "Missing required value in 'lock-version'" def test_pylock_missing_created_by() -> None: @@ -176,7 +176,7 @@ def test_pylock_missing_created_by() -> None: } with pytest.raises(PylockRequiredKeyError) as exc_info: from_dict(data) - assert str(exc_info.value) == "Missing required field 'created-by'" + assert str(exc_info.value) == "Missing required value in 'created-by'" def test_pylock_missing_packages() -> None: @@ -186,7 +186,7 @@ def test_pylock_missing_packages() -> None: } with pytest.raises(PylockRequiredKeyError) as exc_info: from_dict(data) - assert str(exc_info.value) == "Missing required field 'packages'" + assert str(exc_info.value) == "Missing required value in 'packages'" def test_pylock_packages_without_dist() -> None: @@ -198,9 +198,9 @@ def test_pylock_packages_without_dist() -> None: with pytest.raises(PylockValidationError) as exc_info: from_dict(data) assert str(exc_info.value) == ( - "Error in item 0 of 'packages': " "Exactly one of vcs, directory, archive must be set " - "if sdist and wheels are not set" + "if sdist and wheels are not set " + "in 'packages[0]'" ) @@ -226,9 +226,9 @@ def test_pylock_packages_with_dist_and_archive() -> None: with pytest.raises(PylockValidationError) as exc_info: from_dict(data) assert str(exc_info.value) == ( - "Error in item 0 of 'packages': " "None of vcs, directory, archive must be set " - "if sdist or wheels are set" + "if sdist or wheels are set " + "in 'packages[0]'" ) @@ -298,9 +298,7 @@ def test_pylock_invalid_archive() -> None: with pytest.raises(PylockValidationError) as exc_info: from_dict(data) assert str(exc_info.value) == ( - "Error in item 0 of 'packages': " - "Error in 'archive': " - "path or url must be provided" + "path or url must be provided in 'packages[0].archive'" ) @@ -332,9 +330,7 @@ def test_pylock_invalid_wheel() -> None: with pytest.raises(PylockValidationError) as exc_info: from_dict(data) assert str(exc_info.value) == ( - "Error in item 0 of 'packages': " - "Error in item 0 of 'wheels': " - "Missing required field 'hashes'" + "Missing required value in 'packages[0].wheels[0].hashes'" ) @@ -351,10 +347,10 @@ def test_pylock_invalid_environments() -> None: with pytest.raises(PylockValidationError) as exc_info: from_dict(data) assert str(exc_info.value) == ( - "Error in item 1 of 'environments': " "Expected a marker variable or quoted string\n" ' invalid_marker == "..."\n' - " ^" + " ^ " + "in 'environments[1]'" ) @@ -371,7 +367,7 @@ def test_pylock_invalid_environments_type() -> None: with pytest.raises(PylockValidationError) as exc_info: from_dict(data) assert str(exc_info.value) == ( - "Item 1 of 'environments' has unexpected type int (expected str)" + "Unexpected type int (expected str) in 'environments[1]'" ) @@ -422,7 +418,9 @@ def test_pylock_package_not_a_table() -> None: } with pytest.raises(PylockValidationError) as exc_info: from_dict(data) - assert str(exc_info.value) == "Item 0 of 'packages' is not a table" + assert str(exc_info.value) == ( + "Unexpected type str (expected Mapping) in 'packages[0]'" + ) @pytest.mark.parametrize( @@ -441,7 +439,7 @@ def test_pylock_package_not_a_table() -> None: ), ( "sha256:...", - "'hashes' has unexpected type str (expected Mapping)", + "Unexpected type str (expected Mapping) in 'hashes'", ), ], ) From c1601cf779b95065b8ea5ce0fd740e8d2cda13e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Wed, 7 May 2025 11:42:22 +0200 Subject: [PATCH 03/24] pylock: simplify --- src/packaging/pylock.py | 16 +++------------- tests/test_pylock.py | 10 ---------- 2 files changed, 3 insertions(+), 23 deletions(-) diff --git a/src/packaging/pylock.py b/src/packaging/pylock.py index 7b3370b1e..209e94c1c 100644 --- a/src/packaging/pylock.py +++ b/src/packaging/pylock.py @@ -4,7 +4,7 @@ import logging import re import sys -from collections.abc import Iterable, Mapping, Sequence +from collections.abc import Mapping, Sequence from dataclasses import dataclass from datetime import datetime from pathlib import Path @@ -218,16 +218,6 @@ def _get_required_list_of_objects( return result -def _exactly_one(iterable: Iterable[object]) -> bool: - found = False - for item in iterable: - if item: - if found: - return False - found = True - return found - - def _validate_path_url(path: str | None, url: str | None) -> None: if not path and not url: raise PylockValidationError("path or url must be provided") @@ -522,14 +512,14 @@ def __init__( if not is_normalized_name(self.name): raise PylockValidationError(f"Package name {self.name!r} is not normalized") if self.sdist or self.wheels: - if any([self.vcs, self.directory, self.archive]): + if self.vcs or self.directory or self.archive: raise PylockValidationError( "None of vcs, directory, archive " "must be set if sdist or wheels are set" ) else: # no sdist nor wheels - if not _exactly_one([self.vcs, self.directory, self.archive]): + if not (bool(self.vcs) ^ bool(self.directory) ^ bool(self.archive)): raise PylockValidationError( "Exactly one of vcs, directory, archive must be set " "if sdist and wheels are not set" diff --git a/tests/test_pylock.py b/tests/test_pylock.py index a136810bb..0cc7c1c3e 100644 --- a/tests/test_pylock.py +++ b/tests/test_pylock.py @@ -18,7 +18,6 @@ PylockRequiredKeyError, PylockUnsupportedVersionError, PylockValidationError, - _exactly_one, from_dict, is_valid_pylock_path, ) @@ -44,15 +43,6 @@ def test_pylock_file_name(file_name: str, valid: bool) -> None: assert is_valid_pylock_path(Path(file_name)) is valid -def test_exactly_one() -> None: - assert not _exactly_one([]) - assert not _exactly_one([False]) - assert not _exactly_one([False, False]) - assert not _exactly_one([True, True]) - assert _exactly_one([True]) - assert _exactly_one([True, False]) - - # This is the PEP 751 example, with the following differences: # - a minor modification to the 'environments' field to use double quotes # instead of single quotes, since that is what 'packaging' does when From 06351c312fe226206343f237d317accecd310b87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Fri, 9 May 2025 12:08:51 +0200 Subject: [PATCH 04/24] pylock: make from_dict a classmethod again My idea was to have from_dict independent of the class to cope for future evolution, but if a new version of the standard can't be implemented as a subclass of Pylock, return type of from_dict would change, so it would be a breaking change. If/when a new version of the spec arrive, it will still be time to design the API evolution, depending of the actual changes. --- src/packaging/pylock.py | 7 +++---- tests/test_pylock.py | 40 ++++++++++++++++++++-------------------- 2 files changed, 23 insertions(+), 24 deletions(-) diff --git a/src/packaging/pylock.py b/src/packaging/pylock.py index 209e94c1c..4e194acb8 100644 --- a/src/packaging/pylock.py +++ b/src/packaging/pylock.py @@ -611,7 +611,6 @@ def _from_dict(cls, d: Mapping[str, Any]) -> Self: tool=_get(d, Mapping, "tool"), # type: ignore[type-abstract] ) - -def from_dict(d: Mapping[str, Any]) -> Pylock: - """Create a Pylock object from a dictionary.""" - return Pylock._from_dict(d) + @classmethod + def from_dict(cls, d: Mapping[str, Any]) -> Self: + return cls._from_dict(d) diff --git a/tests/test_pylock.py b/tests/test_pylock.py index 0cc7c1c3e..af981b4b6 100644 --- a/tests/test_pylock.py +++ b/tests/test_pylock.py @@ -15,10 +15,10 @@ PackageDirectory, PackageVcs, PackageWheel, + Pylock, PylockRequiredKeyError, PylockUnsupportedVersionError, PylockValidationError, - from_dict, is_valid_pylock_path, ) from packaging.specifiers import SpecifierSet @@ -99,7 +99,7 @@ def test_pylock_file_name(file_name: str, valid: bool) -> None: def test_toml_roundtrip() -> None: pylock_dict = tomllib.loads(PEP751_EXAMPLE) - pylock = from_dict(pylock_dict) + pylock = Pylock.from_dict(pylock_dict) # Check that the roundrip via Pylock dataclasses produces the same toml # output, modulo TOML serialization differences. assert tomli_w.dumps(pylock.to_dict()) == tomli_w.dumps(pylock_dict) @@ -112,7 +112,7 @@ def test_pylock_version(version: str) -> None: "created-by": "pip", "packages": [], } - from_dict(data) + Pylock.from_dict(data) def test_pylock_unsupported_version() -> None: @@ -122,7 +122,7 @@ def test_pylock_unsupported_version() -> None: "packages": [], } with pytest.raises(PylockUnsupportedVersionError): - from_dict(data) + Pylock.from_dict(data) def test_pylock_invalid_version() -> None: @@ -132,7 +132,7 @@ def test_pylock_invalid_version() -> None: "packages": [], } with pytest.raises(PylockValidationError) as exc_info: - from_dict(data) + Pylock.from_dict(data) assert str(exc_info.value) == "Invalid version: '2.x' in 'lock-version'" @@ -143,7 +143,7 @@ def test_pylock_unexpected_type() -> None: "packages": [], } with pytest.raises(PylockValidationError) as exc_info: - from_dict(data) + Pylock.from_dict(data) assert str(exc_info.value) == ( "Unexpected type float (expected str) in 'lock-version'" ) @@ -155,7 +155,7 @@ def test_pylock_missing_version() -> None: "packages": [], } with pytest.raises(PylockRequiredKeyError) as exc_info: - from_dict(data) + Pylock.from_dict(data) assert str(exc_info.value) == "Missing required value in 'lock-version'" @@ -165,7 +165,7 @@ def test_pylock_missing_created_by() -> None: "packages": [], } with pytest.raises(PylockRequiredKeyError) as exc_info: - from_dict(data) + Pylock.from_dict(data) assert str(exc_info.value) == "Missing required value in 'created-by'" @@ -175,7 +175,7 @@ def test_pylock_missing_packages() -> None: "created-by": "uv", } with pytest.raises(PylockRequiredKeyError) as exc_info: - from_dict(data) + Pylock.from_dict(data) assert str(exc_info.value) == "Missing required value in 'packages'" @@ -186,7 +186,7 @@ def test_pylock_packages_without_dist() -> None: "packages": [{"name": "example", "version": "1.0"}], } with pytest.raises(PylockValidationError) as exc_info: - from_dict(data) + Pylock.from_dict(data) assert str(exc_info.value) == ( "Exactly one of vcs, directory, archive must be set " "if sdist and wheels are not set " @@ -214,7 +214,7 @@ def test_pylock_packages_with_dist_and_archive() -> None: ], } with pytest.raises(PylockValidationError) as exc_info: - from_dict(data) + Pylock.from_dict(data) assert str(exc_info.value) == ( "None of vcs, directory, archive must be set " "if sdist or wheels are set " @@ -241,7 +241,7 @@ def test_pylock_basic_package() -> None: } ], } - pylock = from_dict(data) + pylock = Pylock.from_dict(data) assert pylock.environments == [Marker('os_name == "posix"')] package = pylock.packages[0] assert package.version == Version("1.0") @@ -265,7 +265,7 @@ def test_pylock_vcs_package() -> None: } ], } - pylock = from_dict(data) + pylock = Pylock.from_dict(data) assert pylock.to_dict() == data @@ -286,7 +286,7 @@ def test_pylock_invalid_archive() -> None: ], } with pytest.raises(PylockValidationError) as exc_info: - from_dict(data) + Pylock.from_dict(data) assert str(exc_info.value) == ( "path or url must be provided in 'packages[0].archive'" ) @@ -318,7 +318,7 @@ def test_pylock_invalid_wheel() -> None: ], } with pytest.raises(PylockValidationError) as exc_info: - from_dict(data) + Pylock.from_dict(data) assert str(exc_info.value) == ( "Missing required value in 'packages[0].wheels[0].hashes'" ) @@ -335,7 +335,7 @@ def test_pylock_invalid_environments() -> None: "packages": [], } with pytest.raises(PylockValidationError) as exc_info: - from_dict(data) + Pylock.from_dict(data) assert str(exc_info.value) == ( "Expected a marker variable or quoted string\n" ' invalid_marker == "..."\n' @@ -355,7 +355,7 @@ def test_pylock_invalid_environments_type() -> None: "packages": [], } with pytest.raises(PylockValidationError) as exc_info: - from_dict(data) + Pylock.from_dict(data) assert str(exc_info.value) == ( "Unexpected type int (expected str) in 'environments[1]'" ) @@ -370,7 +370,7 @@ def test_pylock_extras_and_groups() -> None: "default-groups": ["dev"], "packages": [], } - pylock = from_dict(data) + pylock = Pylock.from_dict(data) assert pylock.extras == ["feat1", "feat2"] assert pylock.dependency_groups == ["dev", "docs"] assert pylock.default_groups == ["dev"] @@ -394,7 +394,7 @@ def test_pylock_tool() -> None: ], "tool": {"pip": {"version": "25.2"}}, } - pylock = from_dict(data) + pylock = Pylock.from_dict(data) assert pylock.tool == {"pip": {"version": "25.2"}} package = pylock.packages[0] assert package.tool == {"pip": {"foo": "bar"}} @@ -407,7 +407,7 @@ def test_pylock_package_not_a_table() -> None: "packages": ["example"], } with pytest.raises(PylockValidationError) as exc_info: - from_dict(data) + Pylock.from_dict(data) assert str(exc_info.value) == ( "Unexpected type str (expected Mapping) in 'packages[0]'" ) From 5540bc43f8aa504f3ad147be1fb135e308db83a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sat, 26 Jul 2025 18:50:58 +0200 Subject: [PATCH 05/24] pylock: Apply suggestions from code review Co-authored-by: Brett Cannon --- src/packaging/pylock.py | 52 ++++++++++++++++++++--------------------- tests/test_pylock.py | 4 ++-- 2 files changed, 27 insertions(+), 29 deletions(-) diff --git a/src/packaging/pylock.py b/src/packaging/pylock.py index 4e194acb8..b584c046c 100644 --- a/src/packaging/pylock.py +++ b/src/packaging/pylock.py @@ -84,9 +84,8 @@ def _toml_dict_factory(data: list[tuple[str, Any]]) -> dict[str, Any]: def _get(d: Mapping[str, Any], expected_type: type[T], key: str) -> T | None: - """Get value from dictionary and verify expected type.""" - value = d.get(key) - if value is None: + """Get a value from the dictionary and verify it's the expected type.""" + if (value := d.get(key)) is None: return None if not isinstance(value, expected_type): raise PylockValidationError( @@ -98,9 +97,8 @@ def _get(d: Mapping[str, Any], expected_type: type[T], key: str) -> T | None: def _get_required(d: Mapping[str, Any], expected_type: type[T], key: str) -> T: - """Get required value from dictionary and verify expected type.""" - value = _get(d, expected_type, key) - if value is None: + """Get a required value from the dictionary and verify it's the expected type.""" + if (value := _get(d, expected_type, key)) is None: raise PylockRequiredKeyError(key) return value @@ -108,9 +106,8 @@ def _get_required(d: Mapping[str, Any], expected_type: type[T], key: str) -> T: def _get_sequence( d: Mapping[str, Any], expected_item_type: type[T], key: str ) -> Sequence[T] | None: - """Get list value from dictionary and verify expected items type.""" - value = _get(d, Sequence, key) # type: ignore[type-abstract] - if value is None: + """Get a list value from the dictionary and verify it's the expected items type.""" + if (value := _get(d, Sequence, key)) is None: # type: ignore[type-abstract] return None for i, item in enumerate(value): if not isinstance(item, expected_item_type): @@ -128,12 +125,12 @@ def _get_as( target_type: type[SingleArgConstructorT], key: str, ) -> SingleArgConstructorT | None: - """Get value from dictionary, verify expected type, convert to target type. + """Get a value from the dictionary, verify it's the expected type, + and convert to the target type. This assumes the target_type constructor accepts the value. """ - value = _get(d, expected_type, key) - if value is None: + if (value := _get(d, expected_type, key)) is None: return None try: return target_type(value) @@ -147,10 +144,9 @@ def _get_required_as( target_type: type[SingleArgConstructorT], key: str, ) -> SingleArgConstructorT: - """Get required value from dictionary, verify expected type, - convert to target type.""" - value = _get_as(d, expected_type, target_type, key) - if value is None: + """Get a required value from the dict, verify it's the expected type, + and convert to the target type.""" + if (value := _get_as(d, expected_type, target_type, key)) is None: raise PylockRequiredKeyError(key) return value @@ -162,22 +158,23 @@ def _get_sequence_as( key: str, ) -> Sequence[SingleArgConstructorT] | None: """Get list value from dictionary and verify expected items type.""" - value = _get_sequence(d, expected_item_type, key) - if value is None: + if (value := _get_sequence(d, expected_item_type, key)) is None: return None result = [] for i, item in enumerate(value): try: - result.append(target_item_type(item)) + typed_item = target_item_type(item) except Exception as e: raise PylockValidationError(e, context=f"{key}[{i}]") from e + else: + result.append(typed_item) return result def _get_object( d: Mapping[str, Any], target_type: type[FromMappingProtocolT], key: str ) -> FromMappingProtocolT | None: - """Get dictionary value from dictionary and convert to dataclass.""" + """Get a dictionary value from the dictionary and convert it to a dataclass.""" value = _get(d, Mapping, key) # type: ignore[type-abstract] if value is None: return None @@ -190,9 +187,8 @@ def _get_object( def _get_sequence_of_objects( d: Mapping[str, Any], target_item_type: type[FromMappingProtocolT], key: str ) -> Sequence[FromMappingProtocolT] | None: - """Get list value from dictionary and convert items to dataclass.""" - value = _get(d, Sequence, key) # type: ignore[type-abstract] - if value is None: + """Get a list value from the dictionary and convert its items to a dataclass.""" + if (value := _get(d, Sequence, key)) is None: # type: ignore[type-abstract] return None result = [] for i, item in enumerate(value): @@ -202,18 +198,20 @@ def _get_sequence_of_objects( context=f"{key}[{i}]", ) try: - result.append(target_item_type._from_dict(item)) + typed_item = target_item_type._from_dict(item) except Exception as e: raise PylockValidationError(e, context=f"{key}[{i}]") from e + else: + result.append(typed_item) return result def _get_required_list_of_objects( d: Mapping[str, Any], target_type: type[FromMappingProtocolT], key: str ) -> Sequence[FromMappingProtocolT]: - """Get required list value from dictionary and convert items to dataclass.""" - result = _get_sequence_of_objects(d, target_type, key) - if result is None: + """Get a required list value from the dictionary and convert its items to a + dataclass.""" + if (result := _get_sequence_of_objects(d, target_type, key)) is None: raise PylockRequiredKeyError(key) return result diff --git a/tests/test_pylock.py b/tests/test_pylock.py index af981b4b6..8ba2015e6 100644 --- a/tests/test_pylock.py +++ b/tests/test_pylock.py @@ -100,7 +100,7 @@ def test_pylock_file_name(file_name: str, valid: bool) -> None: def test_toml_roundtrip() -> None: pylock_dict = tomllib.loads(PEP751_EXAMPLE) pylock = Pylock.from_dict(pylock_dict) - # Check that the roundrip via Pylock dataclasses produces the same toml + # Check that the roundrip via Pylock dataclasses produces the same TOML # output, modulo TOML serialization differences. assert tomli_w.dumps(pylock.to_dict()) == tomli_w.dumps(pylock_dict) @@ -311,7 +311,7 @@ def test_pylock_invalid_wheel() -> None: { "name": "example-1.0-py3-none-any.whl", "path": "./example-1.0-py3-none-any.whl", - # "hashes": {"sha256": "f" * 40}, + # Purposefully no "hashes" key. } ], } From f1fcc639fc4e5a764e12c66472884bd19d68dd0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sat, 26 Jul 2025 19:27:39 +0200 Subject: [PATCH 06/24] pylock: type name as NormalizedName Also make _get_*_as function more versatile by accepting a callable for the target type. --- src/packaging/pylock.py | 35 ++++++++++++++++++----------------- tests/test_pylock.py | 9 +++++---- 2 files changed, 23 insertions(+), 21 deletions(-) diff --git a/src/packaging/pylock.py b/src/packaging/pylock.py index b584c046c..adf7f27b8 100644 --- a/src/packaging/pylock.py +++ b/src/packaging/pylock.py @@ -11,6 +11,7 @@ from typing import ( TYPE_CHECKING, Any, + Callable, Protocol, TypeVar, ) @@ -40,6 +41,7 @@ ] T = TypeVar("T") +T2 = TypeVar("T2") class FromMappingProtocol(Protocol): # pragma: no cover @@ -50,12 +52,6 @@ def _from_dict(cls, d: Mapping[str, Any]) -> Self: ... FromMappingProtocolT = TypeVar("FromMappingProtocolT", bound=FromMappingProtocol) -class SingleArgConstructor(Protocol): # pragma: no cover - def __init__(self, value: Any) -> None: ... - - -SingleArgConstructorT = TypeVar("SingleArgConstructorT", bound=SingleArgConstructor) - PYLOCK_FILE_NAME_RE = re.compile(r"^pylock\.([^.]+)\.toml$") @@ -122,9 +118,9 @@ def _get_sequence( def _get_as( d: Mapping[str, Any], expected_type: type[T], - target_type: type[SingleArgConstructorT], + target_type: Callable[[T], T2], key: str, -) -> SingleArgConstructorT | None: +) -> T2 | None: """Get a value from the dictionary, verify it's the expected type, and convert to the target type. @@ -141,9 +137,9 @@ def _get_as( def _get_required_as( d: Mapping[str, Any], expected_type: type[T], - target_type: type[SingleArgConstructorT], + target_type: Callable[[T], T2], key: str, -) -> SingleArgConstructorT: +) -> T2: """Get a required value from the dict, verify it's the expected type, and convert to the target type.""" if (value := _get_as(d, expected_type, target_type, key)) is None: @@ -154,9 +150,9 @@ def _get_required_as( def _get_sequence_as( d: Mapping[str, Any], expected_item_type: type[T], - target_item_type: type[SingleArgConstructorT], + target_item_type: Callable[[T], T2], key: str, -) -> Sequence[SingleArgConstructorT] | None: +) -> Sequence[T2] | None: """Get list value from dictionary and verify expected items type.""" if (value := _get_sequence(d, expected_item_type, key)) is None: return None @@ -216,6 +212,13 @@ def _get_required_list_of_objects( return result +def _validate_normalized_name(name: str) -> NormalizedName: + """Validate that a string is a NormalizedName.""" + if not is_normalized_name(name): + raise PylockValidationError(f"Name {name!r} is not normalized") + return NormalizedName(name) + + def _validate_path_url(path: str | None, url: str | None) -> None: if not path and not url: raise PylockValidationError("path or url must be provided") @@ -478,7 +481,7 @@ class Package: def __init__( self, *, - name: str, + name: NormalizedName, version: Version | None = None, marker: Marker | None = None, requires_python: SpecifierSet | None = None, @@ -507,8 +510,6 @@ def __init__( object.__setattr__(self, "attestation_identities", attestation_identities) object.__setattr__(self, "tool", tool) # __post_init__ in Python 3.10+ - if not is_normalized_name(self.name): - raise PylockValidationError(f"Package name {self.name!r} is not normalized") if self.sdist or self.wheels: if self.vcs or self.directory or self.archive: raise PylockValidationError( @@ -526,7 +527,7 @@ def __init__( @classmethod def _from_dict(cls, d: Mapping[str, Any]) -> Self: package = cls( - name=_get_required(d, str, "name"), + name=_get_required_as(d, str, _validate_normalized_name, "name"), version=_get_as(d, str, Version, "version"), requires_python=_get_as(d, str, SpecifierSet, "requires-python"), dependencies=_get_sequence(d, Mapping, "dependencies"), # type: ignore[type-abstract] @@ -600,7 +601,7 @@ def _from_dict(cls, d: Mapping[str, Any]) -> Self: return cls( lock_version=_get_required_as(d, str, Version, "lock-version"), environments=_get_sequence_as(d, str, Marker, "environments"), - extras=_get_sequence(d, str, "extras"), + extras=_get_sequence_as(d, str, _str_to_normalized_name, "extras"), dependency_groups=_get_sequence(d, str, "dependency-groups"), default_groups=_get_sequence(d, str, "default-groups"), created_by=_get_required(d, str, "created-by"), diff --git a/tests/test_pylock.py b/tests/test_pylock.py index 8ba2015e6..bc675f6c0 100644 --- a/tests/test_pylock.py +++ b/tests/test_pylock.py @@ -22,6 +22,7 @@ is_valid_pylock_path, ) from packaging.specifiers import SpecifierSet +from packaging.utils import NormalizedName from packaging.version import Version if sys.version_info >= (3, 11): @@ -450,18 +451,18 @@ def test_hash_validation(hashes: dict[str, Any], expected_error: str) -> None: def test_package_name_validation() -> None: with pytest.raises(PylockValidationError) as exc_info: - Package(name="Example") - assert str(exc_info.value) == "Package name 'Example' is not normalized" + Package._from_dict({"name": "Example"}) + assert str(exc_info.value) == "Name 'Example' is not normalized in 'name'" def test_is_direct() -> None: direct_package = Package( - name="example", + name=NormalizedName("example"), directory=PackageDirectory(path="."), ) assert direct_package.is_direct wheel_package = Package( - name="example", + name=NormalizedName("example"), wheels=[ PackageWheel( url="https://example.com/example-1.0-py3-none-any.whl", From 5b6009cc142b95553d765bcd60c629c1e4890039 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sat, 26 Jul 2025 19:37:26 +0200 Subject: [PATCH 07/24] pylock: type extras as NormalizedName, and validate --- src/packaging/pylock.py | 4 ++-- tests/test_pylock.py | 13 +++++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/packaging/pylock.py b/src/packaging/pylock.py index adf7f27b8..ea87ea7fb 100644 --- a/src/packaging/pylock.py +++ b/src/packaging/pylock.py @@ -553,7 +553,7 @@ class Pylock: lock_version: Version environments: Sequence[Marker] | None # = None requires_python: SpecifierSet | None # = None - extras: Sequence[str] | None # = None + extras: Sequence[NormalizedName] | None # = None dependency_groups: Sequence[str] | None # = None default_groups: Sequence[str] | None # = None created_by: str @@ -601,7 +601,7 @@ def _from_dict(cls, d: Mapping[str, Any]) -> Self: return cls( lock_version=_get_required_as(d, str, Version, "lock-version"), environments=_get_sequence_as(d, str, Marker, "environments"), - extras=_get_sequence_as(d, str, _str_to_normalized_name, "extras"), + extras=_get_sequence_as(d, str, _validate_normalized_name, "extras"), dependency_groups=_get_sequence(d, str, "dependency-groups"), default_groups=_get_sequence(d, str, "default-groups"), created_by=_get_required(d, str, "created-by"), diff --git a/tests/test_pylock.py b/tests/test_pylock.py index bc675f6c0..197a0a319 100644 --- a/tests/test_pylock.py +++ b/tests/test_pylock.py @@ -455,6 +455,19 @@ def test_package_name_validation() -> None: assert str(exc_info.value) == "Name 'Example' is not normalized in 'name'" +def test_extras_name_validation() -> None: + with pytest.raises(PylockValidationError) as exc_info: + Pylock.from_dict( + { + "lock-version": "1.0", + "created-by": "pip", + "extras": ["extra", "Feature"], + "packages": [], + } + ) + assert str(exc_info.value) == "Name 'Feature' is not normalized in 'extras[1]'" + + def test_is_direct() -> None: direct_package = Package( name=NormalizedName("example"), From cde13847b11c59b53759304b67df88d037ed7721 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sun, 27 Jul 2025 11:32:49 +0200 Subject: [PATCH 08/24] pylock: move PEP751 example to a standalone file --- tests/pylock/pylock.spec-example.toml | 68 +++++++++++++++++++++++++++ tests/test_pylock.py | 60 ++--------------------- 2 files changed, 72 insertions(+), 56 deletions(-) create mode 100644 tests/pylock/pylock.spec-example.toml diff --git a/tests/pylock/pylock.spec-example.toml b/tests/pylock/pylock.spec-example.toml new file mode 100644 index 000000000..80d3f14ee --- /dev/null +++ b/tests/pylock/pylock.spec-example.toml @@ -0,0 +1,68 @@ +# This is the example from PEP 751, with the following differences: +# - a minor modification to the 'environments' field to use double quotes +# instead of single quotes, since that is what 'packaging' does when +# serializing markers; +# - added an index field, which was not demonstrated in the PEP 751 example. +# - removed spaces in require-python specifiers + +lock-version = '1.0' +environments = ['sys_platform == "win32"', 'sys_platform == "linux"'] +requires-python = '==3.12' +created-by = 'mousebender' + +[[packages]] +name = 'attrs' +version = '25.1.0' +requires-python = '>=3.8' + + [[packages.wheels]] + name = 'attrs-25.1.0-py3-none-any.whl' + upload-time = 2025-01-25T11:30:10.164985+00:00 + url = 'https://files.pythonhosted.org/packages/fc/30/d4986a882011f9df997a55e6becd864812ccfcd821d64aac8570ee39f719/attrs-25.1.0-py3-none-any.whl' + size = 63152 + hashes = {sha256 = 'c75a69e28a550a7e93789579c22aa26b0f5b83b75dc4e08fe092980051e1090a'} + + [[packages.attestation-identities]] + environment = 'release-pypi' + kind = 'GitHub' + repository = 'python-attrs/attrs' + workflow = 'pypi-package.yml' + +[[packages]] +name = 'cattrs' +version = '24.1.2' +requires-python = '>=3.8' +dependencies = [ + {name = 'attrs'}, +] +index = 'https://pypi.org/simple' + + [[packages.wheels]] + name = 'cattrs-24.1.2-py3-none-any.whl' + upload-time = 2024-09-22T14:58:34.812643+00:00 + url = 'https://files.pythonhosted.org/packages/c8/d5/867e75361fc45f6de75fe277dd085627a9db5ebb511a87f27dc1396b5351/cattrs-24.1.2-py3-none-any.whl' + size = 66446 + hashes = {sha256 = '67c7495b760168d931a10233f979b28dc04daf853b30752246f4f8471c6d68d0'} + +[[packages]] +name = 'numpy' +version = '2.2.3' +requires-python = '>=3.10' + + [[packages.wheels]] + name = 'numpy-2.2.3-cp312-cp312-win_amd64.whl' + upload-time = 2025-02-13T16:51:21.821880+00:00 + url = 'https://files.pythonhosted.org/packages/42/6e/55580a538116d16ae7c9aa17d4edd56e83f42126cb1dfe7a684da7925d2c/numpy-2.2.3-cp312-cp312-win_amd64.whl' + size = 12626357 + hashes = {sha256 = '83807d445817326b4bcdaaaf8e8e9f1753da04341eceec705c001ff342002e5d'} + + [[packages.wheels]] + name = 'numpy-2.2.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl' + upload-time = 2025-02-13T16:50:00.079662+00:00 + url = 'https://files.pythonhosted.org/packages/39/04/78d2e7402fb479d893953fb78fa7045f7deb635ec095b6b4f0260223091a/numpy-2.2.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl' + size = 16116679 + hashes = {sha256 = '3b787adbf04b0db1967798dba8da1af07e387908ed1553a0d6e74c084d1ceafe'} + +[tool.mousebender] +command = ['.', 'lock', '--platform', 'cpython3.12-windows-x64', '--platform', 'cpython3.12-manylinux2014-x64', 'cattrs', 'numpy'] +run-on = 2025-03-06T12:28:57.760769 diff --git a/tests/test_pylock.py b/tests/test_pylock.py index 197a0a319..e5ed9cc5b 100644 --- a/tests/test_pylock.py +++ b/tests/test_pylock.py @@ -3,7 +3,6 @@ import sys from datetime import datetime from pathlib import Path -from textwrap import dedent from typing import Any import pytest @@ -44,62 +43,11 @@ def test_pylock_file_name(file_name: str, valid: bool) -> None: assert is_valid_pylock_path(Path(file_name)) is valid -# This is the PEP 751 example, with the following differences: -# - a minor modification to the 'environments' field to use double quotes -# instead of single quotes, since that is what 'packaging' does when -# serializing markers; -# - added an index field, which was not demonstrated in the PEP 751 example. - -PEP751_EXAMPLE = dedent( - """\ - lock-version = '1.0' - environments = ["sys_platform == \\"win32\\"", "sys_platform == \\"linux\\""] - requires-python = '==3.12' - created-by = 'mousebender' - - [[packages]] - name = 'attrs' - version = '25.1.0' - requires-python = '>=3.8' - wheels = [ - {name = 'attrs-25.1.0-py3-none-any.whl', upload-time = 2025-01-25T11:30:10.164985+00:00, url = 'https://files.pythonhosted.org/packages/fc/30/d4986a882011f9df997a55e6becd864812ccfcd821d64aac8570ee39f719/attrs-25.1.0-py3-none-any.whl', size = 63152, hashes = {sha256 = 'c75a69e28a550a7e93789579c22aa26b0f5b83b75dc4e08fe092980051e1090a'}}, - ] - [[packages.attestation-identities]] - environment = 'release-pypi' - kind = 'GitHub' - repository = 'python-attrs/attrs' - workflow = 'pypi-package.yml' - - [[packages]] - name = 'cattrs' - version = '24.1.2' - requires-python = '>=3.8' - dependencies = [ - {name = 'attrs'}, - ] - index = 'https://pypi.org/simple' - wheels = [ - {name = 'cattrs-24.1.2-py3-none-any.whl', upload-time = 2024-09-22T14:58:34.812643+00:00, url = 'https://files.pythonhosted.org/packages/c8/d5/867e75361fc45f6de75fe277dd085627a9db5ebb511a87f27dc1396b5351/cattrs-24.1.2-py3-none-any.whl', size = 66446, hashes = {sha256 = '67c7495b760168d931a10233f979b28dc04daf853b30752246f4f8471c6d68d0'}}, - ] - - [[packages]] - name = 'numpy' - version = '2.2.3' - requires-python = '>=3.10' - wheels = [ - {name = 'numpy-2.2.3-cp312-cp312-win_amd64.whl', upload-time = 2025-02-13T16:51:21.821880+00:00, url = 'https://files.pythonhosted.org/packages/42/6e/55580a538116d16ae7c9aa17d4edd56e83f42126cb1dfe7a684da7925d2c/numpy-2.2.3-cp312-cp312-win_amd64.whl', size = 12626357, hashes = {sha256 = '83807d445817326b4bcdaaaf8e8e9f1753da04341eceec705c001ff342002e5d'}}, - {name = 'numpy-2.2.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl', upload-time = 2025-02-13T16:50:00.079662+00:00, url = 'https://files.pythonhosted.org/packages/39/04/78d2e7402fb479d893953fb78fa7045f7deb635ec095b6b4f0260223091a/numpy-2.2.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl', size = 16116679, hashes = {sha256 = '3b787adbf04b0db1967798dba8da1af07e387908ed1553a0d6e74c084d1ceafe'}}, - ] - - [tool.mousebender] - command = ['.', 'lock', '--platform', 'cpython3.12-windows-x64', '--platform', 'cpython3.12-manylinux2014-x64', 'cattrs', 'numpy'] - run-on = 2025-03-06T12:28:57.760769 - """ # noqa: E501 -) - - def test_toml_roundtrip() -> None: - pylock_dict = tomllib.loads(PEP751_EXAMPLE) + pep751_example = ( + Path(__file__).parent / "pylock" / "pylock.spec-example.toml" + ).read_text() + pylock_dict = tomllib.loads(pep751_example) pylock = Pylock.from_dict(pylock_dict) # Check that the roundrip via Pylock dataclasses produces the same TOML # output, modulo TOML serialization differences. From 258d35b220e5b7fe3bfd2089385687743861c0e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sun, 27 Jul 2025 12:08:13 +0200 Subject: [PATCH 09/24] pylock: make more module level stuff "private" --- src/packaging/pylock.py | 50 ++++++++++++++++++++--------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/src/packaging/pylock.py b/src/packaging/pylock.py index ea87ea7fb..a15d706f5 100644 --- a/src/packaging/pylock.py +++ b/src/packaging/pylock.py @@ -40,23 +40,23 @@ "is_valid_pylock_path", ] -T = TypeVar("T") -T2 = TypeVar("T2") +_T = TypeVar("_T") +_T2 = TypeVar("_T2") -class FromMappingProtocol(Protocol): # pragma: no cover +class _FromMappingProtocol(Protocol): # pragma: no cover @classmethod def _from_dict(cls, d: Mapping[str, Any]) -> Self: ... -FromMappingProtocolT = TypeVar("FromMappingProtocolT", bound=FromMappingProtocol) +_FromMappingProtocolT = TypeVar("_FromMappingProtocolT", bound=_FromMappingProtocol) -PYLOCK_FILE_NAME_RE = re.compile(r"^pylock\.([^.]+)\.toml$") +_PYLOCK_FILE_NAME_RE = re.compile(r"^pylock\.([^.]+)\.toml$") def is_valid_pylock_path(path: Path) -> bool: - return path.name == "pylock.toml" or bool(PYLOCK_FILE_NAME_RE.match(path.name)) + return path.name == "pylock.toml" or bool(_PYLOCK_FILE_NAME_RE.match(path.name)) def _toml_key(key: str) -> str: @@ -79,7 +79,7 @@ def _toml_dict_factory(data: list[tuple[str, Any]]) -> dict[str, Any]: } -def _get(d: Mapping[str, Any], expected_type: type[T], key: str) -> T | None: +def _get(d: Mapping[str, Any], expected_type: type[_T], key: str) -> _T | None: """Get a value from the dictionary and verify it's the expected type.""" if (value := d.get(key)) is None: return None @@ -92,7 +92,7 @@ def _get(d: Mapping[str, Any], expected_type: type[T], key: str) -> T | None: return value -def _get_required(d: Mapping[str, Any], expected_type: type[T], key: str) -> T: +def _get_required(d: Mapping[str, Any], expected_type: type[_T], key: str) -> _T: """Get a required value from the dictionary and verify it's the expected type.""" if (value := _get(d, expected_type, key)) is None: raise PylockRequiredKeyError(key) @@ -100,8 +100,8 @@ def _get_required(d: Mapping[str, Any], expected_type: type[T], key: str) -> T: def _get_sequence( - d: Mapping[str, Any], expected_item_type: type[T], key: str -) -> Sequence[T] | None: + d: Mapping[str, Any], expected_item_type: type[_T], key: str +) -> Sequence[_T] | None: """Get a list value from the dictionary and verify it's the expected items type.""" if (value := _get(d, Sequence, key)) is None: # type: ignore[type-abstract] return None @@ -117,10 +117,10 @@ def _get_sequence( def _get_as( d: Mapping[str, Any], - expected_type: type[T], - target_type: Callable[[T], T2], + expected_type: type[_T], + target_type: Callable[[_T], _T2], key: str, -) -> T2 | None: +) -> _T2 | None: """Get a value from the dictionary, verify it's the expected type, and convert to the target type. @@ -136,10 +136,10 @@ def _get_as( def _get_required_as( d: Mapping[str, Any], - expected_type: type[T], - target_type: Callable[[T], T2], + expected_type: type[_T], + target_type: Callable[[_T], _T2], key: str, -) -> T2: +) -> _T2: """Get a required value from the dict, verify it's the expected type, and convert to the target type.""" if (value := _get_as(d, expected_type, target_type, key)) is None: @@ -149,10 +149,10 @@ def _get_required_as( def _get_sequence_as( d: Mapping[str, Any], - expected_item_type: type[T], - target_item_type: Callable[[T], T2], + expected_item_type: type[_T], + target_item_type: Callable[[_T], _T2], key: str, -) -> Sequence[T2] | None: +) -> Sequence[_T2] | None: """Get list value from dictionary and verify expected items type.""" if (value := _get_sequence(d, expected_item_type, key)) is None: return None @@ -168,8 +168,8 @@ def _get_sequence_as( def _get_object( - d: Mapping[str, Any], target_type: type[FromMappingProtocolT], key: str -) -> FromMappingProtocolT | None: + d: Mapping[str, Any], target_type: type[_FromMappingProtocolT], key: str +) -> _FromMappingProtocolT | None: """Get a dictionary value from the dictionary and convert it to a dataclass.""" value = _get(d, Mapping, key) # type: ignore[type-abstract] if value is None: @@ -181,8 +181,8 @@ def _get_object( def _get_sequence_of_objects( - d: Mapping[str, Any], target_item_type: type[FromMappingProtocolT], key: str -) -> Sequence[FromMappingProtocolT] | None: + d: Mapping[str, Any], target_item_type: type[_FromMappingProtocolT], key: str +) -> Sequence[_FromMappingProtocolT] | None: """Get a list value from the dictionary and convert its items to a dataclass.""" if (value := _get(d, Sequence, key)) is None: # type: ignore[type-abstract] return None @@ -203,8 +203,8 @@ def _get_sequence_of_objects( def _get_required_list_of_objects( - d: Mapping[str, Any], target_type: type[FromMappingProtocolT], key: str -) -> Sequence[FromMappingProtocolT]: + d: Mapping[str, Any], target_type: type[_FromMappingProtocolT], key: str +) -> Sequence[_FromMappingProtocolT]: """Get a required list value from the dictionary and convert its items to a dataclass.""" if (result := _get_sequence_of_objects(d, target_type, key)) is None: From 46284113790967624e640284f7280ceff5aee88e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sun, 27 Jul 2025 12:23:35 +0200 Subject: [PATCH 10/24] pylock: re-enable default None values These comments were remnants of dataclass experiments. --- src/packaging/pylock.py | 64 ++++++++++++++++++++--------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/src/packaging/pylock.py b/src/packaging/pylock.py index a15d706f5..28f9977dc 100644 --- a/src/packaging/pylock.py +++ b/src/packaging/pylock.py @@ -271,10 +271,10 @@ class PylockUnsupportedVersionError(PylockValidationError): @dataclass(frozen=True, init=False) class PackageVcs: type: str - url: str | None # = None - path: str | None # = None - requested_revision: str | None # = None - commit_id: str + url: str | None = None + path: str | None = None + requested_revision: str | None = None + commit_id: str # type: ignore[misc] subdirectory: str | None = None def __init__( @@ -338,21 +338,21 @@ def _from_dict(cls, d: Mapping[str, Any]) -> Self: @dataclass(frozen=True, init=False) class PackageArchive: - url: str | None # = None - path: str | None # = None - size: int | None # = None - upload_time: datetime | None # = None - hashes: Mapping[str, str] + url: str | None = None + path: str | None = None + size: int | None = None + upload_time: datetime | None = None + hashes: Mapping[str, str] # type: ignore[misc] subdirectory: str | None = None def __init__( self, *, - hashes: Mapping[str, str], url: str | None = None, path: str | None = None, size: int | None = None, upload_time: datetime | None = None, + hashes: Mapping[str, str], subdirectory: str | None = None, ) -> None: # In Python 3.10+ make dataclass kw_only=True and remove __init__ @@ -380,22 +380,22 @@ def _from_dict(cls, d: Mapping[str, Any]) -> Self: @dataclass(frozen=True, init=False) class PackageSdist: - name: str | None # = None - upload_time: datetime | None # = None - url: str | None # = None - path: str | None # = None - size: int | None # = None - hashes: Mapping[str, str] + name: str | None = None + upload_time: datetime | None = None + url: str | None = None + path: str | None = None + size: int | None = None + hashes: Mapping[str, str] # type: ignore[misc] def __init__( self, *, - hashes: Mapping[str, str], name: str | None = None, upload_time: datetime | None = None, url: str | None = None, path: str | None = None, size: int | None = None, + hashes: Mapping[str, str], ) -> None: # In Python 3.10+ make dataclass kw_only=True and remove __init__ object.__setattr__(self, "name", name) @@ -422,22 +422,22 @@ def _from_dict(cls, d: Mapping[str, Any]) -> Self: @dataclass(frozen=True, init=False) class PackageWheel: - name: str # | None - upload_time: datetime | None # = None - url: str | None # = None - path: str | None # = None - size: int | None # = None - hashes: Mapping[str, str] + name: str | None = None + upload_time: datetime | None = None + url: str | None = None + path: str | None = None + size: int | None = None + hashes: Mapping[str, str] # type: ignore[misc] def __init__( self, *, - hashes: Mapping[str, str], name: str | None = None, upload_time: datetime | None = None, url: str | None = None, path: str | None = None, size: int | None = None, + hashes: Mapping[str, str], ) -> None: # In Python 3.10+ make dataclass kw_only=True and remove __init__ object.__setattr__(self, "name", name) @@ -551,25 +551,25 @@ def is_direct(self) -> bool: @dataclass(frozen=True, init=False) class Pylock: lock_version: Version - environments: Sequence[Marker] | None # = None - requires_python: SpecifierSet | None # = None - extras: Sequence[NormalizedName] | None # = None - dependency_groups: Sequence[str] | None # = None - default_groups: Sequence[str] | None # = None - created_by: str - packages: Sequence[Package] + environments: Sequence[Marker] | None = None + requires_python: SpecifierSet | None = None + extras: Sequence[NormalizedName] | None = None + dependency_groups: Sequence[str] | None = None + default_groups: Sequence[str] | None = None + created_by: str # type: ignore[misc] + packages: Sequence[Package] # type: ignore[misc] tool: Mapping[str, Any] | None = None def __init__( self, *, lock_version: Version, - created_by: str, environments: Sequence[Marker] | None = None, requires_python: SpecifierSet | None = None, extras: Sequence[str] | None = None, dependency_groups: Sequence[str] | None = None, default_groups: Sequence[str] | None = None, + created_by: str, packages: Sequence[Package], tool: Mapping[str, Any] | None = None, ) -> None: From b9d782ba58a72b6470102b10e76b304ee916eee0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sun, 27 Jul 2025 13:55:45 +0200 Subject: [PATCH 11/24] pylock: add validate method --- src/packaging/pylock.py | 4 ++++ tests/test_pylock.py | 42 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/src/packaging/pylock.py b/src/packaging/pylock.py index 28f9977dc..88e1cddae 100644 --- a/src/packaging/pylock.py +++ b/src/packaging/pylock.py @@ -613,3 +613,7 @@ def _from_dict(cls, d: Mapping[str, Any]) -> Self: @classmethod def from_dict(cls, d: Mapping[str, Any]) -> Self: return cls._from_dict(d) + + def validate(self) -> None: + """Validate the Pylock instance against the specification.""" + self.from_dict(self.to_dict()) diff --git a/tests/test_pylock.py b/tests/test_pylock.py index e5ed9cc5b..0ec6db279 100644 --- a/tests/test_pylock.py +++ b/tests/test_pylock.py @@ -432,3 +432,45 @@ def test_is_direct() -> None: ], ) assert not wheel_package.is_direct + + +def test_validate() -> None: + pylock = Pylock( + lock_version=Version("1.0"), + created_by="some_tool", + packages=[ + Package( + name=NormalizedName("example-package"), + version=Version("1.0.0"), + wheels=[ + PackageWheel( + url="https://example.com/example_package-1.0.0-py3-none-any.whl", + hashes={"sha256": "0fd.."}, + ) + ], + ) + ], + ) + pylock.validate() # Should not raise any exceptions + pylock = Pylock( + lock_version=Version("1.0"), + created_by="some_tool", + packages=[ + Package( + name=NormalizedName("example_package"), # not normalized + version=Version("1.0.0"), + wheels=[ + PackageWheel( + url="https://example.com/example_package-1.0.0-py3-none-any.whl", + hashes={"sha256": "0fd.."}, + ) + ], + ) + ], + ) + with pytest.raises(PylockValidationError) as exc_info: + pylock.validate() + assert ( + str(exc_info.value) + == "Name 'example_package' is not normalized in 'packages[0].name'" + ) From 3df903d80a476681afaf7e5561424c790b1f7a77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sun, 27 Jul 2025 14:34:12 +0200 Subject: [PATCH 12/24] pylock: validate everything in from_dict This is more consitent than having some validations done in the constructor and other in from_dict. --- src/packaging/pylock.py | 91 ++++++++++++++++++++--------------------- tests/test_pylock.py | 6 +-- 2 files changed, 48 insertions(+), 49 deletions(-) diff --git a/src/packaging/pylock.py b/src/packaging/pylock.py index 88e1cddae..c307f20ac 100644 --- a/src/packaging/pylock.py +++ b/src/packaging/pylock.py @@ -224,11 +224,12 @@ def _validate_path_url(path: str | None, url: str | None) -> None: raise PylockValidationError("path or url must be provided") -def _validate_hashes(hashes: Mapping[str, Any]) -> None: +def _validate_hashes(hashes: Mapping[str, Any]) -> Mapping[str, Any]: if not hashes: raise PylockValidationError("At least one hash must be provided") if not all(isinstance(hash, str) for hash in hashes.values()): raise PylockValidationError("Hash values must be strings") + return hashes class PylockValidationError(Exception): @@ -294,12 +295,10 @@ def __init__( object.__setattr__(self, "requested_revision", requested_revision) object.__setattr__(self, "commit_id", commit_id) object.__setattr__(self, "subdirectory", subdirectory) - # __post_init__ in Python 3.10+ - _validate_path_url(self.path, self.url) @classmethod def _from_dict(cls, d: Mapping[str, Any]) -> Self: - return cls( + package_vcs = cls( type=_get_required(d, str, "type"), url=_get(d, str, "url"), path=_get(d, str, "path"), @@ -307,6 +306,8 @@ def _from_dict(cls, d: Mapping[str, Any]) -> Self: commit_id=_get_required(d, str, "commit-id"), subdirectory=_get(d, str, "subdirectory"), ) + _validate_path_url(package_vcs.path, package_vcs.url) + return package_vcs @dataclass(frozen=True, init=False) @@ -362,20 +363,19 @@ def __init__( object.__setattr__(self, "upload_time", upload_time) object.__setattr__(self, "hashes", hashes) object.__setattr__(self, "subdirectory", subdirectory) - # __post_init__ in Python 3.10+ - _validate_path_url(self.path, self.url) - _validate_hashes(self.hashes) @classmethod def _from_dict(cls, d: Mapping[str, Any]) -> Self: - return cls( + package_archive = cls( url=_get(d, str, "url"), path=_get(d, str, "path"), size=_get(d, int, "size"), upload_time=_get(d, datetime, "upload-time"), - hashes=_get_required(d, Mapping, "hashes"), # type: ignore[type-abstract] + hashes=_get_required_as(d, Mapping, _validate_hashes, "hashes"), # type: ignore[type-abstract] subdirectory=_get(d, str, "subdirectory"), ) + _validate_path_url(package_archive.path, package_archive.url) + return package_archive @dataclass(frozen=True, init=False) @@ -404,20 +404,19 @@ def __init__( object.__setattr__(self, "path", path) object.__setattr__(self, "size", size) object.__setattr__(self, "hashes", hashes) - # __post_init__ in Python 3.10+ - _validate_path_url(self.path, self.url) - _validate_hashes(self.hashes) @classmethod def _from_dict(cls, d: Mapping[str, Any]) -> Self: - return cls( + package_sdist = cls( name=_get(d, str, "name"), upload_time=_get(d, datetime, "upload-time"), url=_get(d, str, "url"), path=_get(d, str, "path"), size=_get(d, int, "size"), - hashes=_get_required(d, Mapping, "hashes"), # type: ignore[type-abstract] + hashes=_get_required_as(d, Mapping, _validate_hashes, "hashes"), # type: ignore[type-abstract] ) + _validate_path_url(package_sdist.path, package_sdist.url) + return package_sdist @dataclass(frozen=True, init=False) @@ -446,20 +445,19 @@ def __init__( object.__setattr__(self, "path", path) object.__setattr__(self, "size", size) object.__setattr__(self, "hashes", hashes) - # __post_init__ in Python 3.10+ - _validate_path_url(self.path, self.url) - _validate_hashes(self.hashes) @classmethod def _from_dict(cls, d: Mapping[str, Any]) -> Self: - return cls( + package_wheel = cls( name=_get(d, str, "name"), upload_time=_get(d, datetime, "upload-time"), url=_get(d, str, "url"), path=_get(d, str, "path"), size=_get(d, int, "size"), - hashes=_get_required(d, Mapping, "hashes"), # type: ignore[type-abstract] + hashes=_get_required_as(d, Mapping, _validate_hashes, "hashes"), # type: ignore[type-abstract] ) + _validate_path_url(package_wheel.path, package_wheel.url) + return package_wheel @dataclass(frozen=True, init=False) @@ -509,20 +507,6 @@ def __init__( object.__setattr__(self, "wheels", wheels) object.__setattr__(self, "attestation_identities", attestation_identities) object.__setattr__(self, "tool", tool) - # __post_init__ in Python 3.10+ - if self.sdist or self.wheels: - if self.vcs or self.directory or self.archive: - raise PylockValidationError( - "None of vcs, directory, archive " - "must be set if sdist or wheels are set" - ) - else: - # no sdist nor wheels - if not (bool(self.vcs) ^ bool(self.directory) ^ bool(self.archive)): - raise PylockValidationError( - "Exactly one of vcs, directory, archive must be set " - "if sdist and wheels are not set" - ) @classmethod def _from_dict(cls, d: Mapping[str, Any]) -> Self: @@ -541,6 +525,21 @@ def _from_dict(cls, d: Mapping[str, Any]) -> Self: attestation_identities=_get_sequence(d, Mapping, "attestation-identities"), # type: ignore[type-abstract] tool=_get(d, Mapping, "tool"), # type: ignore[type-abstract] ) + if package.sdist or package.wheels: + if package.vcs or package.directory or package.archive: + raise PylockValidationError( + "None of vcs, directory, archive " + "must be set if sdist or wheels are set" + ) + else: + # no sdist nor wheels + if not ( + bool(package.vcs) ^ bool(package.directory) ^ bool(package.archive) + ): + raise PylockValidationError( + "Exactly one of vcs, directory, archive must be set " + "if sdist and wheels are not set" + ) return package @property @@ -583,22 +582,10 @@ def __init__( object.__setattr__(self, "created_by", created_by) object.__setattr__(self, "packages", packages) object.__setattr__(self, "tool", tool) - # __post_init__ in Python 3.10+ - if self.lock_version < Version("1") or self.lock_version >= Version("2"): - raise PylockUnsupportedVersionError( - f"pylock version {self.lock_version} is not supported" - ) - if self.lock_version > Version("1.0"): - logging.warning( - "pylock minor version %s is not supported", self.lock_version - ) - - def to_dict(self) -> Mapping[str, Any]: - return dataclasses.asdict(self, dict_factory=_toml_dict_factory) @classmethod def _from_dict(cls, d: Mapping[str, Any]) -> Self: - return cls( + pylock = cls( lock_version=_get_required_as(d, str, Version, "lock-version"), environments=_get_sequence_as(d, str, Marker, "environments"), extras=_get_sequence_as(d, str, _validate_normalized_name, "extras"), @@ -609,11 +596,23 @@ def _from_dict(cls, d: Mapping[str, Any]) -> Self: packages=_get_required_list_of_objects(d, Package, "packages"), tool=_get(d, Mapping, "tool"), # type: ignore[type-abstract] ) + if pylock.lock_version < Version("1") or pylock.lock_version >= Version("2"): + raise PylockUnsupportedVersionError( + f"pylock version {pylock.lock_version} is not supported" + ) + if pylock.lock_version > Version("1.0"): + logging.warning( + "pylock minor version %s is not supported", pylock.lock_version + ) + return pylock @classmethod def from_dict(cls, d: Mapping[str, Any]) -> Self: return cls._from_dict(d) + def to_dict(self) -> Mapping[str, Any]: + return dataclasses.asdict(self, dict_factory=_toml_dict_factory) + def validate(self) -> None: """Validate the Pylock instance against the specification.""" self.from_dict(self.to_dict()) diff --git a/tests/test_pylock.py b/tests/test_pylock.py index 0ec6db279..29345a95b 100644 --- a/tests/test_pylock.py +++ b/tests/test_pylock.py @@ -243,7 +243,7 @@ def test_pylock_invalid_archive() -> None: def test_pylock_invalid_vcs() -> None: with pytest.raises(PylockValidationError) as exc_info: - PackageVcs(type="git", url=None, path=None, commit_id="f" * 40) + PackageVcs._from_dict({"type": "git", "commit-id": "f" * 40}) assert str(exc_info.value) == "path or url must be provided" @@ -370,11 +370,11 @@ def test_pylock_package_not_a_table() -> None: "sha256": "f" * 40, "md5": 1, }, - "Hash values must be strings", + "Hash values must be strings in 'hashes'", ), ( {}, - "At least one hash must be provided", + "At least one hash must be provided in 'hashes'", ), ( "sha256:...", From a22bcfabf4bed6963436bcca718ebd16f78ec00b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sun, 27 Jul 2025 14:42:33 +0200 Subject: [PATCH 13/24] pylock: add documentation --- docs/index.rst | 1 + docs/pylock.rst | 83 +++++++++++++++++++++++++++++++++++++++++ src/packaging/pylock.py | 17 ++++++++- 3 files changed, 99 insertions(+), 2 deletions(-) create mode 100644 docs/pylock.rst diff --git a/docs/index.rst b/docs/index.rst index 04579c4f2..cc9dca61c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -30,6 +30,7 @@ The ``packaging`` library uses calendar-based versioning (``YY.N``). requirements metadata tags + pylock utils .. toctree:: diff --git a/docs/pylock.rst b/docs/pylock.rst new file mode 100644 index 000000000..3f2e0d359 --- /dev/null +++ b/docs/pylock.rst @@ -0,0 +1,83 @@ +Lock Files +========== + +.. currentmodule:: packaging.pylock + +Parse and validate `pylock.toml files `_. + +Usage +----- + +.. code-block:: python + + import tomllib + from pathlib import Path + + from packaging.pylock import Package, PackageWheel, Pylock + from packaging.utils import NormalizedName + from packaging.version import Version + + # validate a pylock file name + assert is_valid_pylock_path(Path("pylock.example.toml")) + + # parse and validate pylock file + toml_dict = tomllib.loads(Path("pylock.toml").read_text(encoding="utf-8")) + pylock = PyLock.from_dict(toml_dict) + # the resulting pylock object is validated against the specification, + # else a PylockValidationError is raised + + # generate a pylock file + pylock = Pylock( + lock_version=Version("1.0"), + created_by="some_tool", + packages=[ + Package( + name=NormalizedName("example-package"), + version=Version("1.0.0"), + wheels=[ + PackageWheel( + url="https://example.com/example_package-1.0.0-py3-none-any.whl", + hashes={"sha256": "0fd.."}, + ) + ], + ) + ], + ) + toml_dict = pylock.to_dict() + # use a third-party library to serialize to TOML + + # you can validate a manually constructed Pylock class + pylock.validate() + +Reference +--------- + +.. autofunction:: is_valid_pylock_path + +The following frozen keyword-only dataclasses are used to represent the +structure of a pylock file. The attributes correspond to the fields in the +pylock file specification. + +.. autoclass:: Pylock + :members: from_dict, to_dict, validate + :exclude-members: __init__, __new__ + +.. class:: Package + +.. class:: PackageWheel + +.. class:: PackageSdist + +.. class:: PackageArchive + +.. class:: PackageVcs + +.. class:: PackageDirectory + +The following exception may be raised by this module: + +.. autoexception:: PylockValidationError + :exclude-members: __init__, __new__ + +.. autoexception:: PylockUnsupportedVersionError + :exclude-members: __init__, __new__ diff --git a/src/packaging/pylock.py b/src/packaging/pylock.py index c307f20ac..8c4f4c149 100644 --- a/src/packaging/pylock.py +++ b/src/packaging/pylock.py @@ -56,6 +56,7 @@ def _from_dict(cls, d: Mapping[str, Any]) -> Self: ... def is_valid_pylock_path(path: Path) -> bool: + """Check if the given path is a valid pylock file path.""" return path.name == "pylock.toml" or bool(_PYLOCK_FILE_NAME_RE.match(path.name)) @@ -233,6 +234,8 @@ def _validate_hashes(hashes: Mapping[str, Any]) -> Mapping[str, Any]: class PylockValidationError(Exception): + """Raised when when input data is not spec-compliant.""" + context: str | None = None message: str @@ -266,7 +269,7 @@ def __init__(self, key: str) -> None: class PylockUnsupportedVersionError(PylockValidationError): - pass + """Raised when encountering an unsupported `lock_version`.""" @dataclass(frozen=True, init=False) @@ -549,6 +552,8 @@ def is_direct(self) -> bool: @dataclass(frozen=True, init=False) class Pylock: + """A class representing a pylock file.""" + lock_version: Version environments: Sequence[Marker] | None = None requires_python: SpecifierSet | None = None @@ -608,11 +613,19 @@ def _from_dict(cls, d: Mapping[str, Any]) -> Self: @classmethod def from_dict(cls, d: Mapping[str, Any]) -> Self: + """Create and validate a Pylock instance from a TOML dictionary. + + Raises :class:`PylockValidationError` if the input data is not + spec-compliant. + """ return cls._from_dict(d) def to_dict(self) -> Mapping[str, Any]: + """Convert the Pylock instance to a TOML dictionary.""" return dataclasses.asdict(self, dict_factory=_toml_dict_factory) def validate(self) -> None: - """Validate the Pylock instance against the specification.""" + """Validate the Pylock instance against the specification. + + Raises :class:`PylockValidationError` otherwise.""" self.from_dict(self.to_dict()) From 6460df384a8b22e9305e5bc1d47a318c4e93891b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sat, 13 Sep 2025 10:50:16 +0200 Subject: [PATCH 14/24] pylock: improve version check readability and test coverage --- src/packaging/pylock.py | 2 +- tests/test_pylock.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/packaging/pylock.py b/src/packaging/pylock.py index 8c4f4c149..f572a798e 100644 --- a/src/packaging/pylock.py +++ b/src/packaging/pylock.py @@ -601,7 +601,7 @@ def _from_dict(cls, d: Mapping[str, Any]) -> Self: packages=_get_required_list_of_objects(d, Package, "packages"), tool=_get(d, Mapping, "tool"), # type: ignore[type-abstract] ) - if pylock.lock_version < Version("1") or pylock.lock_version >= Version("2"): + if not Version("1") <= pylock.lock_version < Version("2"): raise PylockUnsupportedVersionError( f"pylock version {pylock.lock_version} is not supported" ) diff --git a/tests/test_pylock.py b/tests/test_pylock.py index 29345a95b..22c3ec255 100644 --- a/tests/test_pylock.py +++ b/tests/test_pylock.py @@ -64,9 +64,10 @@ def test_pylock_version(version: str) -> None: Pylock.from_dict(data) -def test_pylock_unsupported_version() -> None: +@pytest.mark.parametrize("version", ["0.9", "2", "2.0", "2.1"]) +def test_pylock_unsupported_version(version: str) -> None: data = { - "lock-version": "2.0", + "lock-version": version, "created-by": "pip", "packages": [], } From c9eb0f241dc73e5e32598b92eed79a1266537a16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sat, 13 Sep 2025 11:07:35 +0200 Subject: [PATCH 15/24] pylock: fix and clarify package validation logic --- src/packaging/pylock.py | 28 +++++++++++++--------------- tests/test_pylock.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 15 deletions(-) diff --git a/src/packaging/pylock.py b/src/packaging/pylock.py index f572a798e..ff503e171 100644 --- a/src/packaging/pylock.py +++ b/src/packaging/pylock.py @@ -528,21 +528,19 @@ def _from_dict(cls, d: Mapping[str, Any]) -> Self: attestation_identities=_get_sequence(d, Mapping, "attestation-identities"), # type: ignore[type-abstract] tool=_get(d, Mapping, "tool"), # type: ignore[type-abstract] ) - if package.sdist or package.wheels: - if package.vcs or package.directory or package.archive: - raise PylockValidationError( - "None of vcs, directory, archive " - "must be set if sdist or wheels are set" - ) - else: - # no sdist nor wheels - if not ( - bool(package.vcs) ^ bool(package.directory) ^ bool(package.archive) - ): - raise PylockValidationError( - "Exactly one of vcs, directory, archive must be set " - "if sdist and wheels are not set" - ) + distributions = bool(package.sdist) + len(package.wheels or []) + direct_urls = ( + bool(package.vcs) + bool(package.directory) + bool(package.archive) + ) + if distributions > 0 and direct_urls > 0: + raise PylockValidationError( + "None of vcs, directory, archive must be set if sdist or wheels are set" + ) + if distributions == 0 and direct_urls != 1: + raise PylockValidationError( + "Exactly one of vcs, directory, archive must be set " + "if sdist and wheels are not set" + ) return package @property diff --git a/tests/test_pylock.py b/tests/test_pylock.py index 22c3ec255..a2022ae97 100644 --- a/tests/test_pylock.py +++ b/tests/test_pylock.py @@ -172,6 +172,39 @@ def test_pylock_packages_with_dist_and_archive() -> None: ) +def test_pylock_packages_with_archive_directory_and_vcs() -> None: + data = { + "lock-version": "1.0", + "created-by": "pip", + "packages": [ + { + "name": "example", + "version": "1.0", + "archive": { + "path": "example.tar.gz", + "hashes": {"sha256": "f" * 40}, + }, + "vcs": { + "type": "git", + "url": "https://githhub/pypa/packaging", + "commit-id": "...", + }, + "directory": { + "path": ".", + "editable": False, + }, + } + ], + } + with pytest.raises(PylockValidationError) as exc_info: + Pylock.from_dict(data) + assert str(exc_info.value) == ( + "Exactly one of vcs, directory, archive must be set " + "if sdist and wheels are not set " + "in 'packages[0]'" + ) + + def test_pylock_basic_package() -> None: data = { "lock-version": "1.0", From 3537c648848e94af90ef111a3b3c1a3902bd21c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Thu, 9 Oct 2025 14:51:24 +0200 Subject: [PATCH 16/24] pylock: add positional-only parrameter marker to from_dict --- src/packaging/pylock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/packaging/pylock.py b/src/packaging/pylock.py index ff503e171..bca851623 100644 --- a/src/packaging/pylock.py +++ b/src/packaging/pylock.py @@ -610,7 +610,7 @@ def _from_dict(cls, d: Mapping[str, Any]) -> Self: return pylock @classmethod - def from_dict(cls, d: Mapping[str, Any]) -> Self: + def from_dict(cls, d: Mapping[str, Any], /) -> Self: """Create and validate a Pylock instance from a TOML dictionary. Raises :class:`PylockValidationError` if the input data is not From f36caf7aef58772abc7d3e92e7ed940755e8d00a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Thu, 9 Oct 2025 14:59:14 +0200 Subject: [PATCH 17/24] pylock: handle review nit --- src/packaging/pylock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/packaging/pylock.py b/src/packaging/pylock.py index bca851623..50598e469 100644 --- a/src/packaging/pylock.py +++ b/src/packaging/pylock.py @@ -259,7 +259,7 @@ def __init__( def __str__(self) -> str: if self.context: - return f"{self.message} in '{self.context}'" + return f"{self.message} in {self.context!r}" return self.message From 2e41829323ef5b017621386ae1eb537746cbd942 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sun, 12 Oct 2025 12:57:06 +0200 Subject: [PATCH 18/24] pylock: simplify validation of sequences of objects --- src/packaging/pylock.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/packaging/pylock.py b/src/packaging/pylock.py index 50598e469..4dd9c1492 100644 --- a/src/packaging/pylock.py +++ b/src/packaging/pylock.py @@ -185,15 +185,10 @@ def _get_sequence_of_objects( d: Mapping[str, Any], target_item_type: type[_FromMappingProtocolT], key: str ) -> Sequence[_FromMappingProtocolT] | None: """Get a list value from the dictionary and convert its items to a dataclass.""" - if (value := _get(d, Sequence, key)) is None: # type: ignore[type-abstract] + if (value := _get_sequence(d, Mapping, key)) is None: # type: ignore[type-abstract] return None result = [] for i, item in enumerate(value): - if not isinstance(item, Mapping): - raise PylockValidationError( - f"Unexpected type {type(item).__name__} (expected Mapping)", - context=f"{key}[{i}]", - ) try: typed_item = target_item_type._from_dict(item) except Exception as e: From 066a479a15a47825cc44d71f262d2435199d069d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sun, 12 Oct 2025 12:58:58 +0200 Subject: [PATCH 19/24] pylock: rename helper function for consistency --- src/packaging/pylock.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/packaging/pylock.py b/src/packaging/pylock.py index 4dd9c1492..c72f95c2c 100644 --- a/src/packaging/pylock.py +++ b/src/packaging/pylock.py @@ -198,12 +198,12 @@ def _get_sequence_of_objects( return result -def _get_required_list_of_objects( - d: Mapping[str, Any], target_type: type[_FromMappingProtocolT], key: str +def _get_required_sequence_of_objects( + d: Mapping[str, Any], target_item_type: type[_FromMappingProtocolT], key: str ) -> Sequence[_FromMappingProtocolT]: """Get a required list value from the dictionary and convert its items to a dataclass.""" - if (result := _get_sequence_of_objects(d, target_type, key)) is None: + if (result := _get_sequence_of_objects(d, target_item_type, key)) is None: raise PylockRequiredKeyError(key) return result @@ -591,7 +591,7 @@ def _from_dict(cls, d: Mapping[str, Any]) -> Self: default_groups=_get_sequence(d, str, "default-groups"), created_by=_get_required(d, str, "created-by"), requires_python=_get_as(d, str, SpecifierSet, "requires-python"), - packages=_get_required_list_of_objects(d, Package, "packages"), + packages=_get_required_sequence_of_objects(d, Package, "packages"), tool=_get(d, Mapping, "tool"), # type: ignore[type-abstract] ) if not Version("1") <= pylock.lock_version < Version("2"): From bdf84b30642b66f2b54766d6c0acdfceedf72cee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sun, 12 Oct 2025 13:04:27 +0200 Subject: [PATCH 20/24] pylock: add missing walrus assignment for consistency --- src/packaging/pylock.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/packaging/pylock.py b/src/packaging/pylock.py index c72f95c2c..8c7e6b8ca 100644 --- a/src/packaging/pylock.py +++ b/src/packaging/pylock.py @@ -172,8 +172,7 @@ def _get_object( d: Mapping[str, Any], target_type: type[_FromMappingProtocolT], key: str ) -> _FromMappingProtocolT | None: """Get a dictionary value from the dictionary and convert it to a dataclass.""" - value = _get(d, Mapping, key) # type: ignore[type-abstract] - if value is None: + if (value := _get(d, Mapping, key)) is None: # type: ignore[type-abstract] return None try: return target_type._from_dict(value) From 2fa5f17d439b869ca1b3ea9cb020523ee0598291 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sun, 12 Oct 2025 14:22:19 +0200 Subject: [PATCH 21/24] pylock: fix contructor type annotation for extras --- src/packaging/pylock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/packaging/pylock.py b/src/packaging/pylock.py index 8c7e6b8ca..806042780 100644 --- a/src/packaging/pylock.py +++ b/src/packaging/pylock.py @@ -562,7 +562,7 @@ def __init__( lock_version: Version, environments: Sequence[Marker] | None = None, requires_python: SpecifierSet | None = None, - extras: Sequence[str] | None = None, + extras: Sequence[NormalizedName] | None = None, dependency_groups: Sequence[str] | None = None, default_groups: Sequence[str] | None = None, created_by: str, From 686d20477662662477a02d414f1c978c644ad9b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sun, 12 Oct 2025 14:27:55 +0200 Subject: [PATCH 22/24] pylock: reject str and bytes when Sequence is expected --- src/packaging/pylock.py | 6 ++++++ tests/test_pylock.py | 14 ++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/src/packaging/pylock.py b/src/packaging/pylock.py index 806042780..69d65f88a 100644 --- a/src/packaging/pylock.py +++ b/src/packaging/pylock.py @@ -106,6 +106,12 @@ def _get_sequence( """Get a list value from the dictionary and verify it's the expected items type.""" if (value := _get(d, Sequence, key)) is None: # type: ignore[type-abstract] return None + if isinstance(value, (str, bytes)): + # special case: str and bytes are Sequences, but we want to reject it + raise PylockValidationError( + f"Unexpected type {type(value).__name__} (expected Sequence)", + context=key, + ) for i, item in enumerate(value): if not isinstance(item, expected_item_type): raise PylockValidationError( diff --git a/tests/test_pylock.py b/tests/test_pylock.py index a2022ae97..5d7cc145c 100644 --- a/tests/test_pylock.py +++ b/tests/test_pylock.py @@ -508,3 +508,17 @@ def test_validate() -> None: str(exc_info.value) == "Name 'example_package' is not normalized in 'packages[0].name'" ) + + +def test_validate_sequence_of_str() -> None: + pylock = Pylock( + lock_version=Version("1.0"), + created_by="some_tool", + packages=[], + dependency_groups="abc", # should be a sequence of str + ) + with pytest.raises(PylockValidationError) as exc_info: + pylock.validate() + assert str(exc_info.value) == ( + "Unexpected type str (expected Sequence) in 'dependency-groups'" + ) From 394ed848233981463961e9090a74db4669d488a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sun, 12 Oct 2025 15:55:05 +0200 Subject: [PATCH 23/24] pylock: make internal exception class private This was not meant to be public --- src/packaging/pylock.py | 8 ++++---- tests/test_pylock.py | 7 +++---- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/packaging/pylock.py b/src/packaging/pylock.py index 69d65f88a..43daed9d1 100644 --- a/src/packaging/pylock.py +++ b/src/packaging/pylock.py @@ -96,7 +96,7 @@ def _get(d: Mapping[str, Any], expected_type: type[_T], key: str) -> _T | None: def _get_required(d: Mapping[str, Any], expected_type: type[_T], key: str) -> _T: """Get a required value from the dictionary and verify it's the expected type.""" if (value := _get(d, expected_type, key)) is None: - raise PylockRequiredKeyError(key) + raise _PylockRequiredKeyError(key) return value @@ -150,7 +150,7 @@ def _get_required_as( """Get a required value from the dict, verify it's the expected type, and convert to the target type.""" if (value := _get_as(d, expected_type, target_type, key)) is None: - raise PylockRequiredKeyError(key) + raise _PylockRequiredKeyError(key) return value @@ -209,7 +209,7 @@ def _get_required_sequence_of_objects( """Get a required list value from the dictionary and convert its items to a dataclass.""" if (result := _get_sequence_of_objects(d, target_item_type, key)) is None: - raise PylockRequiredKeyError(key) + raise _PylockRequiredKeyError(key) return result @@ -263,7 +263,7 @@ def __str__(self) -> str: return self.message -class PylockRequiredKeyError(PylockValidationError): +class _PylockRequiredKeyError(PylockValidationError): def __init__(self, key: str) -> None: super().__init__("Missing required value", context=key) diff --git a/tests/test_pylock.py b/tests/test_pylock.py index 5d7cc145c..dfc45eac8 100644 --- a/tests/test_pylock.py +++ b/tests/test_pylock.py @@ -15,7 +15,6 @@ PackageVcs, PackageWheel, Pylock, - PylockRequiredKeyError, PylockUnsupportedVersionError, PylockValidationError, is_valid_pylock_path, @@ -104,7 +103,7 @@ def test_pylock_missing_version() -> None: "created-by": "pip", "packages": [], } - with pytest.raises(PylockRequiredKeyError) as exc_info: + with pytest.raises(PylockValidationError) as exc_info: Pylock.from_dict(data) assert str(exc_info.value) == "Missing required value in 'lock-version'" @@ -114,7 +113,7 @@ def test_pylock_missing_created_by() -> None: "lock-version": "1.0", "packages": [], } - with pytest.raises(PylockRequiredKeyError) as exc_info: + with pytest.raises(PylockValidationError) as exc_info: Pylock.from_dict(data) assert str(exc_info.value) == "Missing required value in 'created-by'" @@ -124,7 +123,7 @@ def test_pylock_missing_packages() -> None: "lock-version": "1.0", "created-by": "uv", } - with pytest.raises(PylockRequiredKeyError) as exc_info: + with pytest.raises(PylockValidationError) as exc_info: Pylock.from_dict(data) assert str(exc_info.value) == "Missing required value in 'packages'" From 96ce64a0e8ecc443f99d1f3e390f72695f8ba4ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Fri, 17 Oct 2025 11:02:51 +0200 Subject: [PATCH 24/24] pylock: validate attestation-identities kind field --- src/packaging/pylock.py | 7 ++++++ tests/test_pylock.py | 55 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+) diff --git a/src/packaging/pylock.py b/src/packaging/pylock.py index 43daed9d1..b7ec8ec46 100644 --- a/src/packaging/pylock.py +++ b/src/packaging/pylock.py @@ -541,6 +541,13 @@ def _from_dict(cls, d: Mapping[str, Any]) -> Self: "Exactly one of vcs, directory, archive must be set " "if sdist and wheels are not set" ) + for i, attestation_identity in enumerate(package.attestation_identities or []): + try: + _get_required(attestation_identity, str, "kind") + except Exception as e: + raise PylockValidationError( + e, context=f"attestation-identities[{i}]" + ) from e return package @property diff --git a/tests/test_pylock.py b/tests/test_pylock.py index dfc45eac8..c4246ca8d 100644 --- a/tests/test_pylock.py +++ b/tests/test_pylock.py @@ -521,3 +521,58 @@ def test_validate_sequence_of_str() -> None: assert str(exc_info.value) == ( "Unexpected type str (expected Sequence) in 'dependency-groups'" ) + + +def test_validate_attestation_identity_missing_kind() -> None: + data = { + "lock-version": "1.0", + "created-by": "pip", + "packages": [ + { + "name": "example", + "version": "1.0", + "directory": { + "path": ".", + }, + "attestation-identities": [ + { + # missing "kind" field + "value": "some-value", + } + ], + } + ], + } + with pytest.raises(PylockValidationError) as exc_info: + Pylock.from_dict(data) + assert str(exc_info.value) == ( + "Missing required value in 'packages[0].attestation-identities[0].kind'" + ) + + +def test_validate_attestation_identity_invalid_kind() -> None: + data = { + "lock-version": "1.0", + "created-by": "pip", + "packages": [ + { + "name": "example", + "version": "1.0", + "directory": { + "path": ".", + }, + "attestation-identities": [ + { + "kind": 123, + "value": "some-value", + } + ], + } + ], + } + with pytest.raises(PylockValidationError) as exc_info: + Pylock.from_dict(data) + assert str(exc_info.value) == ( + "Unexpected type int (expected str) " + "in 'packages[0].attestation-identities[0].kind'" + )