From 083ccba9d4c6270e6ce5db28280cd6b1f976185d Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Wed, 2 Apr 2025 15:54:00 +0800 Subject: [PATCH 01/18] refactor: extract metadata generation logic and decouple from wheel 0.30.0 --- azdev/__init__.py | 2 +- azdev/operations/extensions/metadata.py | 504 ++++++++++++++++++++++++ azdev/operations/extensions/util.py | 9 +- setup.py | 2 +- 4 files changed, 509 insertions(+), 8 deletions(-) create mode 100644 azdev/operations/extensions/metadata.py diff --git a/azdev/__init__.py b/azdev/__init__.py index 4322e44ee..d5d2fb9d8 100644 --- a/azdev/__init__.py +++ b/azdev/__init__.py @@ -4,4 +4,4 @@ # license information. # ----------------------------------------------------------------------------- -__VERSION__ = '0.2.2' +__VERSION__ = '0.2.2a1' diff --git a/azdev/operations/extensions/metadata.py b/azdev/operations/extensions/metadata.py new file mode 100644 index 000000000..1f9635688 --- /dev/null +++ b/azdev/operations/extensions/metadata.py @@ -0,0 +1,504 @@ +""" +Tools for converting old- to new-style metadata. +""" + +import email.parser +import os.path +import re +import textwrap +import zipfile + +from collections import OrderedDict, namedtuple + +from importlib.metadata import entry_points +from packaging.requirements import Requirement # pip install packagin + +METADATA_VERSION = "2.0" + +PLURAL_FIELDS = {"classifier": "classifiers", + "provides_dist": "provides", + "provides_extra": "extras"} + +SKIP_FIELDS = set() + +CONTACT_FIELDS = (({"email": "author_email", "name": "author"}, + "author"), + ({"email": "maintainer_email", "name": "maintainer"}, + "maintainer")) + +# commonly filled out as "UNKNOWN" by distutils: +UNKNOWN_FIELDS = {"author", "author_email", "platform", "home_page", "license"} + +# Wheel itself is probably the only program that uses non-extras markers +# in METADATA/PKG-INFO. Support its syntax with the extra at the end only. +EXTRA_RE = re.compile("""^(?P.*?)(;\s*(?P.*?)(extra == '(?P.*?)')?)$""") +KEYWORDS_RE = re.compile("[\0-,]+") + +MayRequiresKey = namedtuple('MayRequiresKey', ('condition', 'extra')) + + +class OrderedDefaultDict(OrderedDict): + def __init__(self, *args, **kwargs): + if not args: + self.default_factory = None + else: + if not (args[0] is None or callable(args[0])): + raise TypeError('first argument must be callable or None') + self.default_factory = args[0] + args = args[1:] + super(OrderedDefaultDict, self).__init__(*args, **kwargs) + + def __missing__(self, key): + if self.default_factory is None: + raise KeyError(key) + self[key] = default = self.default_factory() + return default + + +def read_pkg_info(path): + with zipfile.ZipFile(path, 'r') as zf: + for file_name in zf.namelist(): + if file_name.endswith("METADATA"): + with zf.open(file_name, "r") as metadata_file: + content = email.parser.Parser().parsestr(metadata_file.read().decode("utf-8")) + return content + return {} + + +def unique(iterable): + """ + Yield unique values in iterable, preserving order. + """ + seen = set() + for value in iterable: + if value not in seen: + seen.add(value) + yield value + + +def handle_requires(metadata, pkg_info, key): + """ + Place the runtime requirements from pkg_info into metadata. + Ensures requirements are in standard format without spaces around ~= + """ + # may_requires = DefaultDict(list) + may_requires = OrderedDefaultDict(list) + for value in sorted(pkg_info.get_all(key)): + extra_match = EXTRA_RE.search(value) + if extra_match: + groupdict = extra_match.groupdict() + condition = groupdict['condition'] + extra = groupdict['extra'] + package = groupdict['package'] + if condition.endswith(' and '): + condition = condition[:-5] + else: + condition, extra = None, None + package = value + + # Standardize the package requirement format by removing spaces around ~= + if ' ~=' in package: + package = package.replace(' ~=', '~=') + + key = MayRequiresKey(condition, extra) + may_requires[key].append(package) + + if may_requires: + metadata['run_requires'] = [] + + def sort_key(item): + # Both condition and extra could be None, which can't be compared + # against strings in Python 3. + key, value = item + if key.condition is None: + return '' + return key.condition + + for key, value in sorted(may_requires.items(), key=sort_key): + may_requirement = OrderedDict((('requires', value),)) + # may_requirement = defaultdict((('requires', value),)) + if key.extra: + may_requirement['extra'] = key.extra + if key.condition: + may_requirement['environment'] = key.condition + metadata['run_requires'].append(may_requirement) + + if 'extras' not in metadata: + metadata['extras'] = [] + metadata['extras'].extend([key.extra for key in may_requires.keys() if key.extra]) + + +def get_wheel_generator(whl_path): + """ + Extract the Generator value from the WHEEL file in a .whl package. + + Args: + whl_path (str): Path to the .whl file. + + Returns: + str: The Generator value, or None if not found. + """ + try: + # Ensure the file is a valid zip file + if not zipfile.is_zipfile(whl_path): + print(f"The file {whl_path} is not a valid zip file.") + return "bdist_wheel" + + # Open the .whl file + with zipfile.ZipFile(whl_path, 'r') as zf: + # Locate the WHEEL file + wheel_file = None + for file_name in zf.namelist(): + if ".dist-info/WHEEL" in file_name: + wheel_file = file_name + break + + if not wheel_file: + print(f"WHEEL file not found in {whl_path}.") + return "bdist_wheel" + + # Read and parse the WHEEL file + with zf.open(wheel_file) as wf: + for line in wf: + decoded_line = line.decode("utf-8").strip() + if decoded_line.startswith("Generator:"): + _, generator_value = decoded_line.split(":", 1) + return generator_value.strip() + + print(f"Generator key not found in {wheel_file}.") + return "bdist_wheel" + + except Exception as e: + print(f"An error occurred while processing {whl_path}: {e}") + return "bdist_wheel" + + +def pkginfo_to_dict(path, distribution=None): + """ + Convert PKG-INFO to a prototype Metadata 2.0 (PEP 426) dict. + + The description is included under the key ['description'] rather than + being written to a separate file. + + path: path to wheel file + distribution: optional distutils Distribution() + """ + # metadata = DefaultDict( + # lambda: DefaultDict( lambda: DefaultDict(defaultdict))) + + metadata = OrderedDefaultDict( + lambda: OrderedDefaultDict(lambda: OrderedDefaultDict(OrderedDict))) + + metadata["generator"] = get_wheel_generator(path) + try: + pkg_info = read_pkg_info(path) + except Exception: + with open(path, 'rb') as pkg_info_file: + pkg_info = email.parser.Parser().parsestr(pkg_info_file.read().decode('utf-8')) + + # description = None + + if pkg_info['Summary']: + metadata['summary'] = pkginfo_unicode(pkg_info, 'Summary') + del pkg_info['Summary'] + + # if pkg_info['Description']: + # description = dedent_description(pkg_info) + # del pkg_info['Description'] + # else: + # payload = pkg_info.get_payload() + # if isinstance(payload, bytes): + # # Avoid a Python 2 Unicode error. + # # We still suffer ? glyphs on Python 3. + # payload = payload.decode('utf-8') + # if payload: + # description = payload + + # if description: + # pkg_info['description'] = description + + for key in sorted(unique(k.lower() for k in pkg_info.keys())): + low_key = key.replace('-', '_') + + if low_key in SKIP_FIELDS: + continue + + if low_key in UNKNOWN_FIELDS and pkg_info.get(key) == 'UNKNOWN': + continue + + if low_key in sorted(PLURAL_FIELDS): + metadata[PLURAL_FIELDS[low_key]] = pkg_info.get_all(key) + + elif low_key == "requires_dist": + handle_requires(metadata, pkg_info, key) + + elif low_key == 'provides_extra': + if 'extras' not in metadata: + metadata['extras'] = [] + metadata['extras'].extend(pkg_info.get_all(key)) + + elif low_key == 'home_page': + metadata['extensions']['python.details']['project_urls'] = {'Home': pkg_info[key]} + + elif low_key == 'keywords': + metadata['keywords'] = KEYWORDS_RE.split(pkg_info[key]) + + else: + metadata[low_key] = pkg_info[key] + + metadata['metadata_version'] = METADATA_VERSION + + if 'extras' in metadata: + metadata['extras'] = sorted(set(metadata['extras'])) + + # include more information if distribution is available + if distribution: + for requires, attr in (('test_requires', 'tests_require'),): + try: + requirements = getattr(distribution, attr) + if isinstance(requirements, list): + new_requirements = sorted(convert_requirements(requirements)) + metadata[requires] = [{'requires': new_requirements}] + except AttributeError: + pass + + # handle contacts + contacts = [] + for contact_type, role in CONTACT_FIELDS: + contact = OrderedDict() + # contact = defaultdict() + for key in sorted(contact_type): + if contact_type[key] in metadata: + contact[key] = metadata.pop(contact_type[key]) + if contact: + contact['role'] = role + contacts.append(contact) + if contacts: + metadata['extensions']['python.details']['contacts'] = contacts + + # handle document_names + # check for DESCRIPTION.rst file in the wheel package + if zipfile.is_zipfile(path): + with zipfile.ZipFile(path, 'r') as zf: + for file_name in zf.namelist(): + if 'DESCRIPTION.rst' in file_name: + metadata['extensions']['python.details']['document_names'] = { + 'description': 'DESCRIPTION.rst' + } + + # convert entry points to exports + try: + with open(os.path.join(os.path.dirname(path), "entry_points.txt"), "r") as ep_file: + ep_map = entry_points() + # ep_map = pkg_resources.EntryPoint.parse_map(ep_file.read()) + exports = OrderedDict() + # exports = defaultdict() + for group, items in sorted(ep_map.items()): + exports[group] = OrderedDict() + # exports[group] = defaultdict() + for item in sorted(map(str, items.values())): + name, export = item.split(' = ', 1) + exports[group][name] = export + if exports: + metadata['extensions']['python.exports'] = exports + except IOError: + pass + + # copy console_scripts entry points to commands + if 'python.exports' in metadata['extensions']: + for (ep_script, wrap_script) in (('console_scripts', 'wrap_console'), + ('gui_scripts', 'wrap_gui')): + if ep_script in metadata['extensions']['python.exports']: + metadata['extensions']['python.commands'][wrap_script] = \ + metadata['extensions']['python.exports'][ep_script] + + return recursive_ordered_to_dict(metadata) + + +def recursive_ordered_to_dict(obj): + """ + Recursively process a data structure to convert OrderedDict and dict to sorted dict. + + Args: + obj: The input object to process (can be OrderedDict, dict, list, or any type). + + Returns: + dict: A dict-transformed version of the input object, with keys sorted. + """ + if isinstance(obj, (OrderedDict, dict)): + # Convert to dict and recursively process values, sorting keys + return {k: recursive_ordered_to_dict(v) for k, v in sorted(obj.items())} + elif isinstance(obj, list): + # Recursively process each item in the list + return [recursive_ordered_to_dict(item) for item in obj] + else: + # Return the object as-is for non-iterable types + return obj + + +def requires_to_requires_dist(requirement): + """Compose the version predicates for requirement in PEP 345 fashion.""" + requires_dist = [] + for op, ver in requirement.specs: + requires_dist.append(op + ver) + if not requires_dist: + return '' + return " (%s)" % ','.join(sorted(requires_dist)) + + +def convert_requirements(requirements): + """Yield Requires-Dist: strings for parsed requirements strings.""" + for req in requirements: + parsed_requirement = Requirement(req) + # parsed_requirement = pkg_resources.Requirement.parse(req) + spec = requires_to_requires_dist(parsed_requirement) + extras = ",".join(parsed_requirement.extras) + if extras: + extras = "[%s]" % extras + yield (parsed_requirement.project_name + extras + spec) + + +def safe_extra(extra): + """Mimics pkg_resources.safe_extra functionality. + Convert an arbitrary string to a standard 'extra' name + + Any runs of non-alphanumeric characters are replaced with a single '_', + and the result is always lowercased. + """ + return re.sub('[^A-Za-z0-9.-]+', '_', extra).lower() + + +def generate_requirements(extras_require): + """ + Convert requirements from a setup()-style dictionary to ('Requires-Dist', 'requirement') + and ('Provides-Extra', 'extra') tuples. + + extras_require is a dictionary of {extra: [requirements]} as passed to setup(), + using the empty extra {'': [requirements]} to hold install_requires. + """ + for extra, depends in extras_require.items(): + condition = '' + if extra and ':' in extra: # setuptools extra:condition syntax + extra, condition = extra.split(':', 1) + extra = safe_extra(extra) + # extra = pkg_resources.safe_extra(extra) + if extra: + yield ('Provides-Extra', extra) + if condition: + condition += " and " + condition += "extra == '%s'" % extra + if condition: + condition = '; ' + condition + for new_req in convert_requirements(depends): + yield ('Requires-Dist', new_req + condition) + + +def split_sections(s): + """Mimics pkg_resources.split_sections. + + Split a string or iterable thereof into (section, content) pairs. + + Each ``section`` is a stripped version of the section header ("[section]"), + and each ``content`` is a list of stripped lines excluding blank lines and + comment-only lines. If there are any such lines before the first section + header, they're returned in a first ``section`` of ``None``. + """ + + def yield_lines(content): + """Yields non-blank, non-comment lines from the input.""" + for line in content: + line = line.strip() + if line and not line.startswith("#"): + yield line + + section = None + content = [] + for line in yield_lines(s): + if line.startswith("[") and line.endswith("]"): + if section or content: + yield section, content + section = line[1:-1].strip() + content = [] + elif line.startswith("[") and not line.endswith("]"): + raise ValueError("Invalid section heading", line) + else: + content.append(line) + + # wrap up the last segment + if section or content: + yield section, content + + +def pkginfo_to_metadata(egg_info_path, pkginfo_path): + """ + Convert .egg-info directory with PKG-INFO to the Metadata 1.3 aka + old-draft Metadata 2.0 format. + """ + pkg_info = read_pkg_info(pkginfo_path) + pkg_info.replace_header('Metadata-Version', '2.0') + requires_path = os.path.join(egg_info_path, 'requires.txt') + if os.path.exists(requires_path): + with open(requires_path) as requires_file: + requires = requires_file.read() + for extra, reqs in sorted(split_sections(requires), key=lambda x: x[0] or ''): + # for extra, reqs in sorted(pkg_resources.split_sections(requires), key=lambda x: x[0] or ''): + for item in generate_requirements({extra: reqs}): + pkg_info[item[0]] = item[1] + + description = pkg_info['Description'] + if description: + pkg_info.set_payload(dedent_description(pkg_info)) + del pkg_info['Description'] + + return pkg_info + + +def pkginfo_unicode(pkg_info, field): + """Hack to coax Unicode out of an email Message() - Python 3.3+""" + text = pkg_info[field] + field = field.lower() + if not isinstance(text, str): + if not hasattr(pkg_info, 'raw_items'): # Python 3.2 + return str(text) + for item in pkg_info.raw_items(): + if item[0].lower() == field: + text = item[1].encode('ascii', 'surrogateescape') \ + .decode('utf-8') + break + + return text + + +def dedent_description(pkg_info): + """ + Dedent and convert pkg_info['Description'] to Unicode. + """ + description = pkg_info['Description'] + + # Python 3 Unicode handling, sorta. + surrogates = False + if not isinstance(description, str): + surrogates = True + description = pkginfo_unicode(pkg_info, 'Description') + + description_lines = description.splitlines() + description_dedent = '\n'.join( + # if the first line of long_description is blank, + # the first line here will be indented. + (description_lines[0].lstrip(), + textwrap.dedent('\n'.join(description_lines[1:])), + '\n')) + + if surrogates: + description_dedent = description_dedent \ + .encode("utf8") \ + .decode("ascii", "surrogateescape") + + return description_dedent + + +if __name__ == "__main__": + import sys + import pprint + + pprint.pprint(pkginfo_to_dict(sys.argv[1])) diff --git a/azdev/operations/extensions/util.py b/azdev/operations/extensions/util.py index ead988fd8..f51942e21 100644 --- a/azdev/operations/extensions/util.py +++ b/azdev/operations/extensions/util.py @@ -12,6 +12,7 @@ from knack.util import CLIError from azdev.utilities import EXTENSION_PREFIX +from azdev.operations.extensions.metadata import pkginfo_to_dict WHEEL_INFO_RE = re.compile( @@ -44,8 +45,7 @@ def _get_azext_metadata(ext_dir): def get_ext_metadata(ext_dir, ext_file, ext_name): - # Modification of https://github.com/Azure/azure-cli/blob/dev/src/azure-cli-core/azure/cli/core/extension.py#L89 - WHL_METADATA_FILENAME = 'metadata.json' + generated_metadata = pkginfo_to_dict(ext_file) with zipfile.ZipFile(ext_file, 'r') as zip_ref: zip_ref.extractall(ext_dir) metadata = {} @@ -56,10 +56,7 @@ def get_ext_metadata(ext_dir, ext_file, ext_name): for dist_info_dirname in dist_info_dirs: parsed_dist_info_dir = WHEEL_INFO_RE(dist_info_dirname) if parsed_dist_info_dir and parsed_dist_info_dir.groupdict().get('name') == ext_name.replace('-', '_'): - whl_metadata_filepath = os.path.join(ext_dir, dist_info_dirname, WHL_METADATA_FILENAME) - if os.path.isfile(whl_metadata_filepath): - with open(whl_metadata_filepath) as f: - metadata.update(json.load(f)) + metadata.update(generated_metadata) return metadata diff --git a/setup.py b/setup.py index f211a0249..0d27016dc 100644 --- a/setup.py +++ b/setup.py @@ -86,7 +86,7 @@ 'azure-cli-diff-tool~=0.1.0', 'packaging', 'tqdm', - 'wheel==0.30.0', + 'setuptools', 'microsoft-security-utilities-secret-masker~=1.0.0b4' ], package_data={ From 09f9ba01d3490b973042009fd0e7ec1528105df3 Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Mon, 7 Apr 2025 11:31:44 +0800 Subject: [PATCH 02/18] Add test for metadata --- azdev/operations/extensions/metadata.py | 2 +- azdev/operations/tests/test_metadata.py | 171 ++++++++++++++++++++++++ 2 files changed, 172 insertions(+), 1 deletion(-) create mode 100644 azdev/operations/tests/test_metadata.py diff --git a/azdev/operations/extensions/metadata.py b/azdev/operations/extensions/metadata.py index 1f9635688..d1117d254 100644 --- a/azdev/operations/extensions/metadata.py +++ b/azdev/operations/extensions/metadata.py @@ -31,7 +31,7 @@ # Wheel itself is probably the only program that uses non-extras markers # in METADATA/PKG-INFO. Support its syntax with the extra at the end only. -EXTRA_RE = re.compile("""^(?P.*?)(;\s*(?P.*?)(extra == '(?P.*?)')?)$""") +EXTRA_RE = re.compile(r"""^(?P.*?)(;\s*(?P.*?)(extra == '(?P.*?)')?)$""") KEYWORDS_RE = re.compile("[\0-,]+") MayRequiresKey = namedtuple('MayRequiresKey', ('condition', 'extra')) diff --git a/azdev/operations/tests/test_metadata.py b/azdev/operations/tests/test_metadata.py new file mode 100644 index 000000000..ad5b487eb --- /dev/null +++ b/azdev/operations/tests/test_metadata.py @@ -0,0 +1,171 @@ +import os +import requests +from deepdiff import DeepDiff +from azdev.operations.extensions.metadata import pkginfo_to_dict + + +def download_wheel(url, dest_dir): + """ + Download wheel file from URL to destination directory + """ + if not os.path.exists(dest_dir): + os.makedirs(dest_dir) + + filename = os.path.basename(url) + dest_path = os.path.join(dest_dir, filename) + + response = requests.get(url) + response.raise_for_status() + + with open(dest_path, 'wb') as f: + f.write(response.content) + + return dest_path + + +def clean_metadata(original_metadata): + """ + Remove specified keys from the metadata. + :param original_metadata: Original metadata + :return: Cleaned metadata + """ + keys_to_remove = { + "azext.isPreview", + "azext.minCliCoreVersion", + "azext.isExperimental", + "azext.isExprimental", + "azext.maxCliCoreVersion" + } + return {k: v for k, v in original_metadata.items() if k not in keys_to_remove} + + +def compare_metadata(wheel_url, expected_metadata): + """ + Compare metadata between wheel file and expected metadata + """ + temp_dir = 'temp_wheels' + + try: + # Download the wheel + print(f"Downloading wheel from {wheel_url}") + wheel_path = download_wheel(wheel_url, temp_dir) + + # Get metadata from wheel + wheel_metadata = pkginfo_to_dict(wheel_path) + + # Compare metadata + expected_metadata_cleaned = clean_metadata(expected_metadata) + print(f"Metadata from index.json: \n{expected_metadata}") + print(f"Metadata from index.json cleaned: \n{expected_metadata_cleaned}") + print(f"Metadata from python wheel package: \n{wheel_metadata}") + diff = DeepDiff(wheel_metadata, expected_metadata_cleaned, ignore_order=True) + + if diff: + print("Metadata mismatch found:") + print(f"Differences:\n{diff}") + return False + + print("Metadata built from python wheel package matches metadata from index.json.") + return True + + finally: + # Cleanup + if os.path.exists(temp_dir): + import shutil + shutil.rmtree(temp_dir) + +def test_specific_wheel(): + """ + Test specific wheel metadata consistency + """ + wheel_url = "https://azuremlsdktestpypi.blob.core.windows.net/wheels/sdk-cli-v2-public/ml-2.36.1-py3-none-any.whl" + metadata_from_index = { + "azext.minCliCoreVersion": "2.15.0", + "classifiers": [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Intended Audience :: System Administrators", + "Environment :: Console", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "License :: OSI Approved :: MIT License" + ], + "description_content_type": "text/x-rst", + "extensions": { + "python.details": { + "contacts": [ + { + "email": "azuremlsdk@microsoft.com", + "name": "Microsoft Corporation", + "role": "author" + } + ], + "document_names": { + "description": "DESCRIPTION.rst" + }, + "project_urls": { + "Home": "https://docs.microsoft.com/azure/machine-learning/azure-machine-learning-release-notes-cli-v2?view=azureml-api-2" + } + } + }, + "extras": [], + "generator": "bdist_wheel (0.30.0)", + "license": "MIT", + "metadata_version": "2.0", + "name": "ml", + "run_requires": [ + { + "requires": [ + "azure-common (>=1.1)", + "azure-common>=1.1", + "azure-identity (==1.17.1)", + "azure-identity==1.17.1", + "azure-mgmt-resource (<23.0.0,>=3.0.0)", + "azure-mgmt-resource<23.0.0,>=3.0.0", + "azure-mgmt-resourcegraph (<9.0.0,>=2.0.0)", + "azure-mgmt-resourcegraph<9.0.0,>=2.0.0", + "azure-monitor-opentelemetry", + "azure-monitor-opentelemetry", + "azure-storage-blob (>=12.10.0)", + "azure-storage-blob>=12.10.0", + "azure-storage-file-datalake (>=12.2.0)", + "azure-storage-file-datalake>=12.2.0", + "azure-storage-file-share", + "azure-storage-file-share", + "colorama", + "colorama", + "cryptography", + "cryptography", + "docker", + "docker", + "isodate", + "isodate", + "jsonschema (>=4.0.0)", + "jsonschema>=4.0.0", + "marshmallow (>=3.5)", + "marshmallow>=3.5", + "pydash (>=6.0.0)", + "pydash>=6.0.0", + "pyjwt", + "pyjwt", + "strictyaml", + "strictyaml", + "tqdm", + "tqdm", + "typing-extensions", + "typing-extensions" + ] + } + ], + "summary": "Microsoft Azure Command-Line Tools AzureMachineLearningWorkspaces Extension", + "version": "2.36.1" + } + + assert compare_metadata(wheel_url, metadata_from_index), "Metadata comparison failed" + +if __name__ == "__main__": + test_specific_wheel() From fd9e3847628723e57316f26037a12d2617afe455 Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Mon, 7 Apr 2025 11:35:02 +0800 Subject: [PATCH 03/18] update history --- HISTORY.rst | 4 ++++ azdev/__init__.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index bf7b6f596..f761cb6f6 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -2,6 +2,10 @@ Release History =============== +0.2.3b1 +++++++ +* extract metadata generation logic and decouple from wheel 0.30.0 + 0.2.2 ++++++ * Update dependency `azure-cli-diff-tool` to `0.1.0`. diff --git a/azdev/__init__.py b/azdev/__init__.py index d5d2fb9d8..24bae1bbd 100644 --- a/azdev/__init__.py +++ b/azdev/__init__.py @@ -4,4 +4,4 @@ # license information. # ----------------------------------------------------------------------------- -__VERSION__ = '0.2.2a1' +__VERSION__ = '0.2.3b1' From dc7d04a1883a2d76c08d59e16debc346bad1132a Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Mon, 7 Apr 2025 11:41:41 +0800 Subject: [PATCH 04/18] Add license headers --- azdev/operations/extensions/metadata.py | 8 +++++++- azdev/operations/tests/test_metadata.py | 6 ++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/azdev/operations/extensions/metadata.py b/azdev/operations/extensions/metadata.py index d1117d254..2b6e8212f 100644 --- a/azdev/operations/extensions/metadata.py +++ b/azdev/operations/extensions/metadata.py @@ -1,5 +1,11 @@ +# ----------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# ----------------------------------------------------------------------------- + """ -Tools for converting old- to new-style metadata. +Tools for generate metadata for index.json. """ import email.parser diff --git a/azdev/operations/tests/test_metadata.py b/azdev/operations/tests/test_metadata.py index ad5b487eb..cf871bd0a 100644 --- a/azdev/operations/tests/test_metadata.py +++ b/azdev/operations/tests/test_metadata.py @@ -1,3 +1,9 @@ +# ----------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# ----------------------------------------------------------------------------- + import os import requests from deepdiff import DeepDiff From 6aa5a045b1c106240054b6d33bdc86613629d2a0 Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Mon, 7 Apr 2025 13:22:50 +0800 Subject: [PATCH 05/18] minor fix --- HISTORY.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index f761cb6f6..bc6a53e30 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -3,7 +3,7 @@ Release History =============== 0.2.3b1 -++++++ ++++++++ * extract metadata generation logic and decouple from wheel 0.30.0 0.2.2 @@ -15,7 +15,7 @@ Release History * `azdev extension cal-next-version`: Adjust `minor` or `patch` update for previous preview versioning pattern. 0.2.0 -+++++ +++++++ * `azdev generated-breaking-change-report`: Support multi-line upcoming breaking change announcement 0.1.99 From a95ab78a733b04d121939d9fac854de9adf1ca96 Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Mon, 7 Apr 2025 13:49:39 +0800 Subject: [PATCH 06/18] minor fix --- azdev/operations/extensions/metadata.py | 2 ++ azdev/operations/tests/test_metadata.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/azdev/operations/extensions/metadata.py b/azdev/operations/extensions/metadata.py index 2b6e8212f..b647985b3 100644 --- a/azdev/operations/extensions/metadata.py +++ b/azdev/operations/extensions/metadata.py @@ -1,3 +1,5 @@ +# pylint: disable=C0325,R1725,W0612,R1704,W0718,R0914,E1101,R0912,R0915,R1705 + # ----------------------------------------------------------------------------- # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See License.txt in the project root for diff --git a/azdev/operations/tests/test_metadata.py b/azdev/operations/tests/test_metadata.py index cf871bd0a..0d57d3a72 100644 --- a/azdev/operations/tests/test_metadata.py +++ b/azdev/operations/tests/test_metadata.py @@ -1,3 +1,5 @@ +# pylint: disable=C0301 + # ----------------------------------------------------------------------------- # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See License.txt in the project root for From 20a2c2946d3877dd489ccf7d16ab9be56db9c363 Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Mon, 7 Apr 2025 13:57:46 +0800 Subject: [PATCH 07/18] minor fix --- azdev/operations/extensions/metadata.py | 4 +--- azdev/operations/tests/test_metadata.py | 2 ++ 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/azdev/operations/extensions/metadata.py b/azdev/operations/extensions/metadata.py index b647985b3..6c75e30fd 100644 --- a/azdev/operations/extensions/metadata.py +++ b/azdev/operations/extensions/metadata.py @@ -296,9 +296,7 @@ def pkginfo_to_dict(path, distribution=None): # convert entry points to exports try: - with open(os.path.join(os.path.dirname(path), "entry_points.txt"), "r") as ep_file: - ep_map = entry_points() - # ep_map = pkg_resources.EntryPoint.parse_map(ep_file.read()) + ep_map = entry_points() exports = OrderedDict() # exports = defaultdict() for group, items in sorted(ep_map.items()): diff --git a/azdev/operations/tests/test_metadata.py b/azdev/operations/tests/test_metadata.py index 0d57d3a72..64504996b 100644 --- a/azdev/operations/tests/test_metadata.py +++ b/azdev/operations/tests/test_metadata.py @@ -82,6 +82,7 @@ def compare_metadata(wheel_url, expected_metadata): import shutil shutil.rmtree(temp_dir) + def test_specific_wheel(): """ Test specific wheel metadata consistency @@ -175,5 +176,6 @@ def test_specific_wheel(): assert compare_metadata(wheel_url, metadata_from_index), "Metadata comparison failed" + if __name__ == "__main__": test_specific_wheel() From 49c528023a6b556d2160a2da5eaf296d866ff752 Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Tue, 8 Apr 2025 17:20:11 +0800 Subject: [PATCH 08/18] minor fix --- azdev/operations/extensions/metadata.py | 3 ++- azdev/operations/setup.py | 21 ++++++++++++--------- azure-pipelines-cli.yml | 2 ++ setup.py | 2 +- 4 files changed, 17 insertions(+), 11 deletions(-) diff --git a/azdev/operations/extensions/metadata.py b/azdev/operations/extensions/metadata.py index 6c75e30fd..e00c58121 100644 --- a/azdev/operations/extensions/metadata.py +++ b/azdev/operations/extensions/metadata.py @@ -296,7 +296,8 @@ def pkginfo_to_dict(path, distribution=None): # convert entry points to exports try: - ep_map = entry_points() + with open(os.path.join(os.path.dirname(path), "entry_points.txt"), "r"): + ep_map = entry_points() exports = OrderedDict() # exports = defaultdict() for group, items in sorted(ep_map.items()): diff --git a/azdev/operations/setup.py b/azdev/operations/setup.py index 3782908ee..d0501a79a 100644 --- a/azdev/operations/setup.py +++ b/azdev/operations/setup.py @@ -49,7 +49,8 @@ def _install_extensions(ext_paths): # install specified extensions for path in ext_paths or []: - result = pip_cmd('install -e {}'.format(path), "Adding extension '{}'...".format(path)) + result = pip_cmd('install -e {} --config-settings editable_mode=compat'.format(path), + "Adding extension '{}'...".format(path)) if result.error: raise result.error # pylint: disable=raising-bad-type @@ -90,38 +91,40 @@ def _install_cli(cli_path, deps=None): # Resolve dependencies from setup.py files. # command modules have dependency on azure-cli-core so install this first pip_cmd( - "install -e {}".format(os.path.join(cli_src, 'azure-cli-telemetry')), + "install -e {} --config-settings editable_mode=compat".format(os.path.join(cli_src, 'azure-cli-telemetry')), "Installing `azure-cli-telemetry`..." ) pip_cmd( - "install -e {}".format(os.path.join(cli_src, 'azure-cli-core')), + "install -e {} --config-settings editable_mode=compat".format(os.path.join(cli_src, 'azure-cli-core')), "Installing `azure-cli-core`..." ) # azure cli has dependencies on the above packages so install this one last pip_cmd( - "install -e {}".format(os.path.join(cli_src, 'azure-cli')), + "install -e {} --config-settings editable_mode=compat".format(os.path.join(cli_src, 'azure-cli')), "Installing `azure-cli`..." ) pip_cmd( - "install -e {}".format(os.path.join(cli_src, 'azure-cli-testsdk')), + "install -e {} --config-settings editable_mode=compat".format(os.path.join(cli_src, 'azure-cli-testsdk')), "Installing `azure-cli-testsdk`..." ) else: # First install packages without dependencies, # then resolve dependencies from requirements.*.txt file. pip_cmd( - "install -e {} --no-deps".format(os.path.join(cli_src, 'azure-cli-telemetry')), + "install -e {} --no-deps --config-settings editable_mode=compat".format( + os.path.join(cli_src, 'azure-cli-telemetry')), "Installing `azure-cli-telemetry`..." ) pip_cmd( - "install -e {} --no-deps".format(os.path.join(cli_src, 'azure-cli-core')), + "install -e {} --no-deps --config-settings editable_mode=compat".format( + os.path.join(cli_src, 'azure-cli-core')), "Installing `azure-cli-core`..." ) pip_cmd( - "install -e {} --no-deps".format(os.path.join(cli_src, 'azure-cli')), + "install -e {} --no-deps --config-settings editable_mode=compat".format(os.path.join(cli_src, 'azure-cli')), "Installing `azure-cli`..." ) @@ -129,7 +132,7 @@ def _install_cli(cli_path, deps=None): # azure-cli package for running commands. # Here we need to install with dependencies for azdev test. pip_cmd( - "install -e {}".format(os.path.join(cli_src, 'azure-cli-testsdk')), + "install -e {} --config-settings editable_mode=compat".format(os.path.join(cli_src, 'azure-cli-testsdk')), "Installing `azure-cli-testsdk`..." ) import platform diff --git a/azure-pipelines-cli.yml b/azure-pipelines-cli.yml index ae5f55d0c..b567ecbc1 100644 --- a/azure-pipelines-cli.yml +++ b/azure-pipelines-cli.yml @@ -103,6 +103,7 @@ jobs: azdev setup -c ./azure-cli -r ./azure-cli-extensions azdev --version + az --version python -m pytest azdev/ --ignore=azdev/mod_templates --junitxml=junit/test-results.xml --cov=azdev --cov-report=xml - task: PublishTestResults@2 @@ -280,6 +281,7 @@ jobs: set -ev . scripts/ci/install.sh azdev --version + az --version displayName: 'Azdev Setup' - bash: | set -ev diff --git a/setup.py b/setup.py index 0d27016dc..ffbb6d15c 100644 --- a/setup.py +++ b/setup.py @@ -86,7 +86,7 @@ 'azure-cli-diff-tool~=0.1.0', 'packaging', 'tqdm', - 'setuptools', + 'setuptools>=64.0.0', 'microsoft-security-utilities-secret-masker~=1.0.0b4' ], package_data={ From 6c953f1bcb50fe623e23df0398aafd5aac5df302 Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Tue, 8 Apr 2025 17:59:22 +0800 Subject: [PATCH 09/18] minor fix --- azdev/operations/setup.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/azdev/operations/setup.py b/azdev/operations/setup.py index d0501a79a..a85e82f54 100644 --- a/azdev/operations/setup.py +++ b/azdev/operations/setup.py @@ -49,8 +49,10 @@ def _install_extensions(ext_paths): # install specified extensions for path in ext_paths or []: - result = pip_cmd('install -e {} --config-settings editable_mode=compat'.format(path), - "Adding extension '{}'...".format(path)) + result = pip_cmd( + f'install -e {path} --config-settings editable_mode=compat', + f"Adding extension '{path}'..." + ) if result.error: raise result.error # pylint: disable=raising-bad-type From 45c73cf8f9615b2862f9bf0586a2afde2f6ab636 Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Tue, 8 Apr 2025 18:34:47 +0800 Subject: [PATCH 10/18] minor fix --- azdev/operations/code_gen.py | 2 +- azdev/operations/extensions/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/azdev/operations/code_gen.py b/azdev/operations/code_gen.py index e8dafc48d..758387f5c 100644 --- a/azdev/operations/code_gen.py +++ b/azdev/operations/code_gen.py @@ -297,6 +297,6 @@ def _create_package(prefix, repo_path, is_ext, name='test', display_name=None, d _generate_files(env, kwargs, test_files, dest_path) if is_ext: - result = pip_cmd('install -e {}'.format(new_package_path), "Installing `{}{}`...".format(prefix, name)) + result = pip_cmd('install -e {} editable_mode=compat'.format(new_package_path), "Installing `{}{}`...".format(prefix, name)) if result.error: raise result.error # pylint: disable=raising-bad-type diff --git a/azdev/operations/extensions/__init__.py b/azdev/operations/extensions/__init__.py index 55097253c..8ab271682 100644 --- a/azdev/operations/extensions/__init__.py +++ b/azdev/operations/extensions/__init__.py @@ -45,7 +45,7 @@ def add_extension(extensions): raise CLIError('extension(s) not found: {}'.format(' '.join(extensions))) for path in paths_to_add: - result = pip_cmd('install -e {}'.format(path), "Adding extension '{}'...".format(path)) + result = pip_cmd('install -e {} editable_mode=compat'.format(path), "Adding extension '{}'...".format(path)) if result.error: raise result.error # pylint: disable=raising-bad-type From d4a1c6d527bba7c841d42971da40814b3588df5d Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Wed, 9 Apr 2025 10:45:34 +0800 Subject: [PATCH 11/18] minor fix --- azdev/operations/extensions/metadata.py | 16 +- azdev/operations/tests/test_metadata.py | 232 +++++++++++++++--------- 2 files changed, 156 insertions(+), 92 deletions(-) diff --git a/azdev/operations/extensions/metadata.py b/azdev/operations/extensions/metadata.py index e00c58121..b8761fbe7 100644 --- a/azdev/operations/extensions/metadata.py +++ b/azdev/operations/extensions/metadata.py @@ -285,14 +285,18 @@ def pkginfo_to_dict(path, distribution=None): metadata['extensions']['python.details']['contacts'] = contacts # handle document_names - # check for DESCRIPTION.rst file in the wheel package + # check for DESCRIPTION.rst and LICENSE.txt file in the wheel package if zipfile.is_zipfile(path): with zipfile.ZipFile(path, 'r') as zf: - for file_name in zf.namelist(): - if 'DESCRIPTION.rst' in file_name: - metadata['extensions']['python.details']['document_names'] = { - 'description': 'DESCRIPTION.rst' - } + has_description = any('DESCRIPTION.rst' in name for name in zf.namelist()) + has_license = any('LICENSE.txt' in name for name in zf.namelist()) + + if has_description or has_license: + document_names = metadata['extensions']['python.details'].setdefault('document_names', {}) + if has_description: + document_names['description'] = 'DESCRIPTION.rst' + if has_license: + document_names['license'] = 'LICENSE.txt' # convert entry points to exports try: diff --git a/azdev/operations/tests/test_metadata.py b/azdev/operations/tests/test_metadata.py index 64504996b..8dd7003a9 100644 --- a/azdev/operations/tests/test_metadata.py +++ b/azdev/operations/tests/test_metadata.py @@ -67,6 +67,7 @@ def compare_metadata(wheel_url, expected_metadata): print(f"Metadata from index.json cleaned: \n{expected_metadata_cleaned}") print(f"Metadata from python wheel package: \n{wheel_metadata}") diff = DeepDiff(wheel_metadata, expected_metadata_cleaned, ignore_order=True) + if diff: print("Metadata mismatch found:") @@ -83,99 +84,158 @@ def compare_metadata(wheel_url, expected_metadata): shutil.rmtree(temp_dir) -def test_specific_wheel(): +def test_wheel(): """ Test specific wheel metadata consistency """ - wheel_url = "https://azuremlsdktestpypi.blob.core.windows.net/wheels/sdk-cli-v2-public/ml-2.36.1-py3-none-any.whl" - metadata_from_index = { - "azext.minCliCoreVersion": "2.15.0", - "classifiers": [ - "Development Status :: 5 - Production/Stable", - "Intended Audience :: Developers", - "Intended Audience :: System Administrators", - "Environment :: Console", - "Programming Language :: Python", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "License :: OSI Approved :: MIT License" - ], - "description_content_type": "text/x-rst", - "extensions": { - "python.details": { - "contacts": [ - { - "email": "azuremlsdk@microsoft.com", - "name": "Microsoft Corporation", - "role": "author" + wheel_url = [ + "https://azuremlsdktestpypi.blob.core.windows.net/wheels/sdk-cli-v2-public/ml-2.36.1-py3-none-any.whl", + "https://azurecliext.blob.core.windows.net/release/azure_cli_ml-1.41.0-py3-none-any.whl" + ] + metadata_from_index = [ + { + "azext.minCliCoreVersion": "2.15.0", + "classifiers": [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Intended Audience :: System Administrators", + "Environment :: Console", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "License :: OSI Approved :: MIT License" + ], + "description_content_type": "text/x-rst", + "extensions": { + "python.details": { + "contacts": [ + { + "email": "azuremlsdk@microsoft.com", + "name": "Microsoft Corporation", + "role": "author" + } + ], + "document_names": { + "description": "DESCRIPTION.rst" + }, + "project_urls": { + "Home": "https://docs.microsoft.com/azure/machine-learning/azure-machine-learning-release-notes-cli-v2?view=azureml-api-2" } - ], - "document_names": { - "description": "DESCRIPTION.rst" - }, - "project_urls": { - "Home": "https://docs.microsoft.com/azure/machine-learning/azure-machine-learning-release-notes-cli-v2?view=azureml-api-2" } - } + }, + "extras": [], + "generator": "bdist_wheel (0.30.0)", + "license": "MIT", + "metadata_version": "2.0", + "name": "ml", + "run_requires": [ + { + "requires": [ + "azure-common (>=1.1)", + "azure-common>=1.1", + "azure-identity (==1.17.1)", + "azure-identity==1.17.1", + "azure-mgmt-resource (<23.0.0,>=3.0.0)", + "azure-mgmt-resource<23.0.0,>=3.0.0", + "azure-mgmt-resourcegraph (<9.0.0,>=2.0.0)", + "azure-mgmt-resourcegraph<9.0.0,>=2.0.0", + "azure-monitor-opentelemetry", + "azure-monitor-opentelemetry", + "azure-storage-blob (>=12.10.0)", + "azure-storage-blob>=12.10.0", + "azure-storage-file-datalake (>=12.2.0)", + "azure-storage-file-datalake>=12.2.0", + "azure-storage-file-share", + "azure-storage-file-share", + "colorama", + "colorama", + "cryptography", + "cryptography", + "docker", + "docker", + "isodate", + "isodate", + "jsonschema (>=4.0.0)", + "jsonschema>=4.0.0", + "marshmallow (>=3.5)", + "marshmallow>=3.5", + "pydash (>=6.0.0)", + "pydash>=6.0.0", + "pyjwt", + "pyjwt", + "strictyaml", + "strictyaml", + "tqdm", + "tqdm", + "typing-extensions", + "typing-extensions" + ] + } + ], + "summary": "Microsoft Azure Command-Line Tools AzureMachineLearningWorkspaces Extension", + "version": "2.36.1" }, - "extras": [], - "generator": "bdist_wheel (0.30.0)", - "license": "MIT", - "metadata_version": "2.0", - "name": "ml", - "run_requires": [ - { - "requires": [ - "azure-common (>=1.1)", - "azure-common>=1.1", - "azure-identity (==1.17.1)", - "azure-identity==1.17.1", - "azure-mgmt-resource (<23.0.0,>=3.0.0)", - "azure-mgmt-resource<23.0.0,>=3.0.0", - "azure-mgmt-resourcegraph (<9.0.0,>=2.0.0)", - "azure-mgmt-resourcegraph<9.0.0,>=2.0.0", - "azure-monitor-opentelemetry", - "azure-monitor-opentelemetry", - "azure-storage-blob (>=12.10.0)", - "azure-storage-blob>=12.10.0", - "azure-storage-file-datalake (>=12.2.0)", - "azure-storage-file-datalake>=12.2.0", - "azure-storage-file-share", - "azure-storage-file-share", - "colorama", - "colorama", - "cryptography", - "cryptography", - "docker", - "docker", - "isodate", - "isodate", - "jsonschema (>=4.0.0)", - "jsonschema>=4.0.0", - "marshmallow (>=3.5)", - "marshmallow>=3.5", - "pydash (>=6.0.0)", - "pydash>=6.0.0", - "pyjwt", - "pyjwt", - "strictyaml", - "strictyaml", - "tqdm", - "tqdm", - "typing-extensions", - "typing-extensions" - ] - } - ], - "summary": "Microsoft Azure Command-Line Tools AzureMachineLearningWorkspaces Extension", - "version": "2.36.1" - } + { + "azext.minCliCoreVersion": "2.3.1", + "classifiers": [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "Intended Audience :: System Administrators", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9" + ], + "description_content_type": "text/x-rst", + "extensions": { + "python.details": { + "contacts": [ + { + "email": "azpycli@microsoft.com", + "name": "Microsoft Corporation", + "role": "author" + } + ], + "document_names": { + "description": "DESCRIPTION.rst", + "license": "LICENSE.txt" + }, + "project_urls": { + "Home": "https://docs.microsoft.com/python/api/overview/azure/ml/?view=azure-ml-py" + } + } + }, + "extras": [], + "generator": "bdist_wheel (0.30.0)", + "license": "Proprietary https://aka.ms/azureml-preview-sdk-license ", + "metadata_version": "2.0", + "name": "azure-cli-ml", + "requires_python": ">=3.5,<4", + "run_requires": [ + { + "requires": [ + "adal (>=1.2.1)", + "azureml-cli-common (~=1.41)", + "cryptography (<=3.3.2)", + "docker (>=3.7.2)", + "msrest (>=0.6.6)", + "pyyaml (>=5.1.0)", + "requests (>=2.21.0)" + ] + } + ], + "summary": "Microsoft Azure Command-Line Tools AzureML Command Module", + "version": "1.41.0" + } + ] - assert compare_metadata(wheel_url, metadata_from_index), "Metadata comparison failed" + for idx, url in enumerate(wheel_url): + assert compare_metadata(url, metadata_from_index[idx]), "Metadata comparison failed" if __name__ == "__main__": - test_specific_wheel() + test_wheel() From d5d7f1cfe06050ff87500c1127b8ff46f297ab67 Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Wed, 9 Apr 2025 10:52:23 +0800 Subject: [PATCH 12/18] minor fix --- azdev/operations/code_gen.py | 2 +- azdev/operations/extensions/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/azdev/operations/code_gen.py b/azdev/operations/code_gen.py index 758387f5c..3311c24d5 100644 --- a/azdev/operations/code_gen.py +++ b/azdev/operations/code_gen.py @@ -297,6 +297,6 @@ def _create_package(prefix, repo_path, is_ext, name='test', display_name=None, d _generate_files(env, kwargs, test_files, dest_path) if is_ext: - result = pip_cmd('install -e {} editable_mode=compat'.format(new_package_path), "Installing `{}{}`...".format(prefix, name)) + result = pip_cmd('install -e {} --config-settings editable_mode=compat'.format(new_package_path), "Installing `{}{}`...".format(prefix, name)) if result.error: raise result.error # pylint: disable=raising-bad-type diff --git a/azdev/operations/extensions/__init__.py b/azdev/operations/extensions/__init__.py index 8ab271682..d9588b552 100644 --- a/azdev/operations/extensions/__init__.py +++ b/azdev/operations/extensions/__init__.py @@ -45,7 +45,7 @@ def add_extension(extensions): raise CLIError('extension(s) not found: {}'.format(' '.join(extensions))) for path in paths_to_add: - result = pip_cmd('install -e {} editable_mode=compat'.format(path), "Adding extension '{}'...".format(path)) + result = pip_cmd('install -e {} --config-settings editable_mode=compat'.format(path), "Adding extension '{}'...".format(path)) if result.error: raise result.error # pylint: disable=raising-bad-type From 68b9e081d20dd2c4882b7db8d4fe1c963cb8521b Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Wed, 9 Apr 2025 11:00:09 +0800 Subject: [PATCH 13/18] minor fix --- azdev/operations/code_gen.py | 5 ++++- azdev/operations/extensions/__init__.py | 5 ++++- azdev/operations/extensions/metadata.py | 2 +- azdev/operations/tests/test_metadata.py | 1 - 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/azdev/operations/code_gen.py b/azdev/operations/code_gen.py index 3311c24d5..25f7f57c9 100644 --- a/azdev/operations/code_gen.py +++ b/azdev/operations/code_gen.py @@ -297,6 +297,9 @@ def _create_package(prefix, repo_path, is_ext, name='test', display_name=None, d _generate_files(env, kwargs, test_files, dest_path) if is_ext: - result = pip_cmd('install -e {} --config-settings editable_mode=compat'.format(new_package_path), "Installing `{}{}`...".format(prefix, name)) + result = pip_cmd( + f'install -e {new_package_path} --config-settings editable_mode=compat', + f"Installing `{prefix}{name}`..." + ) if result.error: raise result.error # pylint: disable=raising-bad-type diff --git a/azdev/operations/extensions/__init__.py b/azdev/operations/extensions/__init__.py index d9588b552..3d4f56930 100644 --- a/azdev/operations/extensions/__init__.py +++ b/azdev/operations/extensions/__init__.py @@ -45,7 +45,10 @@ def add_extension(extensions): raise CLIError('extension(s) not found: {}'.format(' '.join(extensions))) for path in paths_to_add: - result = pip_cmd('install -e {} --config-settings editable_mode=compat'.format(path), "Adding extension '{}'...".format(path)) + result = pip_cmd( + f'install -e {path} --config-settings editable_mode=compat', + f"Adding extension '{path}'..." + ) if result.error: raise result.error # pylint: disable=raising-bad-type diff --git a/azdev/operations/extensions/metadata.py b/azdev/operations/extensions/metadata.py index b8761fbe7..88ac15c14 100644 --- a/azdev/operations/extensions/metadata.py +++ b/azdev/operations/extensions/metadata.py @@ -290,7 +290,7 @@ def pkginfo_to_dict(path, distribution=None): with zipfile.ZipFile(path, 'r') as zf: has_description = any('DESCRIPTION.rst' in name for name in zf.namelist()) has_license = any('LICENSE.txt' in name for name in zf.namelist()) - + if has_description or has_license: document_names = metadata['extensions']['python.details'].setdefault('document_names', {}) if has_description: diff --git a/azdev/operations/tests/test_metadata.py b/azdev/operations/tests/test_metadata.py index 8dd7003a9..8277a1b24 100644 --- a/azdev/operations/tests/test_metadata.py +++ b/azdev/operations/tests/test_metadata.py @@ -67,7 +67,6 @@ def compare_metadata(wheel_url, expected_metadata): print(f"Metadata from index.json cleaned: \n{expected_metadata_cleaned}") print(f"Metadata from python wheel package: \n{wheel_metadata}") diff = DeepDiff(wheel_metadata, expected_metadata_cleaned, ignore_order=True) - if diff: print("Metadata mismatch found:") From 9aac32d5176704786e5e1de4d8e635d3d40dc1f3 Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Wed, 9 Apr 2025 11:29:37 +0800 Subject: [PATCH 14/18] minor fix --- azdev/operations/tests/test_metadata.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azdev/operations/tests/test_metadata.py b/azdev/operations/tests/test_metadata.py index 8277a1b24..f2be7650b 100644 --- a/azdev/operations/tests/test_metadata.py +++ b/azdev/operations/tests/test_metadata.py @@ -90,7 +90,7 @@ def test_wheel(): wheel_url = [ "https://azuremlsdktestpypi.blob.core.windows.net/wheels/sdk-cli-v2-public/ml-2.36.1-py3-none-any.whl", "https://azurecliext.blob.core.windows.net/release/azure_cli_ml-1.41.0-py3-none-any.whl" - ] + ] metadata_from_index = [ { "azext.minCliCoreVersion": "2.15.0", @@ -230,7 +230,7 @@ def test_wheel(): "summary": "Microsoft Azure Command-Line Tools AzureML Command Module", "version": "1.41.0" } - ] + ] for idx, url in enumerate(wheel_url): assert compare_metadata(url, metadata_from_index[idx]), "Metadata comparison failed" From 72a078dd619a0070a9e9c3b3d55e74da3181b58f Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Wed, 9 Apr 2025 11:49:01 +0800 Subject: [PATCH 15/18] minor fix --- azdev/operations/extensions/metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azdev/operations/extensions/metadata.py b/azdev/operations/extensions/metadata.py index 88ac15c14..a5fac1336 100644 --- a/azdev/operations/extensions/metadata.py +++ b/azdev/operations/extensions/metadata.py @@ -197,7 +197,7 @@ def pkginfo_to_dict(path, distribution=None): metadata = OrderedDefaultDict( lambda: OrderedDefaultDict(lambda: OrderedDefaultDict(OrderedDict))) - metadata["generator"] = get_wheel_generator(path) + metadata["generator"] = "bdist_wheel (0.30.0)" try: pkg_info = read_pkg_info(path) except Exception: From c7c6a5fd79ebcf9612097ef1701735fc1aca3b5b Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Wed, 9 Apr 2025 12:54:44 +0800 Subject: [PATCH 16/18] minor fix --- azdev/operations/tests/test_metadata.py | 123 ++++++++++++++++++++++-- 1 file changed, 116 insertions(+), 7 deletions(-) diff --git a/azdev/operations/tests/test_metadata.py b/azdev/operations/tests/test_metadata.py index f2be7650b..ac4c1703b 100644 --- a/azdev/operations/tests/test_metadata.py +++ b/azdev/operations/tests/test_metadata.py @@ -6,8 +6,12 @@ # license information. # ----------------------------------------------------------------------------- +import json import os +import re +import zipfile import requests + from deepdiff import DeepDiff from azdev.operations.extensions.metadata import pkginfo_to_dict @@ -47,6 +51,63 @@ def clean_metadata(original_metadata): return {k: v for k, v in original_metadata.items() if k not in keys_to_remove} +# copy from wheel==0.30.0 +WHEEL_INFO_RE = re.compile( + r"""^(?P(?P.+?)(-(?P\d.+?))?) + ((-(?P\d.*?))?-(?P.+?)-(?P.+?)-(?P.+?) + \.whl|\.dist-info)$""", + re.VERBOSE).match + + +def _get_extension_modname(ext_dir): + # Modification of https://github.com/Azure/azure-cli/blob/dev/src/azure-cli-core/azure/cli/core/extension.py#L153 + EXTENSIONS_MOD_PREFIX = 'azext_' + pos_mods = [n for n in os.listdir(ext_dir) + if n.startswith(EXTENSIONS_MOD_PREFIX) and os.path.isdir(os.path.join(ext_dir, n))] + if len(pos_mods) != 1: + raise AssertionError("Expected 1 module to load starting with " + "'{}': got {}".format(EXTENSIONS_MOD_PREFIX, pos_mods)) + return pos_mods[0] + + +def _get_azext_metadata(ext_dir): + # Modification of https://github.com/Azure/azure-cli/blob/dev/src/azure-cli-core/azure/cli/core/extension.py#L109 + AZEXT_METADATA_FILENAME = 'azext_metadata.json' + azext_metadata = None + ext_modname = _get_extension_modname(ext_dir=ext_dir) + azext_metadata_filepath = os.path.join(ext_dir, ext_modname, AZEXT_METADATA_FILENAME) + if os.path.isfile(azext_metadata_filepath): + with open(azext_metadata_filepath) as f: + azext_metadata = json.load(f) + return azext_metadata + + +def get_ext_metadata(ext_dir, ext_file, ext_name): + generated_metadata = pkginfo_to_dict(ext_file) + print(f"generated_metadata from python wheel package: \n{generated_metadata}") + + with zipfile.ZipFile(ext_file, 'r') as zip_ref: + zip_ref.extractall(ext_dir) + + metadata = {} + # dist_info_dirs = [f for f in os.listdir(ext_dir) if f.endswith('.dist-info')] + + azext_metadata = _get_azext_metadata(ext_dir) + print(f"azext_metadata from python wheel package: \n{azext_metadata}") + + if not azext_metadata: + raise ValueError('azext_metadata.json for Extension "{}" Metadata is missing'.format(ext_name)) + + metadata.update(azext_metadata) + + metadata.update(generated_metadata) + # for dist_info_dirname in dist_info_dirs: + # parsed_dist_info_dir = WHEEL_INFO_RE(dist_info_dirname) + # if parsed_dist_info_dir and parsed_dist_info_dir.groupdict().get('name') == ext_name.replace('-', '_'): + # metadata.update(generated_metadata) + return metadata + + def compare_metadata(wheel_url, expected_metadata): """ Compare metadata between wheel file and expected metadata @@ -56,17 +117,16 @@ def compare_metadata(wheel_url, expected_metadata): try: # Download the wheel print(f"Downloading wheel from {wheel_url}") - wheel_path = download_wheel(wheel_url, temp_dir) + ext_file = download_wheel(wheel_url, temp_dir) + ext_name = os.path.basename(wheel_url) # Get metadata from wheel - wheel_metadata = pkginfo_to_dict(wheel_path) + wheel_metadata = get_ext_metadata(temp_dir, ext_file, ext_name) # Compare metadata - expected_metadata_cleaned = clean_metadata(expected_metadata) - print(f"Metadata from index.json: \n{expected_metadata}") - print(f"Metadata from index.json cleaned: \n{expected_metadata_cleaned}") print(f"Metadata from python wheel package: \n{wheel_metadata}") - diff = DeepDiff(wheel_metadata, expected_metadata_cleaned, ignore_order=True) + print(f"Metadata from index.json: \n{expected_metadata}") + diff = DeepDiff(wheel_metadata, expected_metadata, ignore_order=True) if diff: print("Metadata mismatch found:") @@ -89,7 +149,8 @@ def test_wheel(): """ wheel_url = [ "https://azuremlsdktestpypi.blob.core.windows.net/wheels/sdk-cli-v2-public/ml-2.36.1-py3-none-any.whl", - "https://azurecliext.blob.core.windows.net/release/azure_cli_ml-1.41.0-py3-none-any.whl" + "https://azurecliext.blob.core.windows.net/release/azure_cli_ml-1.41.0-py3-none-any.whl", + "https://azurecliprod.blob.core.windows.net/cli-extensions/alias-0.5.2-py2.py3-none-any.whl" ] metadata_from_index = [ { @@ -229,6 +290,54 @@ def test_wheel(): ], "summary": "Microsoft Azure Command-Line Tools AzureML Command Module", "version": "1.41.0" + }, + { + "azext.isPreview": True, + "azext.minCliCoreVersion": "2.0.50.dev0", + "classifiers": [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "Intended Audience :: System Administrators", + "Programming Language :: Python", + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.4", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "License :: OSI Approved :: MIT License" + ], + "extensions": { + "python.details": { + "contacts": [ + { + "email": "t-chwong@microsoft.com", + "name": "Ernest Wong", + "role": "author" + } + ], + "document_names": { + "description": "DESCRIPTION.rst" + }, + "project_urls": { + "Home": "https://github.com/Azure/azure-cli-extensions" + } + } + }, + "extras": [], + "generator": "bdist_wheel (0.30.0)", + "license": "MIT", + "metadata_version": "2.0", + "name": "alias", + "run_requires": [ + { + "requires": [ + "jinja2 (~=2.10)" + ] + } + ], + "summary": "Support for command aliases", + "version": "0.5.2" } ] From 2fbfd50572a456789f5a80e7d7df4f0871ec15b9 Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Wed, 9 Apr 2025 15:59:27 +0800 Subject: [PATCH 17/18] minor fix --- azdev/operations/tests/test_metadata.py | 90 +------------------------ 1 file changed, 2 insertions(+), 88 deletions(-) diff --git a/azdev/operations/tests/test_metadata.py b/azdev/operations/tests/test_metadata.py index ac4c1703b..3b8f5deb8 100644 --- a/azdev/operations/tests/test_metadata.py +++ b/azdev/operations/tests/test_metadata.py @@ -115,6 +115,7 @@ def compare_metadata(wheel_url, expected_metadata): temp_dir = 'temp_wheels' try: + print(f"Metadata from index.json: \n{expected_metadata}") # Download the wheel print(f"Downloading wheel from {wheel_url}") ext_file = download_wheel(wheel_url, temp_dir) @@ -125,8 +126,7 @@ def compare_metadata(wheel_url, expected_metadata): # Compare metadata print(f"Metadata from python wheel package: \n{wheel_metadata}") - print(f"Metadata from index.json: \n{expected_metadata}") - diff = DeepDiff(wheel_metadata, expected_metadata, ignore_order=True) + diff = DeepDiff(expected_metadata, wheel_metadata, ignore_order=True) if diff: print("Metadata mismatch found:") @@ -148,96 +148,10 @@ def test_wheel(): Test specific wheel metadata consistency """ wheel_url = [ - "https://azuremlsdktestpypi.blob.core.windows.net/wheels/sdk-cli-v2-public/ml-2.36.1-py3-none-any.whl", "https://azurecliext.blob.core.windows.net/release/azure_cli_ml-1.41.0-py3-none-any.whl", "https://azurecliprod.blob.core.windows.net/cli-extensions/alias-0.5.2-py2.py3-none-any.whl" ] metadata_from_index = [ - { - "azext.minCliCoreVersion": "2.15.0", - "classifiers": [ - "Development Status :: 5 - Production/Stable", - "Intended Audience :: Developers", - "Intended Audience :: System Administrators", - "Environment :: Console", - "Programming Language :: Python", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "License :: OSI Approved :: MIT License" - ], - "description_content_type": "text/x-rst", - "extensions": { - "python.details": { - "contacts": [ - { - "email": "azuremlsdk@microsoft.com", - "name": "Microsoft Corporation", - "role": "author" - } - ], - "document_names": { - "description": "DESCRIPTION.rst" - }, - "project_urls": { - "Home": "https://docs.microsoft.com/azure/machine-learning/azure-machine-learning-release-notes-cli-v2?view=azureml-api-2" - } - } - }, - "extras": [], - "generator": "bdist_wheel (0.30.0)", - "license": "MIT", - "metadata_version": "2.0", - "name": "ml", - "run_requires": [ - { - "requires": [ - "azure-common (>=1.1)", - "azure-common>=1.1", - "azure-identity (==1.17.1)", - "azure-identity==1.17.1", - "azure-mgmt-resource (<23.0.0,>=3.0.0)", - "azure-mgmt-resource<23.0.0,>=3.0.0", - "azure-mgmt-resourcegraph (<9.0.0,>=2.0.0)", - "azure-mgmt-resourcegraph<9.0.0,>=2.0.0", - "azure-monitor-opentelemetry", - "azure-monitor-opentelemetry", - "azure-storage-blob (>=12.10.0)", - "azure-storage-blob>=12.10.0", - "azure-storage-file-datalake (>=12.2.0)", - "azure-storage-file-datalake>=12.2.0", - "azure-storage-file-share", - "azure-storage-file-share", - "colorama", - "colorama", - "cryptography", - "cryptography", - "docker", - "docker", - "isodate", - "isodate", - "jsonschema (>=4.0.0)", - "jsonschema>=4.0.0", - "marshmallow (>=3.5)", - "marshmallow>=3.5", - "pydash (>=6.0.0)", - "pydash>=6.0.0", - "pyjwt", - "pyjwt", - "strictyaml", - "strictyaml", - "tqdm", - "tqdm", - "typing-extensions", - "typing-extensions" - ] - } - ], - "summary": "Microsoft Azure Command-Line Tools AzureMachineLearningWorkspaces Extension", - "version": "2.36.1" - }, { "azext.minCliCoreVersion": "2.3.1", "classifiers": [ From e0f45b80d2a1457dcc09deecf3d11e0d74ba8687 Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Wed, 9 Apr 2025 21:57:45 +0800 Subject: [PATCH 18/18] minor fix --- azdev/operations/extensions/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azdev/operations/extensions/__init__.py b/azdev/operations/extensions/__init__.py index 3d4f56930..9068982f8 100644 --- a/azdev/operations/extensions/__init__.py +++ b/azdev/operations/extensions/__init__.py @@ -46,7 +46,7 @@ def add_extension(extensions): for path in paths_to_add: result = pip_cmd( - f'install -e {path} --config-settings editable_mode=compat', + f'install -e {path} --config-settings editable_mode=compat --no-build-isolation', f"Adding extension '{path}'..." ) if result.error: