From cc79494f0217655dbd7c84cd17c18ed0b183c6d7 Mon Sep 17 00:00:00 2001 From: Silvio Date: Tue, 14 Jan 2025 19:44:48 +0100 Subject: [PATCH 1/3] Allow to specify a different build number for a single package This commit allow to specify a different build number for a single package, to allow to perform a fix to a build of a single package, without the need to do a full rebuild. This is an advanced option, so special care is needed: * A rebuild is only possible if it does not change the version or in any other way the ABI of the package. * On the following full rebuild, it is important to make sure that the build number select is higher then any custom build number used for single packages. --- vinca/main.py | 39 +++++++++++++++++++++++---------------- vinca/template.py | 7 ++++--- vinca/utils.py | 17 +++++++++++++++++ 3 files changed, 44 insertions(+), 19 deletions(-) diff --git a/vinca/main.py b/vinca/main.py index 6721430..fc218c7 100644 --- a/vinca/main.py +++ b/vinca/main.py @@ -16,7 +16,7 @@ from .distro import Distro from vinca import config -from vinca.utils import get_repodata +from vinca.utils import get_repodata, get_pkg_build_number unsatisfied_deps = set() distro = None @@ -218,6 +218,11 @@ def read_vinca_yaml(filepath): vinca_conf["trigger_new_versions"] = vinca_conf.get("trigger_new_versions", False) + if (Path(filepath).parent / "pkg_additional_info.yaml").exists(): + vinca_conf["_pkg_additional_info"] = yaml.load(open(Path(filepath).parent / "pkg_additional_info.yaml")) + else: + vinca_conf["_pkg_additional_info"] = {} + return vinca_conf @@ -864,6 +869,7 @@ def main(): base_dir = os.path.abspath(arguments.dir) vinca_yaml = os.path.join(base_dir, "vinca.yaml") vinca_conf = read_vinca_yaml(vinca_yaml) + snapshot = read_snapshot(arguments.snapshot) from .template import generate_bld_ament_cmake @@ -946,19 +952,19 @@ def main(): # only URLs if "://" in fn: selected_bn = vinca_conf.get("build_number", 0) - distro = vinca_conf["ros_distro"] - all_pkgs = repodata.get("packages", {}) - all_pkgs.update(repodata.get("packages.conda", {})) - for pkg_name, pkg in all_pkgs.items(): - if pkg_name.startswith(f"ros-{distro}"): - if pkg_name.rsplit("-", 2)[0] in additional_recipe_names: - print( - f"Skipping additional recipe for build number computation {pkg_name}" - ) - continue - selected_bn = max(selected_bn, pkg["build_number"]) - - print(f"Selected build number: {selected_bn}") + if not vinca_conf.get("use_explicit_build_number", False): + distro = vinca_conf["ros_distro"] + all_pkgs = repodata.get("packages", {}) + all_pkgs.update(repodata.get("packages.conda", {})) + for pkg_name, pkg in all_pkgs.items(): + if pkg_name.startswith(f"ros-{distro}"): + if pkg_name.rsplit("-", 2)[0] in additional_recipe_names: + print( + f"Skipping additional recipe for build number computation {pkg_name}" + ) + continue + selected_bn = max(selected_bn, pkg["build_number"]) + explicitly_selected_pkgs = [ f"ros-{distro}-{pkg.replace('_', '-')}" @@ -969,14 +975,15 @@ def main(): for _, pkg in all_pkgs.items(): is_built = False if selected_bn is not None: + pkg_build_number = get_pkg_build_number(selected_bn, pkg["name"], vinca_conf) if vinca_conf.get("full_rebuild", True): - if pkg["build_number"] == selected_bn: + if pkg["build_number"] == pkg_build_number: is_built = True else: # remove all packages except explicitly selected ones if ( pkg["name"] not in explicitly_selected_pkgs - or pkg["build_number"] == selected_bn + or pkg["build_number"] == pkg_build_number ): is_built = True else: diff --git a/vinca/template.py b/vinca/template.py index 5b8af6e..17b4c34 100644 --- a/vinca/template.py +++ b/vinca/template.py @@ -7,6 +7,8 @@ from ruamel import yaml from pathlib import Path +from vinca.utils import get_pkg_build_number + TEMPLATE = """\ # yaml-language-server: $schema=https://raw.githubusercontent.com/prefix-dev/recipe-format/main/schema.json @@ -71,7 +73,6 @@ def copyfile_with_exec_permissions(source_file, destination_file): os.chmod(destination_file, current_permissions | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) def write_recipe(source, outputs, vinca_conf, single_file=True): - build_number = vinca_conf.get("build_number", 0) # single_file = False if single_file: file = yaml.YAML() @@ -84,7 +85,7 @@ def write_recipe(source, outputs, vinca_conf, single_file=True): meta["package"]["version"] = f"{datetime.datetime.now():%Y.%m.%d}" meta["recipe"] = meta["package"] del meta["package"] - meta["build"]["number"] = build_number + meta["build"]["number"] = vinca_conf.get("build_number", 0) meta["build"]["post_process"] = post_process_items with open("recipe.yaml", "w") as stream: file.dump(meta, stream) @@ -102,7 +103,7 @@ def write_recipe(source, outputs, vinca_conf, single_file=True): meta["package"]["name"] = o["package"]["name"] meta["package"]["version"] = o["package"]["version"] - meta["build"]["number"] = build_number + meta["build"]["number"] = get_pkg_build_number(vinca_conf.get("build_number", 0), o["package"]["name"], vinca_conf) meta["build"]["post_process"] = post_process_items if test := vinca_conf["_tests"].get(o["package"]["name"]): diff --git a/vinca/utils.py b/vinca/utils.py index 486ef88..8d9d58b 100644 --- a/vinca/utils.py +++ b/vinca/utils.py @@ -65,3 +65,20 @@ def get_repodata(url_or_path, platform=None): with open(fn, "w") as fcache: fcache.write(content.decode("utf-8")) return json.loads(content) + +def ensure_name_is_without_distro_prefix_and_with_underscores(name, vinca_conf): + """ + Ensure that the name is without distro prefix and with underscores + e.g. "ros-humble-pkg-name" -> "pkg_name" + """ + newname = name.replace("-", "_") + distro_prefix = "ros_" + vinca_conf.get("ros_distro") + "_" + if (newname.startswith(distro_prefix) ): + newname = newname.replace(distro_prefix, "") + + return newname + +def get_pkg_build_number(default_build_number, pkg_name, vinca_conf): + normalized_name = ensure_name_is_without_distro_prefix_and_with_underscores(pkg_name, vinca_conf) + pkg_additional_info = vinca_conf["_pkg_additional_info"].get(normalized_name, {}) + return pkg_additional_info.get("build_number", default_build_number) From 52c1b49fe808b72f712306b9c9aa478f2ca2fffe Mon Sep 17 00:00:00 2001 From: Silvio Traversaro Date: Tue, 14 Jan 2025 23:58:04 +0100 Subject: [PATCH 2/3] Set use_explicit_build_number default value to True --- vinca/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vinca/main.py b/vinca/main.py index fc218c7..dbf4419 100644 --- a/vinca/main.py +++ b/vinca/main.py @@ -952,7 +952,7 @@ def main(): # only URLs if "://" in fn: selected_bn = vinca_conf.get("build_number", 0) - if not vinca_conf.get("use_explicit_build_number", False): + if not vinca_conf.get("use_explicit_build_number", True): distro = vinca_conf["ros_distro"] all_pkgs = repodata.get("packages", {}) all_pkgs.update(repodata.get("packages.conda", {})) From debb33e13e2a591a3807f731f5e4b0010d6a16d2 Mon Sep 17 00:00:00 2001 From: Silvio Traversaro Date: Wed, 15 Jan 2025 13:00:10 +0100 Subject: [PATCH 3/3] Remove skip_all_deps and full_rebuild options See https://github.com/RoboStack/vinca/pull/67#issuecomment-2591274823 --- vinca/main.py | 33 ++++++++++++--------------------- vinca/migrate.py | 1 - 2 files changed, 12 insertions(+), 22 deletions(-) diff --git a/vinca/main.py b/vinca/main.py index dbf4419..82ecba9 100644 --- a/vinca/main.py +++ b/vinca/main.py @@ -676,17 +676,16 @@ def get_selected_packages(distro, vinca_conf): for i in vinca_conf["packages_select_by_deps"]: i = i.replace("-", "_") selected_packages = selected_packages.union([i]) - if "skip_all_deps" not in vinca_conf or not vinca_conf["skip_all_deps"]: - if i in skipped_packages: - continue - try: - pkgs = distro.get_depends(i, ignore_pkgs=skipped_packages) - except KeyError: - # handle (rare) package names that use "-" as separator - pkgs = distro.get_depends(i.replace("_", "-")) - selected_packages.remove(i) - selected_packages.add(i.replace("_", "-")) - selected_packages = selected_packages.union(pkgs) + if i in skipped_packages: + continue + try: + pkgs = distro.get_depends(i, ignore_pkgs=skipped_packages) + except KeyError: + # handle (rare) package names that use "-" as separator + pkgs = distro.get_depends(i.replace("_", "-")) + selected_packages.remove(i) + selected_packages.add(i.replace("_", "-")) + selected_packages = selected_packages.union(pkgs) result = sorted(list(selected_packages)) return result @@ -976,16 +975,8 @@ def main(): is_built = False if selected_bn is not None: pkg_build_number = get_pkg_build_number(selected_bn, pkg["name"], vinca_conf) - if vinca_conf.get("full_rebuild", True): - if pkg["build_number"] == pkg_build_number: - is_built = True - else: - # remove all packages except explicitly selected ones - if ( - pkg["name"] not in explicitly_selected_pkgs - or pkg["build_number"] == pkg_build_number - ): - is_built = True + if pkg["build_number"] == pkg_build_number: + is_built = True else: is_built = True diff --git a/vinca/migrate.py b/vinca/migrate.py index 69aff1b..f3a2063 100644 --- a/vinca/migrate.py +++ b/vinca/migrate.py @@ -113,7 +113,6 @@ def create_migration_instructions(arch, packages_to_migrate, trigger_branch): print("Final names: ", ros_names) vinca_conf["packages_select_by_deps"] = ros_names - vinca_conf["skip_all_deps"] = True vinca_conf["is_migration"] = True vinca_conf["skip_existing"] = []