diff --git a/.gitignore b/.gitignore index 16c86f18b..553b141f4 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,6 @@ env .rsettings bin user/ + +# Python install files +*.egg-info diff --git a/installer/parse_install_yaml.py b/installer/parse_install_yaml.py index 10e14dbb7..6c07b9c51 100755 --- a/installer/parse_install_yaml.py +++ b/installer/parse_install_yaml.py @@ -1,6 +1,7 @@ #! /usr/bin/env python3 -from typing import List, Mapping, Optional +from typing import Any, List, Mapping, Optional +from functools import partial from os import environ import sys import yaml @@ -254,5 +255,143 @@ def get_distro_item(item: Mapping, key: str, release_version: str, release_type: return {"system_packages": system_packages, "commands": commands} +def installyaml_parser2(installer: Any, path: str, now: bool = False) -> Mapping: + with open(path) as f: + try: + install_items = yaml.load(f, yaml.CSafeLoader) + except AttributeError: + install_items = yaml.load(f, yaml.SafeLoader) + except (yaml.parser.ParserError, yaml.scanner.ScannerError) as e: + raise ValueError(f"Invalid yaml syntax: {e}") + + if not isinstance(install_items, list): + raise ValueError("Root of install.yaml file should be a YAML sequence") + + system_packages = [] + + commands = [] + + def get_distro_item(item: Mapping, key: str, release_version: str, release_type: str) -> Optional[str]: + if key in item: + value = item[key] + if value is None: + raise ValueError(f"'{key}' is defined, but has no value") + elif len(item) < 3 or "default" not in item: + raise ValueError(f"At least one distro and 'default' should be specified or none in install.yaml") + else: + for version in [release_version, "default"]: + if version in item: + value = item[version][key] + break + + return value + + # Combine now calls + now_cache = { + "system-now": [], + "pip-now": [], + "pip3-now": [], + "ppa-now": [], + "snap-now": [], + "gem-now": [], + } + + for install_item in install_items: + command = None + + try: + install_type = install_item["type"] # type: str + + if install_type == "empty": + return {"system_packages": system_packages, "commands": commands} + + elif install_type == "ros": + try: + source = get_distro_item(install_item, "source", ros_release, "ROS") + except ValueError as e: + raise ValueError(f"[{install_type}]: {e.args[0]}") + + # Both release and default are allowed to be None + if source is None: + continue + + source_type = source["type"] + if source_type == "git": + command = partial(getattr(installer, "tue_install_ros", "git", catkin_git(source))) + elif source_type == "system": + system_packages.append(f"ros-{ros_release}-{source['name']}") + command = partial(getattr(installer, "tue_install_ros", "system", catkin_git(source))) + else: + raise ValueError(f"Unknown ROS install type: '{source_type}'") + + # Non ros targets that are packaged to be built with catkin + elif install_type == "catkin": + source = install_item["source"] + + if not source["type"] == "git": + raise ValueError(f"Unknown catkin install type: '{source['type']}'") + command = partial(getattr(installer, "tue_install_ros", "git", catkin_git(source))) + + elif install_type == "git": + command = partial(getattr(installer, "tue_install_git", "git", type_git(install_item))) + + elif install_type in [ + "target", + "system", + "pip", + "pip3", + "ppa", + "snap", + "gem", + "dpkg", + "target-now", + "system-now", + "pip-now", + "pip3-now", + "ppa-now", + "snap-now", + "gem-now", + ]: + install_type = install_type.replace("-", "_") + if now and "now" not in install_type: + install_type += "_now" + + try: + pkg_name = get_distro_item(install_item, "name", ubuntu_release, "Ubuntu") + except ValueError as e: + raise ValueError(f"[{install_type}]: {e.args[0]}") + + # Both release and default are allowed to be None + if pkg_name is None: + continue + + if "system" in install_type: + system_packages.append(pkg_name) + + # Cache install types which accept multiple pkgs at once + if install_type in now_cache: + now_cache[install_type].append(pkg_name) + continue + command = partial(getattr(installer, f"tue_install_{install_type}", pkg_name)) + + else: + raise ValueError(f"Unknown install type: '{install_type}'") + + except KeyError as e: + raise KeyError(f"Invalid install.yaml file: Key '{e}' could not be found.") + + if not command: + raise ValueError("Invalid install.yaml file") + + commands.append(command) + + for install_type, pkg_list in now_cache.items(): + if pkg_list: + command = partial(getattr(installer, f"tue_install_{install_type}", pkg_list)) + commands.append(command) + + return {"system_packages": system_packages, "commands": commands} + + if __name__ == "__main__": sys.exit(main()) diff --git a/installer/tue-get-dep.bash b/installer/tue-get-dep.bash index aa00f521a..583fe4841 100755 --- a/installer/tue-get-dep.bash +++ b/installer/tue-get-dep.bash @@ -8,7 +8,7 @@ hash xmlstarlet 2> /dev/null || sudo apt-get install --assume-yes -qq xmlstarlet function _show_dep { - [[ "$1" == "ros" ]] && return 0 + [[ "$1" == "ros" ]] || [[ "$1" == "ros1" ]] || [[ "$1" == "ros2" ]] && return 0 [[ "$ROS_ONLY" = "true" && "$1" != ros-* ]] && return 0 diff --git a/setup.py b/setup.py new file mode 100644 index 000000000..3cb955da4 --- /dev/null +++ b/setup.py @@ -0,0 +1,43 @@ +import os.path + +from setuptools import find_packages +from setuptools import setup + +with open(os.path.join(os.path.dirname(__file__), "VERSION"), "r") as f: + version = f.readline().strip() + +print(find_packages(where="src")) + +setup( + name="tue_env", + packages=find_packages(where="src"), + package_dir={"": "src"}, + package_data={"tue_get": ["src/tue_get/resources/*"]}, + include_package_data=True, + version=version, + author="Matthijs van der Burgh", + author_email="MatthijsBurgh@outlook.com", + maintainer="Matthijs van der Burgh", + maintainer_email="MatthijsBurgh@outlook.com", + url="https://github.com/tue-robotics/tue-env", + keywords=["catkin"], + classifiers=[ + "Environment :: Console", + "Intended Audience :: Developers", + "Programming Language :: Python", + ], + description="Plugin for catkin_tools to enable building workspace documentation.", + # entry_points={ + # "catkin_tools.commands.catkin.verbs": [ + # "document = catkin_tools_document:description", + # ], + # "catkin_tools.spaces": [ + # "docs = catkin_tools_document.spaces.docs:description", + # ], + # }, + python_version=">=3.8", + install_requires=[ + "pyyaml", + "termcolor", + ], +) diff --git a/src/tue_get/__init__.py b/src/tue_get/__init__.py new file mode 100644 index 000000000..1998250a4 --- /dev/null +++ b/src/tue_get/__init__.py @@ -0,0 +1,4 @@ +from . import catkin_package_parser +from . import install_yaml_parser +from . import installer_impl +from . import util diff --git a/src/tue_get/catkin_package_parser.py b/src/tue_get/catkin_package_parser.py new file mode 100644 index 000000000..2114ead34 --- /dev/null +++ b/src/tue_get/catkin_package_parser.py @@ -0,0 +1,47 @@ +from typing import List, Mapping, Optional, Tuple + +from catkin_pkg.package import Dependency, Package, parse_package +import os + + +def catkin_package_parser( + path: str, + skip_normal_deps: bool = False, + test_deps: bool = False, + doc_deps: bool = False, + warnings: Optional[List[str]] = None, + context: Optional[Mapping] = None, +) -> Tuple[Package, List[Dependency]]: + if context is None: + context = os.environ + dep_types = [] + if not skip_normal_deps: + dep_types.extend( + [ + "build_depends", + "buildtool_depends", + "build_export_depends", + "buildtool_export_depends", + "exec_depends", + ] + ) + + if test_deps: + dep_types.append("test_depends") + + if doc_deps: + dep_types.append("doc_depends") + + pkg = parse_package(path, warnings=warnings) + pkg.evaluate_conditions(context) + + deps = [] + for dep_type in dep_types: + dep_list = getattr(pkg, dep_type) + for dep in dep_list: # type: Dependency + if dep.evaluated_condition is None: + raise RuntimeError("Dependency condition is None") + if dep.evaluated_condition: + deps.append(dep) + + return pkg, deps diff --git a/src/tue_get/install_yaml_parser.py b/src/tue_get/install_yaml_parser.py new file mode 100644 index 000000000..d2fa4ec26 --- /dev/null +++ b/src/tue_get/install_yaml_parser.py @@ -0,0 +1,223 @@ +from typing import Any, List, Mapping, Optional + +from os import environ +from functools import partial +import yaml + +from lsb_release import get_distro_information + +ros_release = environ["TUE_ROS_DISTRO"] +ubuntu_release = get_distro_information()["CODENAME"] + + +def type_git(install_item: Mapping, allowed_keys: Optional[List[str]] = None) -> Mapping: + """ + Function to check the parsed yaml for install type git and generate the command string. + + The structure of a git install type is: + - type: git + url: + path: [LOCAL_CLONE_PATH] + version: [BRANCH_COMMIT_TAG] + + :param install_item: Extracted yaml component corresponding to install type git + :param allowed_keys: Additional keys to allow apart from the keys defined in install type git + :return: Mapping string containing repository url and optional keyword arguments target-dir and version + """ + + assert install_item["type"] == "git", f"Invalid install type '{install_item['type']}' for type 'git'" + + keys = {"type", "url", "path", "version"} + if allowed_keys: + keys |= set(allowed_keys) + + invalid_keys = [k for k in install_item.keys() if k not in keys] + + if invalid_keys: + raise KeyError(f"The following keys are invalid for install type 'git': {invalid_keys}") + + url = install_item.get("url") + target_dir = install_item.get("path") + version = install_item.get("version") + + if not url: + raise KeyError("'url' is a mandatory key for install type 'git'") + + return {"url": url, "target_dir": target_dir, "version": version} + + +def catkin_git(source: Mapping) -> Mapping: + """ + Function to generate installation command for catkin git targets from the extracted yaml + + The structure of a catkin git install type is: + - type: catkin/ros + source: + type: git + url: + path: [LOCAL_CLONE_PATH] + version: [BRANCH_COMMIT_TAG] + sub-dir: [PACKAGE_SUB_DIRECTORY] + + :param source: Extracted yaml component for the key 'source' corresponding to install type catkin/ros + :return: Mapping containing the keyword arguments to the primary catkin target installation command + """ + + args = type_git(source, allowed_keys=["sub-dir"]) + args["source_type"] = "git" + + args["sub_dir"] = source.get("sub-dir") + + return args + + +def installyaml_parser(installer: Any, path: str, now: bool = False) -> Mapping: + with open(path) as f: + try: + install_items = yaml.load(f, yaml.CSafeLoader) + except AttributeError: + install_items = yaml.load(f, yaml.SafeLoader) + except (yaml.parser.ParserError, yaml.scanner.ScannerError) as e: + raise ValueError(f"Invalid yaml syntax: {e}") + + if not isinstance(install_items, list): + raise ValueError("Root of install.yaml file should be a YAML sequence") + + system_packages = [] + commands = [] + + # Combine now calls + now_cache = { + "system_now": [], + "pip_now": [], + "pip3_now": [], + "ppa_now": [], + "snap_now": [], + "gem_now": [], + } + + def get_distro_item(item: Mapping, key: str, release_version: str, release_type: str) -> Optional[str]: + if key in item: + value = item[key] + if value is None: + raise ValueError(f"'{key}' is defined, but has no value") + elif len(item) < 3 or "default" not in item: + raise ValueError(f"At least one distro and 'default' should be specified or none in install.yaml") + else: + for version in [release_version, "default"]: + if version in item: + value = item[version][key] + break + + return value + + for install_item in install_items: + command = None + + try: + install_type = install_item["type"] # type: str + install_type = install_type.replace("-", "_") # Need after switching to python impl of tue-get + + if install_type == "empty": + return {"system_packages": system_packages, "commands": commands} + + elif install_type == "ros": + try: + source = get_distro_item(install_item, "source", ros_release, "ROS") + except ValueError as e: + raise ValueError(f"[{install_type}]: {e.args[0]}") + + # Both release and default are allowed to be None + if source is None: + continue + + source_type = source["type"] + if source_type == "git": + command = partial(getattr(installer, "tue_install_ros"), **catkin_git(source)) + elif source_type == "system": + system_packages.append(f"ros-{ros_release}-{source['name']}") + command = partial(getattr(installer, "tue_install_ros"), source_type="system", name=source["name"]) + else: + raise ValueError(f"Unknown ROS install type: '{source_type}'") + + # Non ros targets that are packaged to be built with catkin + elif install_type == "catkin": + source = install_item["source"] + + if not source["type"] == "git": + raise ValueError(f"Unknown catkin install type: '{source['type']}'") + command = partial(getattr(installer, "tue_install_ros"), source_type="git", **catkin_git(source)) + + elif install_type == "git": + command = partial(getattr(installer, "tue_install_git"), source_type="git", **type_git(install_item)) + + elif install_type in [ + "target", + "system", + "pip", + "pip3", + "ppa", + "snap", + "gem", + "dpkg", + "target_now", + "system_now", + "pip_now", + "pip3_now", + "ppa_now", + "snap_now", + "gem_now", + ]: + if now and "now" not in install_type: + install_type += "_now" + + try: + pkg_name = get_distro_item(install_item, "name", ubuntu_release, "Ubuntu") + except ValueError as e: + raise ValueError(f"[{install_type}]: {e.args[0]}") + + # Both release and default are allowed to be None + if pkg_name is None: + continue + + if "system" in install_type: + system_packages.append(pkg_name) + + # Cache install types which accept multiple pkgs at once + if install_type in now_cache: + now_cache[install_type].append(pkg_name) + continue + command = partial( + getattr(installer, f"tue_install_{install_type}"), + pkg_name if "target" in install_type else [pkg_name], + ) + + else: + raise ValueError(f"Unknown install type: '{install_type}'") + + except KeyError as e: + raise KeyError(f"Invalid install.yaml file: Key '{e}' could not be found.") + + if not command: + raise ValueError("Invalid install.yaml file") + + commands.append(command) + + for install_type, pkg_list in now_cache.items(): + if pkg_list: + command = partial(getattr(installer, f"tue_install_{install_type}"), pkg_list) + commands.append(command) + + return {"system_packages": system_packages, "commands": commands} + + +if __name__ == "__main__": + import sys + from tue_get.installer_impl import InstallerImpl + + if len(sys.argv) < 2: + print("Provide yaml file to parse") + exit(1) + + impl = InstallerImpl(True) + print(installyaml_parser(impl, sys.argv[1], False)) diff --git a/src/tue_get/installer_impl.py b/src/tue_get/installer_impl.py new file mode 100644 index 000000000..eda9883f9 --- /dev/null +++ b/src/tue_get/installer_impl.py @@ -0,0 +1,1578 @@ +from typing import List, Optional, Tuple + +from catkin_pkg.package import PACKAGE_MANIFEST_FILENAME, InvalidPackage +from contextlib import contextmanager +import datetime +import filecmp +import getpass +import glob +import os +from pathlib import Path +from pip import main as pip_main +from pip._internal.req.constructors import install_req_from_line as pip_install_req_from_line +import re +import shlex +import shutil +import subprocess as sp +from termcolor import colored +from time import sleep + +from tue_get.catkin_package_parser import catkin_package_parser +from tue_get.install_yaml_parser import installyaml_parser +from tue_get.util.background_popen import BackgroundPopen +from tue_get.util.grep import grep_directory, grep_file + +CI = None + + +def is_CI() -> bool: + global CI + if CI is None: + CI = os.environ.get("CI", False) + + return CI + + +def date_stamp() -> str: + return datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S") + + +def _which_split_cmd(cmd: str) -> Tuple[str, List[str]]: + cmds = shlex.split(cmd) + cmds[0] = shutil.which(cmds[0]) + return " ".join(cmds), cmds + + +def _wait_for_dpkg_lock(): + i = 0 + rotate_list = ["-", "\\", "|", "/"] + cmd = "sudo fuser /var/lib/dpkg/lock" + cmd, cmds = _which_split_cmd(cmd) + while sp.run(cmds, stdout=sp.DEVNULL, stderr=sp.DEVNULL).returncode == 0: + print(f"[{rotate_list[i % len(rotate_list)]}] Waiting for dpkg lock", end="\r") + i += 1 + sleep(0.4) + return + + +class InstallerImpl: + _apt_get_updated_file = os.path.join(os.sep, "tmp", "tue_get_apt_get_updated") + _sources_list = os.path.join(os.sep, "etc", "apt", "sources.list") + _sources_list_dir = os.path.join(os.sep, "etc", "apt", "sources.list.d") + + def __init__( + self, + branch: Optional[str] = None, + ros_test_deps: bool = False, + ros_doc_deps: bool = False, + skip_ros_deps: bool = False, + debug: bool = False, + ): + self._branch = branch + self._ros_test_deps = ros_test_deps + self._ros_doc_deps = ros_doc_deps + self._skip_ros_deps = skip_ros_deps + self._debug = debug + self._current_target = "main-loop" + self._current_target_dir = "" + stamp = date_stamp() + self._log_file = os.path.join(os.sep, "tmp", f"tue-get-details-{stamp}") + Path(self._log_file).touch(exist_ok=False) + + try: + self._tue_env_dir = os.environ["TUE_ENV_DIR"] + except KeyError: + print("'TUE_ENV_DIR' is not defined") + raise + + self._dependencies_dir = os.path.join(self._tue_env_dir, ".env", "dependencies") + self._dependencies_on_dir = os.path.join(self._tue_env_dir, ".env", "dependencies-on") + self._installed_dir = os.path.join(self._tue_env_dir, ".env", "installed") + self._version_cache_dir = os.path.join(self._tue_env_dir, ".env", "version_cache") + + os.makedirs(self._dependencies_dir, exist_ok=True) + os.makedirs(self._dependencies_on_dir, exist_ok=True) + os.makedirs(self._installed_dir, exist_ok=True) + + try: + self._targets_dir = os.environ["TUE_ENV_TARGETS_DIR"] + except KeyError: + print("'TUE_ENV_TARGETS_DIR' is not defined") + raise + + if not os.path.isdir(self._targets_dir): + raise RuntimeError(f"TUE_INSTALL_TARGETS_DIR '{self._targets_dir}' does not exist as directory") + + general_state_dir = os.path.join(os.sep, "tmp", "tue-installer") + if not os.path.isdir(general_state_dir): + self.tue_install_debug(f"mkdir {general_state_dir}") + os.makedirs(general_state_dir, mode=0o700, exist_ok=True) + # self.tue_install_debug(f"chmod a+rwx {general_state_dir}") + # os.chmod(general_state_dir, 0o700) + + self._state_dir = os.path.join(general_state_dir, stamp) + self.tue_install_debug(f"mkdir {self._state_dir}") + os.makedirs(self._state_dir, mode=0o700) + + self._warn_logs = [] + self._info_logs = [] + + # ToDo: we might want to use sets here + self._systems = [] + self._ppas = [] + self._pips = [] + self._snaps = [] + self._gems = [] + + self._git_pull_queue = set() + + self._sudo_password = None + + def __del__(self): + shutil.rmtree(self._state_dir) + + def _get_installed_targets(self) -> List[str]: + return [ + pkg for pkg in os.listdir(self._installed_dir) if os.path.isfile(os.path.join(self._installed_dir, pkg)) + ] + + def _target_exist(self, target: str) -> bool: + return os.path.isdir(os.path.join(self._targets_dir, target)) + + def _log_to_file(self, msg: str, end="\n") -> None: + with open(self._log_file, "a") as f: + f.write(msg + end) + + def _write_sorted_deps(self, child: str) -> None: + self._write_sorted_dep_file(os.path.join(self._dependencies_dir, self._current_target), child) + self._write_sorted_dep_file(os.path.join(self._dependencies_on_dir, child), self._current_target) + + @staticmethod + def _write_sorted_dep_file(file: str, target: str) -> None: + if not os.path.isfile(file): + Path(file).touch(exist_ok=False) + targets = set() + else: + with open(file, "r") as f: + targets = set(f.read().splitlines()) + + targets.add(target) + with open(file, "w") as f: + for target in sorted(targets): + f.write(f"{target}\n") + + @contextmanager + def _set_target(self, target: str): + parent_target = self._current_target + parent_target_dir = self._current_target_dir + self._current_target = target + self._current_target_dir = os.path.join(self._targets_dir, target) + yield + self._current_target = parent_target + self._current_target_dir = parent_target_dir + + def _out_handler(self, sub: BackgroundPopen, line: str) -> None: + def _write_stdin(msg) -> None: + if sub.returncode is not None: + self.tue_install_error( + f"Cannot write to stdin of process {sub.pid} as it has already terminated {line=}" + ) + return + sub.stdin.write(f"{msg}\n") + sub.stdin.flush() + + def _clean_output(msg: str) -> str: + return msg.replace("^", "\n").strip() + + line = line.strip() + # ToDo: use regex to get attributes of installer + if line.startswith("[sudo] password for"): + if self._sudo_password is None: + self._sudo_password = getpass.getpass(line) + _write_stdin(self._sudo_password) + elif line.startswith("tue-install-error: "): + self.tue_install_error(_clean_output(line[19:])) + _write_stdin(1) + # ToDo: or should we call sub.kill() here? + elif line.startswith("tue-install-warning: "): + self.tue_install_warning(_clean_output(line[21:])) + _write_stdin(0) + elif line.startswith("tue-install-info: "): + self.tue_install_info(_clean_output(line[18:])) + _write_stdin(0) + elif line.startswith("tue-install-debug: "): + self.tue_install_debug(_clean_output(line[19:])) + _write_stdin(0) + elif line.startswith("tue-install-echo: "): + self.tue_install_echo(_clean_output(line[18:])) + _write_stdin(0) + elif line.startswith("tue-install-tee: "): + self.tue_install_tee(_clean_output(line[17:])) + _write_stdin(0) + elif line.startswith("tue-install-pipe: "): + output = line[18:].split("^^^") + output = map(_clean_output, output) + self.tue_install_pipe(*output) + _write_stdin(0) + elif line.startswith("tue-install-target-now: "): + self.tue_install_target_now(line[24:]) + _write_stdin(0) + elif line.startswith("tue-install-target: "): + success = self.tue_install_target(line[20:]) + _write_stdin(not success) + elif line.startswith("tue-install-git: "): + success = self.tue_install_git(*line[17:].split()) + _write_stdin(not success) + elif line.startswith("tue-install-apply-patch: "): + success = self.tue_install_apply_patch(*line[25:].split()) + _write_stdin(not success) + elif line.startswith("tue-install-cp: "): + success = self.tue_install_cp(*line[15:].split()) + _write_stdin(not success) + elif line.startswith("tue-install-add-text: "): + success = self.tue_install_add_text(*line[21:].split()) + _write_stdin(not success) + elif line.startswith("tue-install-get-releases: "): + success = self.tue_install_get_releases(*line[26:].split()) + _write_stdin(not success) + elif line.startswith("tue-install-system: "): + pkgs = line[20:].split() + success = self.tue_install_system(pkgs) + _write_stdin(not success) + elif line.startswith("tue-install-system-now: "): + pkgs = line[24:].split() + success = self.tue_install_system_now(pkgs) + _write_stdin(not success) + elif line.startswith("tue-install-apt-get-update"): + self.tue_install_apt_get_update() + _write_stdin(0) + elif line.startswith("tue-install-ppa: "): + ppas = line[17:].split() + ppas = [ppa.replace("^", " ").strip() for ppa in ppas] + success = self.tue_install_ppa(ppas) + _write_stdin(not success) + elif line.startswith("tue-install-ppa-now: "): + ppas = line[21:].split() + ppas = [ppa.replace("^", " ").strip() for ppa in ppas] + success = self.tue_install_ppa_now(ppas) + _write_stdin(not success) + elif line.startswith("tue-install-pip: "): + pkgs = line[17:].split() + success = self.tue_install_pip(pkgs) + _write_stdin(not success) + elif line.startswith("tue-install-pip-now: "): + pkgs = line[20:].split() + success = self.tue_install_pip_now(pkgs) + _write_stdin(not success) + elif line.startswith("tue-install-snap: "): + pkgs = line[18:].split() + success = self.tue_install_snap(pkgs) + _write_stdin(not success) + elif line.startswith("tue-install-snap-now: "): + pkgs = line[22:].split() + success = self.tue_install_snap_now(pkgs) + _write_stdin(not success) + elif line.startswith("tue-install-gem: "): + pkgs = line[17:].split(" ") + success = self.tue_install_gem(pkgs) + _write_stdin(not success) + elif line.startswith("tue-install-gem-now: "): + pkgs = line[21:].split(" ") + success = self.tue_install_gem_now(pkgs) + _write_stdin(not success) + else: + self.tue_install_tee(line) + + def _err_handler(self, sub: BackgroundPopen, line: str) -> None: + line = line.strip() + if line.startswith("[sudo] password for"): + if self._sudo_password is None: + self._sudo_password = getpass.getpass(line) + sub.stdin.write(f"{self._sudo_password}\n") + sub.stdin.flush() + else: + self.tue_install_tee(line, color="red") + + def _default_background_popen(self, cmd: str) -> BackgroundPopen: + cmd, cmds = _which_split_cmd(cmd) + self.tue_install_debug(repr(cmd)) + sub = BackgroundPopen( + args=cmds, out_handler=self._out_handler, err_handler=self._err_handler, stdin=sp.PIPE, text=True + ) + sub.wait() + return sub + + def tue_install_error(self, msg: str) -> None: + # Make sure the entire msg is indented, not just the first line + lines = msg.splitlines() + lines = [f" {line}" for line in lines] + msg = "\n".join(lines) + log_msg = f"Error while installing target '{self._current_target}':\n\n{msg}\n\nLogfile: {self._log_file}" + print(colored(log_msg, color="red")) + self._log_to_file(log_msg) + # ToDo: We should stop after this + + def tue_install_warning(self, msg: str) -> None: + log_msg = f"[{self._current_target}] WARNING: {msg}" + colored_log = colored(log_msg, color="yellow", attrs=["bold"]) + self._warn_logs.append(colored_log) + print(colored_log) + self._log_to_file(log_msg) + + def tue_install_info(self, msg: str) -> None: + log_msg = f"[{self._current_target}] INFO: {msg}" + colored_log = colored(log_msg, color="cyan") + self._info_logs.append(colored_log) + print(colored_log) + self._log_to_file(log_msg) + + def tue_install_debug(self, msg: str) -> None: + log_msg = f"[{self._current_target}] DEBUG: {msg}" + colored_log = colored(log_msg, color="blue") + if self._debug: + print(colored_log) + self._log_to_file(log_msg) + + def tue_install_echo(self, msg: str) -> None: + log_msg = f"[{self._current_target}]: {msg}" + colored_log = colored(log_msg, attrs=["bold"]) + print(colored_log) + self._log_to_file(log_msg) + + def tue_install_tee(self, msg: str, color=None, on_color=None, attrs=None) -> None: + print(colored(msg, color=color, on_color=on_color, attrs=attrs)) + self._log_to_file(msg) + + def tue_install_pipe(self, stdout: str, stderr: str) -> None: + """ + Only to be used for bash scripts + Prints stdout, stderr is printed in red + + :param stdout: + :param stderr: + :return: + """ + if stdout: + self.tue_install_tee(stdout) + if stderr: + self.tue_install_tee(stderr, color="red") + + def tue_install_target_now(self, target: str) -> bool: + self.tue_install_debug(f"tue-install-target-now {target=}") + + self.tue_install_debug(f"calling: tue-install-target {target} True") + return self.tue_install_target(target, True) + + def tue_install_target(self, target: str, now: bool = False) -> bool: + self.tue_install_debug(f"tue-install-target {target=} {now=}") + + self.tue_install_debug(f"Installing target: {target}") + + # Check if valid target received as input + if not self._target_exist(target): + self.tue_install_debug(f"Target '{target}' does not exist.") + return False + + # If the target has a parent target, add target as a dependency to the parent target + if self._current_target and self._current_target != "main-loop" and self._current_target != target: + self._write_sorted_deps(target) + + with self._set_target(target): + state_file = os.path.join(self._state_dir, target) + state_file_now = f"{state_file}-now" + + # Determine if this target needs to be executed + execution_needed = True + + if is_CI() and not os.path.isfile(os.path.join(self._current_target_dir, ".ci_ignore")): + self.tue_install_debug( + f"Running installer in CI mode and file {self._current_target_dir}/.ci_ignore exists. " + "No execution is needed." + ) + execution_needed = False + elif os.path.isfile(state_file_now): + self.tue_install_debug( + f"File {state_file_now} does exist, so installation has already been executed with 'now' option. " + "No execution is needed." + ) + execution_needed = False + elif os.path.isfile(state_file): + if now: + self.tue_install_debug( + f"File {state_file_now} doesn't exist, but file {state_file} does. " + "So installation has been executed yet, but not with the 'now' option. " + "Going to execute it with 'now' option." + ) + else: + self.tue_install_debug( + f"File {state_file_now} does exist. 'now' is not enabled, so no execution needed." + ) + execution_needed = False + else: + if now: + self.tue_install_debug( + f"Files {state_file_now} and {state_file} don't exist. Going to execute with 'now' option." + ) + else: + self.tue_install_debug( + f"Files {state_file_now} and {state_file} don't exist. Going to execute without 'now' option." + ) + + if execution_needed: + self.tue_install_debug(f"Starting installation of target: {target}") + install_file = os.path.join(self._current_target_dir, "install") + + # Empty the target's dependency file + dep_file = os.path.join(self._dependencies_dir, target) + self.tue_install_debug(f"Emptying {dep_file}") + if os.path.isfile(dep_file): + with open(dep_file, "a") as f: + f.truncate(0) + + target_processed = False + + install_yaml_file = install_file + ".yaml" + if os.path.isfile(install_yaml_file): + if is_CI() and os.path.isfile(os.path.join(self._current_target_dir, ".ci_ignore_yaml")): + self.tue_install_debug( + "Running in CI mode and found .ci_ignore_yaml file, so skipping install.yaml" + ) + target_processed = True + else: + self.tue_install_debug(f"Parsing {install_yaml_file}") + + cmds = installyaml_parser(self, install_yaml_file, now)["commands"] + if not cmds: + self.tue_install_error(f"Invalid install.yaml: {cmds}") + # ToDo: This depends on behaviour of tue-install-error + return False + + for cmd in cmds: + self.tue_install_debug(str(cmd)) + cmd() or self.tue_install_error( + f"Error while running:\n {cmd}" + ) # ToDo: Fix exception/return value handling + + target_processed = True + + install_bash_file = install_file + ".bash" + if os.path.isfile(install_bash_file): + if is_CI() and os.path.isfile(os.path.join(self._current_target_dir, ".ci_ignore_bash")): + self.tue_install_debug( + "Running in CI mode and found .ci_ignore_bash file, so skipping install.bash" + ) + else: + self.tue_install_debug(f"Sourcing {install_bash_file}") + resource_file = os.path.join(os.path.dirname(__file__), "resources", "installer_impl.bash") + cmd = f'bash -c "source {resource_file} && source {install_bash_file}"' + sub = self._default_background_popen(cmd) + if sub.returncode != 0: + self.tue_install_error(f"Error while running({sub.returncode}):\n {repr(cmd)}") + # ToDo: This depends on behaviour of tue-install-error + return False + + target_processed = True + + if not target_processed: + self.tue_install_warning("Target does not contain a valid install.yaml/bash file") + + if now: + state_file_to_touch = state_file_now + else: + state_file_to_touch = state_file + + Path(state_file_to_touch).touch() + + self.tue_install_debug(f"Finished installing {target}") + return True + + def _show_update_msg(self, repo, msg: str = None): + if msg: + print_msg = "\n" + print_msg += f" {colored(repo, attrs=['bold'])}" + print_msg += f"\n--------------------------------------------------" + print_msg += f"\n{msg}" + print_msg += f"\n--------------------------------------------------" + print_msg += f"\n" + self.tue_install_tee(print_msg) + else: + self.tue_install_tee(f"{colored(repo, attrs=['bold'])}: up-tp-date") + + def _try_branch_git(self, target_dir, version): + self.tue_install_debug(f"_try_branch_git {target_dir=} {version=}") + + cmd = f"git -C {target_dir} checkout {version} --" + self.tue_install_debug(f"{cmd}") + cmd, cmds = _which_split_cmd(cmd) + try_branch_res = sp.check_output(cmds, stderr=sp.STDOUT, text=True).strip() + self.tue_install_debug(f"{try_branch_res=}") + + cmd = f"git -C {target_dir} submodule sync --recursive" + self.tue_install_debug(f"{cmd}") + cmd, cmds = _which_split_cmd(cmd) + submodule_sync_results = sp.run(cmds, stdout=sp.PIPE, stderr=sp.STDOUT, text=True) + submodule_sync_res = submodule_sync_results.stdout.strip() + self.tue_install_debug(f"{submodule_sync_res=}") + + cmd = f"git -C {target_dir} submodule update --init --recursive" + self.tue_install_debug(f"{cmd}") + cmd, cmds = _which_split_cmd(cmd) + submodule_res = sp.check_output(cmds, stderr=sp.STDOUT, text=True).strip() + self.tue_install_debug(f"{submodule_res=}") + + if "Already on " in try_branch_res or "fatal: invalid reference:" in try_branch_res: + try_branch_res = "" + + if submodule_sync_results.returncode != 0 and submodule_sync_res: + try_branch_res += f"\n{submodule_sync_res}" + if submodule_res: + try_branch_res += f"\n{submodule_res}" + + return try_branch_res + + def tue_install_git(self, url: str, target_dir: Optional[str] = None, version: Optional[str] = None) -> bool: + self.tue_install_debug(f"tue-install-git {url=} {target_dir=} {version=}") + + # ToDo: convert _git_https_or_ssh to python + cmd = f"bash -c '_git_https_or_ssh {url}'" + cmd, cmds = _which_split_cmd(cmd) + url_old = url + url = sp.check_output(cmds, text=True).strip() + if not url: + self.tue_install_error( + f"repo: '{url}' is invalid. It is generated from: '{url_old}'\n" + f"The problem will probably be solved by resourcing the setup" + ) + # ToDo: This depends on behaviour of tue-install-error + return False + + if target_dir is None: + # ToDo: convert _git_url_to_repos_dir to python + cmd = f"bash -c '_git_url_to_repos_dir {url}'" + cmd, cmds = _which_split_cmd(cmd) + target_dir = sp.check_output(cmds, text=True).strip() + if not target_dir: + self.tue_install_error(f"Could not create target_dir path from the git url: '{url}'") + # ToDo: This depends on behaviour of tue-install-error + return False + + if not target_dir: + self.tue_install_error(f"target_dir is specified, but empty") + # ToDo: This depends on behaviour of tue-install-error + return False + + if not os.path.isdir(target_dir): + cmd = f"git clone --recursive {url} {target_dir}" + self.tue_install_debug(f"{cmd}") + cmd, cmds = _which_split_cmd(cmd) + clone_results = sp.run(cmds, stdout=sp.PIPE, stderr=sp.STDOUT, text=True) + res = clone_results.stdout.strip() + if clone_results.returncode != 0: + self.tue_install_error(f"Failed to clone {url} to {target_dir}\n{res}") + # ToDo: This depends on behaviour of tue-install-error + return False + + self._git_pull_queue.add(target_dir) + else: + if target_dir in self._git_pull_queue: + self.tue_install_debug("Repo previously pulled, skipping") + res = "" + else: + cmd = f"git -C {target_dir} config --get remote.origin.url" + cmd, cmds = _which_split_cmd(cmd) + self.tue_install_debug(f"{cmd}") + cmd, cmds = _which_split_cmd(cmd) + current_url = sp.check_output(cmds, text=True).strip() + # If different, switch url + if current_url != url: + cmd = f"git -C {target_dir} remote set-url origin {url}" + sub = self._default_background_popen(cmd) + if sub.returncode != 0: + self.tue_install_error( + f"Could not change git url of '{target_dir}' to '{url}'" + f"({sub.returncode}):\n {repr(cmd)}" + ) + # ToDo: This depends on behaviour of tue-install-error + return False + + self.tue_install_info(f"url has switched to '{url}'") + + cmd = f"git -C {target_dir} pull --ff-only --prune" + self.tue_install_debug(f"{cmd}") + cmd, cmds = _which_split_cmd(cmd) + res = sp.check_output(cmds, stderr=sp.STDOUT, text=True).strip() + self.tue_install_debug(f"{res=}") + + self._git_pull_queue.add(target_dir) + + cmd = f"git -C {target_dir} submodule sync --recursive" + self.tue_install_debug(f"{cmd}") + cmd, cmds = _which_split_cmd(cmd) + sp_results = sp.run(cmds, text=True, stdout=sp.PIPE) + submodule_sync_res = sp_results.stdout.strip() + self.tue_install_debug(f"{submodule_sync_res=}") + if sp_results.returncode != 0 and submodule_sync_res: + res += f"\n{submodule_sync_res}" + + cmd = f"git -C {target_dir} submodule update --init --recursive" + self.tue_install_debug(f"{cmd}") + cmd, cmds = _which_split_cmd(cmd) + submodule_res = sp.check_output(cmds, stderr=sp.STDOUT, text=True).strip() + self.tue_install_debug(f"{submodule_res=}") + if submodule_res: + res += f"\n{submodule_res}" + + if "Already up to date." in res: + res = "" + + self.tue_install_debug(f"Desired version: {version}") + version_cache_file = os.path.join(self._version_cache_dir, target_dir.lstrip(os.sep)) + if version: + os.makedirs(os.path.dirname(version_cache_file), exist_ok=True) + with open(version_cache_file, "w") as f: + f.write(version) + + try_branch_res = self._try_branch_git(target_dir, version) + if try_branch_res: + res += f"\n{try_branch_res}" + else: + if os.path.isfile(version_cache_file): + os.remove(version_cache_file) + + if self._branch: + self.tue_install_debug(f"Desired branch: {self._branch}") + try_branch_res = self._try_branch_git(target_dir, self._branch) + if try_branch_res: + res += f"\n{try_branch_res}" + + self._show_update_msg(self._current_target, res) + return True + + def tue_install_apply_patch(self, patch_file: str, target_dir: str) -> bool: + self.tue_install_debug(f"tue-install-apply-patch {patch_file=} {target_dir=}") + + if not patch_file: + self.tue_install_error("Invalid tue-install-apply-patch call: needs patch file as argument") + # ToDo: This depends on behaviour of tue-install-error + return False + + patch_file_path = os.path.join(self._current_target_dir, patch_file) + if not os.path.isfile(patch_file_path): + self.tue_install_error(f"Invalid tue-install-apply-patch call: patch file {patch_file_path} does not exist") + # ToDo: This depends on behaviour of tue-install-error + return False + + if not target_dir: + self.tue_install_error("Invalid tue-install-apply-patch call: needs target directory as argument") + # ToDo: This depends on behaviour of tue-install-error + return False + + if not os.path.isdir(target_dir): + self.tue_install_error( + f"Invalid tue-install-apply-patch call: target directory {target_dir} does not exist" + ) + # ToDo: This depends on behaviour of tue-install-error + return False + + cmd = f"patch -s -N -r - -p0 -d {target_dir} < {patch_file_path}" + sub = self._default_background_popen(cmd) + if sub.returncode != 0: + self.tue_install_error(f"Error while running({sub.returncode}):\n {repr(cmd)}") + # ToDo: This depends on behaviour of tue-install-error + return False + + return True + + def tue_install_cp(self, source: str, target: str) -> bool: + self.tue_install_debug(f"tue-install-cp {source=} {target=}") + + if not source: + self.tue_install_error("Invalid tue-install-cp call: needs source directory as argument") + # ToDo: This depends on behaviour of tue-install-error + return False + + if not target: + self.tue_install_error("Invalid tue-install-cp call: needs target dir as argument") + # ToDo: This depends on behaviour of tue-install-error + return False + + if os.path.isdir(target): + self.tue_install_debug(f"tue-install-cp: target {target} is a directory") + target_dir = target + elif os.path.isfile(target): + self.tue_install_debug(f"tue-install-cp: target {target} is a file") + target_dir = os.path.dirname(target) + else: + self.tue_install_error(f"tue-install-cp: target {target} does not exist") + # ToDo: This depends on behaviour of tue-install-error + return False + + # Check if user is allowed to write on target destination + root_required = True + + if os.access(target, os.W_OK): + root_required = False + + source_path = os.path.join(self._current_target_dir, source) + source_files = glob.glob(source_path) + + for file in source_files: + if not os.path.isfile(file): + self.tue_install_error(f"tue-install-cp: source {file} is not a file") + # ToDo: This depends on behaviour of tue-install-error + return False + + if os.path.isdir(target): + cp_target = os.path.join(target_dir, os.path.basename(file)) + else: + cp_target = target + + if os.path.isfile(cp_target) and filecmp.cmp(file, cp_target): + self.tue_install_debug(f"tue-install-cp: {file} and {cp_target} are identical, skipping") + continue + + self.tue_install_debug(f"tue-install-cp: copying {file} to {cp_target}") + + if root_required: + sudo_cmd = "sudo " + self.tue_install_debug(f"Using elevated privileges (sudo) to copy ${file} to ${cp_target}") + else: + sudo_cmd = "" + + cmd = f"{sudo_cmd}python -c 'import os; os.makedirs(\"{target}\", exist_ok=True)'" + sub = self._default_background_popen(cmd) + if sub.returncode != 0: + self.tue_install_error(f"Error while creating the directory({sub.returncode}):\n {repr(cmd)}") + # ToDo: This depends on behaviour of tue-install-error + return False + + cmd = f'{sudo_cmd}python -c \'import shutil; shutil.copy2("{file}", "{cp_target}")\'' + sub = self._default_background_popen(cmd) + if sub.returncode != 0: + self.tue_install_error(f"Error while copying({sub.returncode}):\n {repr(cmd)}") + # ToDo: This depends on behaviour of tue-install-error + return False + + return True + + def tue_install_add_text(self, source_file: str, target_file: str) -> bool: + self.tue_install_debug(f"tue-install-add-text {source_file=} {target_file=}") + + if not source_file: + self.tue_install_error("Invalid tue-install-add-text call: needs source file as argument") + # ToDo: This depends on behaviour of tue-install-error + return False + + if not target_file: + self.tue_install_error("Invalid tue-install-add-text call: needs target file as argument") + # ToDo: This depends on behaviour of tue-install-error + return False + + if source_file.startswith("/") or source_file.startswith("~"): + self.tue_install_error( + "Invalid tue-install-add-text call: source file must be relative to target directory" + ) + # ToDo: This depends on behaviour of tue-install-error + return False + + if not target_file.startswith("/") and not target_file.startswith("~"): + self.tue_install_error( + "Invalid tue-install-add-text call: target file must be absolute or " "relative to the home directory" + ) + # ToDo: This depends on behaviour of tue-install-error + return False + + root_required = True + if os.access(target_file, os.W_OK): + root_required = False + self.tue_install_debug("tue-install-add-text: NO root required") + else: + self.tue_install_debug("tue-install-add-text: root required") + + if root_required: + sudo_cmd = "sudo " + self.tue_install_debug("Using elevated privileges (sudo) to add text") + else: + sudo_cmd = "" + + source_file_path = os.path.join(self._current_target_dir, source_file) + if not os.path.isfile(source_file_path): + self.tue_install_error(f"tue-install-add-text: source file {source_file_path} does not exist") + # ToDo: This depends on behaviour of tue-install-error + return False + + target_file_path = os.path.expanduser(target_file) + if not os.path.isfile(target_file_path): + self.tue_install_error(f"tue-install-add-text: target file {target_file_path} does not exist") + # ToDo: This depends on behaviour of tue-install-error + return False + + with open(source_file_path, "r") as f: + source_text = f.read().splitlines() + + begin_tag = source_text[0] + end_tag = source_text[-1] + source_body = source_text[1:-1] + + self.tue_install_debug(f"tue-install-add-text: {begin_tag=}, {end_tag=}\n{source_body=}") + + with open(target_file_path, "r") as f: + target_text = f.read().splitlines() + + if begin_tag not in target_text: + self.tue_install_debug( + f"tue-install-add-text: {begin_tag=} not found in {target_file_path=}, " + "appending to {target_file_path}" + ) + source_text = "\n".join(source_text) + cmd = f"bash -c \"echo - e '{source_text}' | {sudo_cmd}tee -a {target_file_path}\"" + else: + self.tue_install_debug( + f"tue-install-add-text: {begin_tag=} found in {target_file_path=}, " + "so comparing the files for changed lines" + ) + + begin_index = target_text.index(begin_tag) + end_index = target_text.index(end_tag) + target_body = target_text[begin_index + 1 : end_index] + self.tue_install_debug(f"tue-install-add-text:\n{target_body=}") + + if source_body == target_body: + self.tue_install_debug("tue-install-add-text: Lines have not changed, so not copying") + return True + + self.tue_install_debug("tue-install-add-text: Lines have changed, so copying") + + target_text = target_text[: begin_index + 1] + source_body + target_text[end_index:] + print(target_text) + target_text = "\n".join(target_text) + + cmd = f"bash -c \"echo -e '{target_text}' | {sudo_cmd}tee {target_file_path}\"" + + cmd, cmds = _which_split_cmd(cmd) + self.tue_install_debug(repr(cmd)) + sub = BackgroundPopen( + args=cmds, + err_handler=self._err_handler, + stdout=sp.DEVNULL, + stdin=sp.PIPE, + text=True, + ) + sub.wait() + if sub.returncode != 0: + self.tue_install_error(f"Error while adding text({sub.returncode}):\n {repr(cmd)}") + # ToDo: This depends on behaviour of tue-install-error + return False + + def tue_install_get_releases(self, url: str, filename: str, output_dir: str, tag: Optional[str] = None) -> bool: + self.tue_install_debug(f"tue-install-get-releases {url=} {filename=} {output_dir=} {tag=}") + return True + + def tue_install_system(self, pkgs: List[str]) -> bool: + self.tue_install_debug(f"tue-install-system {pkgs=}") + if not pkgs: + self.tue_install_error("Invalid tue-install-system call: needs packages as argument") + # ToDo: This depends on behaviour of tue-install-error + return False + + self._systems.extend(pkgs) + return True + + def tue_install_system_now(self, pkgs: List[str]) -> bool: + self.tue_install_debug(f"tue-install-system-now {pkgs=}") + if not pkgs: + self.tue_install_error("Invalid tue-install-system-now call: needs packages as argument") + # ToDo: This depends on behaviour of tue-install-error + return False + + installed_pkgs = [] + + def _out_handler_installed_pkgs(_: BackgroundPopen, line: str) -> None: + installed_pkgs.append(line.strip()) + + # Check if pkg is not already installed dpkg -S does not cover previously removed packages + # Based on https://stackoverflow.com/questions/1298066 + cmd = "dpkg-query -W -f '${package} ${status}\n'" + cmd, cmds = _which_split_cmd(cmd) + self.tue_install_debug(repr(cmd)) + sub = BackgroundPopen( + args=cmds, + out_handler=_out_handler_installed_pkgs, # Needed to prevent buffer to get full + err_handler=self._err_handler, + stdin=sp.PIPE, + text=True, + ) + sub.wait() + if sub.returncode != 0: + self.tue_install_error(f"Error while getting installed packages({sub.returncode}):\n {repr(cmd)}") + # ToDo: This depends on behaviour of tue-install-error + return False + + installed_pkgs = [pkg[:-21] for pkg in installed_pkgs if pkg[-20:] == "install ok installed"] + pkgs_to_install = [] + for pkg in pkgs: + if pkg not in installed_pkgs: + self.tue_install_debug(f"Package {pkg} is not installed") + pkgs_to_install.append(pkg) + else: + self.tue_install_debug(f"Package {pkg} is already installed") + + if not pkgs_to_install: + return True + + # Install packages + apt_get_cmd = f"sudo apt-get install --assume-yes -q {' '.join(pkgs_to_install)}" + self.tue_install_echo(f"Going to run the following command:\n{apt_get_cmd}") + + _wait_for_dpkg_lock() + + if not os.path.isfile(self._apt_get_updated_file): + cmd = "sudo apt-get update" + sub = self._default_background_popen(cmd) + if sub.returncode != 0: + self.tue_install_error(f"Error while updating apt-get({sub.returncode}):\n {repr(cmd)}") + # ToDo: This depends on behaviour of tue-install-error + return False + Path(self._apt_get_updated_file).touch() + + sub = self._default_background_popen(apt_get_cmd) + if sub.returncode != 0: + self.tue_install_error( + f"Error while installing system packages({sub.returncode}):" f"\n {repr(apt_get_cmd)}" + ) + # ToDo: This depends on behaviour of tue-install-error + return False + + self.tue_install_debug(f"Installed {pkgs} ({sub.returncode})") + + return True + + def tue_install_apt_get_update(self): + self.tue_install_debug("tue-install-apt-get-update") + self.tue_install_debug("Requiring an update of apt-get before next 'apt-get install'") + if os.path.isfile(self._apt_get_updated_file): + os.remove(self._apt_get_updated_file) + + def tue_install_ppa(self, ppas: List[str]) -> bool: + self.tue_install_debug(f"tue-install-ppa {ppas=}") + if not ppas: + self.tue_install_error("Invalid tue-install-ppa call: needs ppas as argument") + # ToDo: This depends on behaviour of tue-install-error + return False + + self._ppas.extend(ppas) + return True + + def tue_install_ppa_now(self, ppas: List[str]) -> bool: + self.tue_install_debug(f"tue-install-ppa-now {ppas=}") + + if not ppas: + self.tue_install_error("Invalid tue-install-ppa-now call: needs ppas as argument") + # ToDo: This depends on behaviour of tue-install-error + return False + + ppa_added = False + for ppa in ppas: + if not ppa.startswith("ppa:") and not ppa.startswith("deb "): + self.tue_install_error(f"Invalid ppa: needs to start with 'ppa:' or 'deb ' ({ppa=})") + # ToDo: This depends on behaviour of tue-install-error + return False + + if ppa.startswith("ppa:"): + ppa_pattern = f"^deb.*{ppa[4:]}".replace("[", "\[").replace("]", "\]").replace("/", "\/") + ppa_pattern = re.compile(ppa_pattern) + if grep_directory(self._sources_list_dir, ppa_pattern): + self.tue_install_debug(f"PPA '{ppa}' is already added previously") + continue + + elif ppa.startswith("deb "): + ppa_pattern = f"^{ppa}$".replace("[", "\[").replace("]", "\]").replace("/", "\/") + ppa_pattern = re.compile(ppa_pattern) + if grep_file(self._sources_list, ppa_pattern): + self.tue_install_debug(f"PPA '{ppa}' is already added previously") + continue + + self.tue_install_system_now(["software-properties-common"]) + + self.tue_install_info(f"Adding ppa: {ppa}") + + # Wait for apt-lock first (https://askubuntu.com/a/375031) + _wait_for_dpkg_lock() + + cmd = f"sudo add-apt-repository --no-update --yes '{ppa}'" + sub = self._default_background_popen(cmd) + if sub.returncode != 0: + self.tue_install_error(f"An error occurred while adding ppa: {ppa}") + # ToDo: This depends on behaviour of tue-install-error + return False + + ppa_added = True + + if ppa_added: + self.tue_install_apt_get_update() + + return True + + def tue_install_pip(self, pkgs: List[str]) -> bool: + self.tue_install_debug(f"tue-install-pip {pkgs=}") + if not pkgs: + self.tue_install_error("Invalid tue-install-pip call: needs packages as argument") + # ToDo: This depends on behaviour of tue-install-error + return False + + self._pips.extend(pkgs) + return True + + def tue_install_pip_now(self, pkgs: List[str]) -> bool: + self.tue_install_debug(f"tue-install-pip-now {pkgs=}") + + if not pkgs: + self.tue_install_error("Invalid tue-install-pip-now call: needs packages as argument") + # ToDo: This depends on behaviour of tue-install-error + return False + + pips_to_install = [] + git_pips_to_install = [] + for pkg in pkgs: + if pkg.startswith("git+"): + git_pips_to_install.append(pkg) + continue + + req = pip_install_req_from_line(pkg) + req.check_if_exists(use_user_site=True) + + if req.satisfied_by: + self.tue_install_debug(f"{pkg} is already installed, {req.satisfied_by}") + else: + pips_to_install.append(pkg) + + if pips_to_install: + returncode = pip_main(["install", "--user", *pips_to_install]) + if returncode != 0: + self.tue_install_error(f"An error occurred while installing pip packages: {pips_to_install}") + # ToDo: This depends on behaviour of tue-install-error + return False + + if git_pips_to_install: + for pkg in git_pips_to_install: + returncode = pip_main(["install", "--user", pkg]) + if returncode != 0: + self.tue_install_error(f"An error occurred while installing pip packages: {pkg}") + # ToDo: This depends on behaviour of tue-install-error + return False + + return True + + def tue_install_snap(self, pkgs: List[str]) -> bool: + self.tue_install_debug(f"tue-install-snap {pkgs=}") + if not pkgs: + self.tue_install_error("Invalid tue-install-snap call: needs packages as argument") + # ToDo: This depends on behaviour of tue-install-error + return False + + self._snaps.extend(pkgs) + return True + + def tue_install_snap_now(self, pkgs: List[str]) -> bool: + self.tue_install_debug(f"tue-install-snap-now {pkgs=}") + + if not pkgs: + self.tue_install_error("Invalid tue-install-snap-now call: needs packages as argument") + # ToDo: This depends on behaviour of tue-install-error + return False + + self.tue_install_system_now(["snapd"]) + + cmd = "snap list" + cmd, cmds = _which_split_cmd(cmd) + self.tue_install_debug(repr(cmd)) + sub = BackgroundPopen(args=cmds, err_handler=self._err_handler, stdout=sp.PIPE, text=True) + sub.wait() + if sub.returncode != 0: + self.tue_install_error(f"Error while getting snap list({sub.returncode}):\n {repr(cmd)}") + # ToDo: This depends on behaviour of tue-install-error + return False + + snaps_installed = sub.stdout.read().splitlines()[1:] + snaps_installed = [snap.split()[0] for snap in snaps_installed] + snaps_to_install = [] + for pkg in pkgs: + if pkg not in snaps_installed: + snaps_to_install.append(pkg) + self.tue_install_debug(f"Snap '{pkg}' is not installed yet") + else: + self.tue_install_debug(f"Snap '{pkg}' is already installed") + + if not snaps_to_install: + self.tue_install_debug("No snaps to install") + return True + + for pkg in snaps_to_install: + cmd = f"sudo snap install --classic {pkg}" + sub = self._default_background_popen(cmd) + breakpoint() + if sub.returncode != 0: + self.tue_install_error(f"An error occurred while installing snap: {pkg}") + # ToDo: This depends on behaviour of tue-install-error + return False + + return True + + def tue_install_gem(self, pkgs: List[str]) -> bool: + self.tue_install_debug(f"tue-install-gem {pkgs=}") + if not pkgs: + self.tue_install_error("Invalid tue-install-gem call: needs packages as argument") + # ToDo: This depends on behaviour of tue-install-error + return False + + self._gems.extend(pkgs) + return True + + def tue_install_gem_now(self, pkgs: List[str]) -> bool: + self.tue_install_debug(f"tue-install-gem-now {pkgs=}") + + if not pkgs: + self.tue_install_error("Invalid tue-install-gem-now call: got an empty list of packages as argument.") + # ToDo: This depends on behaviour of tue-install-error + return False + + self.tue_install_system_now(["ruby", "ruby-dev", "rubygems-integration"]) + + cmd = "gem list" + cmd, cmds = _which_split_cmd(cmd) + self.tue_install_debug(repr(cmd)) + sub = BackgroundPopen(args=cmds, err_handler=self._err_handler, stdout=sp.PIPE, text=True) + sub.wait() + if sub.returncode != 0: + self.tue_install_error(f"Error while getting installed gem packages({sub.returncode}):\n {repr(cmd)}") + # ToDo: This depends on behaviour of tue-install-error + return False + + gems_installed = sub.stdout.read().splitlines() + gems_installed = [gem.split(" ")[0] for gem in gems_installed] + gems_to_install = [] + for pkg in pkgs: + if pkg not in gems_installed: + gems_to_install.append(pkg) + self.tue_install_debug(f"gem pkg: {pkg} is not yet installed") + else: + self.tue_install_debug(f"gem pkg: {pkg} is already installed") + + if gems_to_install: + cmd = f"gem install {' '.join(gems_to_install)}" + sub = self._default_background_popen(cmd) + if sub.returncode != 0: + self.tue_install_error( + f"An error occurred while installing gem packages({sub.returncode}):\n {repr(cmd)}" + ) + # ToDo: This depends on behaviour of tue-install-error + return False + + return True + + def tue_install_dpkg_now(self, pkg_file: str) -> bool: + self.tue_install_debug(f"tue-install-dpkg-now {pkg_file=}") + + if not pkg_file: + self.tue_install_error("Invalid tue-install-dpkg-now call: got an empty package file as argument.") + # ToDo: This depends on behaviour of tue-install-error + return False + + if not os.path.isabs(pkg_file): + pkg_file = os.path.join(self._current_target_dir, pkg_file) + + if not os.path.isfile(pkg_file): + self.tue_install_error(f"Invalid tue-install-dpkg-now call: {pkg_file} is not a file.") + # ToDo: This depends on behaviour of tue-install-error + return False + + cmd = f"sudo dpkg -i {pkg_file}" + _ = self._default_background_popen(cmd) + # ToDo: Should we check the return code? + # if sub.returncode != 0: + # self.tue_install_error(f"An error occurred while installing dpkg package({sub.returncode}):" + # f"\n {repr(cmd)}") + # # ToDo: This depends on behaviour of tue-install-error + # return False + + cmd = f"sudo apt-get install -f" + sub = self._default_background_popen(cmd) + if sub.returncode != 0: + self.tue_install_error(f"An error occurred while fixing dpkg install({sub.returncode}):\n {repr(cmd)}") + # ToDo: This depends on behaviour of tue-install-error + return False + + return True + + tue_install_dpkg = tue_install_dpkg_now + + def tue_install_ros(self, source_type: str, **kwargs) -> bool: + self.tue_install_debug(f"tue-install-ros {source_type=} {kwargs=}") + + if not source_type: + self.tue_install_error("Invalid tue-install-ros call: needs source type as argument") + # ToDo: This depends on behaviour of tue-install-error + return False + + tue_ros_distro = os.getenv("TUE_ROS_DISTRO", None) + if not tue_ros_distro: + self.tue_install_error("TUE_ROS_DISTRO is not set") + # ToDo: This depends on behaviour of tue-install-error + return False + tue_ros_version = os.getenv("TUE_ROS_VERSION", None) + if not tue_ros_version: + self.tue_install_error("TUE_ROS_VERSION is not set") + # ToDo: This depends on behaviour of tue-install-error + return False + + # Remove 'ros-' prefix + ros_pkg_name = self._current_target[4:] + if "-" in ros_pkg_name: + correct_ros_pkg_name = ros_pkg_name.replace("-", "_") + self.tue_install_error( + f"A ROS package cannot contain dashes ({ros_pkg_name}), " + f"make sure the package is named '{correct_ros_pkg_name}' and rename the " + f"target to 'ros-{correct_ros_pkg_name}'" + ) + # ToDo: This depends on behaviour of tue-install-error + return False + + # First of all, make sure ROS itself is installed + if not self.tue_install_target(f"ros{tue_ros_version}"): + self.tue_install_error(f"Failed to install ros{tue_ros_version}") + + if source_type == "system": + """ + Install a ROS package from the system repository + kwargs: + - name: The name of the package to install + """ + name = kwargs["name"] + if name is None: + self.tue_install_error("Invalid tue-install-ros call(system): needs 'name' as argument") + # ToDo: This depends on behaviour of tue-install-error + return False + + self.tue_install_debug(f"tue-install-system ros-{tue_ros_distro}-{name}") + if not self.tue_install_system([f"ros-{tue_ros_distro}-{name}"]): + self.tue_install_error(f"Failed to append ros-{tue_ros_distro}-{name}") + # ToDo: This depends on behaviour of tue-install-error + return False + + return True + + if source_type != "git": + self.tue_install_error(f"Unknown ROS source type: {source_type}") + # ToDo: This depends on behaviour of tue-install-error + return False + + """ + Install a ROS package from a git repository + kwargs: + - url: The git repository to clone + - sub_dir (optional): The subdirectory of the repository to install + - version (optional): The version of the package to install + - target_dir (optional): The directory to install the package to + """ + + url = kwargs["url"] + if url is None: + self.tue_install_error("Invalid tue-install-ros call(git): needs url as argument") + + sub_dir = kwargs["sub_dir"] + if sub_dir is None: + sub_dir = "" + version = kwargs["version"] + target_dir = kwargs["target_dir"] + + tue_system_dir = os.getenv("TUE_SYSTEM_DIR", None) + if not tue_system_dir: + self.tue_install_error("ros_package_install_dir is not set") + # ToDo: This depends on behaviour of tue-install-error + return False + + # Make sure the ROS package install dir exists + ros_package_install_dir = os.path.join(tue_system_dir, "src") + if not os.path.isdir(ros_package_install_dir): + self.tue_install_error(f"ros_package_install_dir is not a directory: {ros_package_install_dir}") + # ToDo: This depends on behaviour of tue-install-error + return False + + # If repos_dir is not set, try generating the default path from git url + if target_dir is None: + # ToDo: convert _git_url_to_repos_dir to python + cmd = f"bash -c '_git_url_to_repos_dir {url}'" + cmd, cmds = _which_split_cmd(cmd) + target_dir = sp.check_output(cmds, text=True).strip() + if not target_dir: + self.tue_install_error(f"Could not create target_dir path from the git url: '{url}'") + # ToDo: This depends on behaviour of tue-install-error + return False + + self.tue_install_debug(f"{target_dir=}") + target_sub_dir = os.path.join(target_dir, sub_dir) + + ros_pkg_dir = os.path.join(ros_package_install_dir, ros_pkg_name) + + if not self.tue_install_git(url, target_dir, version): + self.tue_install_error(f"Failed to clone ros package '{ros_pkg_name}' from '{url}'") + + if not os.path.isdir(target_dir): + self.tue_install_error(f"ROS package '{ros_pkg_name}' from '{url}' was not cloned to '{target_dir}'") + # ToDo: This depends on behaviour of tue-install-error + return False + + if not os.path.isdir(target_sub_dir): + self.tue_install_error(f"Subdirectory '{sub_dir}' does not exist for url '{url}'") + # ToDo: This depends on behaviour of tue-install-error + return False + + # Test if the current symbolic link points to the same repository dir. If not, give a warning + # because it means the source URL has changed + if os.path.islink(ros_pkg_dir): + if os.path.realpath(ros_pkg_dir) != os.path.realpath(target_sub_dir): + self.tue_install_info(f"url has changed to {url}/{sub_dir}") + os.remove(ros_pkg_dir) + os.symlink(target_sub_dir, ros_pkg_dir) + elif os.path.isdir(ros_pkg_dir): + self.tue_install_error(f"Can not create a symlink at '{ros_pkg_dir}' as it is a directory") + # ToDo: This depends on behaviour of tue-install-error + return False + elif not os.path.exists(ros_pkg_dir): + self.tue_install_debug(f"Creating symlink from {target_sub_dir} to {ros_pkg_dir}") + os.symlink(target_sub_dir, ros_pkg_dir) + else: + self.tue_install_error(f"'{ros_pkg_dir}' should not exist or be a symlink. Any other option is incorrect") + + if self._skip_ros_deps and not self._ros_test_deps and not self._ros_doc_deps: + self.tue_install_debug("Skipping resolving of ROS dependencies") + return True + + # Resolve ROS dependencies + pkg_xml = os.path.join(ros_pkg_dir, PACKAGE_MANIFEST_FILENAME) + if not os.path.isfile(pkg_xml): + self.tue_install_warning(f"Does not contain a valid ROS {PACKAGE_MANIFEST_FILENAME}") + return True + + self.tue_install_debug(f"Parsing {pkg_xml}") + try: + _, deps = catkin_package_parser(pkg_xml, self._skip_ros_deps, self._ros_test_deps, self._ros_doc_deps) + except IOError as e: + self.tue_install_error(f"Could not parse {pkg_xml}: {e}") + # ToDo: This depends on behaviour of tue-install-error + return False + except InvalidPackage as e: + self.tue_install_error(f"Invalid {PACKAGE_MANIFEST_FILENAME}:\n{e}") + # ToDo: This depends on behaviour of tue-install-error + return False + except ValueError as e: + self.tue_install_error(f"Could not evaluate conditions in {pkg_xml}: {e}") + # ToDo: This depends on behaviour of tue-install-error + return False + except RuntimeError as e: + self.tue_install_error(f"Unevaluated condition found in {pkg_xml}: {e}") + # ToDo: This depends on behaviour of tue-install-error + return False + + for dep in deps: + success = self.tue_install_target(f"ros-{dep.name}") or self.tue_install_target(dep.name) + if not success: + self.tue_install_error(f"Targets 'ros-{dep.name}' and '{dep}' don't exist") + # ToDo: This depends on behaviour of tue-install-error + return False + + # ToDO: TUE_INSTALL_PKG_DIR was set ros_pkg_dir which was then use in tue-install-apply-patch; we are not doing that not (yet) in python + return True + + def install(self, targets: List[str]) -> bool: + if not targets: + self.tue_install_error("No targets to install") + # ToDo: This depends on behaviour of tue-install-error + return False + + missing_targets = [] + for target in targets: + if not self._target_exist(target): + missing_targets.append(target) + + if missing_targets: + self.tue_install_error(f"The following installed targets don't exist (anymore):\n{sorted(missing_targets)}") + # ToDo: This depends on behaviour of tue-install-error + return False + + for target in targets: + self.tue_install_debug(f"Installing target '{target}'") + if not self.tue_install_target(target): + self.tue_install_error(f"Failed to install target '{target}'") + return False + + # Mark as installed + self.tue_install_debug(f"Marking '{target}' as installed after successful installation") + Path(os.path.join(self._installed_dir, target)).touch(exist_ok=True) + + return True + + def update(self, targets: List[str]) -> bool: + if not targets: + targets = self._get_installed_targets() + else: + installed_targets = self._get_installed_targets() + for target in targets: + if target not in installed_targets: + self.tue_install_error(f"Target '{target}' is not installed") + # ToDo: This depends on behaviour of tue-install-error + return False + + missing_targets = [] + for target in targets: + if not self._target_exist(target): + missing_targets.append(target) + + if missing_targets: + self.tue_install_error(f"The following installed targets don't exist (anymore):\n{sorted(missing_targets)}") + # ToDo: This depends on behaviour of tue-install-error + return False + + for target in targets: + self.tue_install_debug(f"Updating target '{target}'") + if not self.tue_install_target(target): + self.tue_install_error(f"Failed to update target '{target}'") + return False + self.tue_install_debug(f"{target} succesfully updated") + + return True + + def print_queued_logs(self) -> bool: + if self._info_logs: + logs = "\n ".join(self._info_logs) + print(f"Some information you may have missed:\n\n {logs}\n") + + if self._warn_logs: + logs = "\n ".join(self._warn_logs) + print(f"Overview of warnings:\n\n {logs}\n") + + return True + + def install_queued_pkgs(self) -> bool: + if self._ppas: + with self._set_target("PPA-ADD"): + self.tue_install_debug(f"calling tue-install-ppa-now {self._ppas}") + if not self.tue_install_ppa_now(self._ppas): + self.tue_install_error(f"Failed to add PPA's: {self._ppas}") + # ToDo: This depends on behaviour of tue-install-error + return False + + if self._systems: + with self._set_target("APT-GET"): + self.tue_install_debug(f"calling tue-install-system-now {self._systems}") + if not self.tue_install_system_now(self._systems): + self.tue_install_error(f"Failed to install system packages: {self._systems}") + # ToDo: This depends on behaviour of tue-install-error + return False + + if self._pips: + with self._set_target("PIP"): + self.tue_install_debug(f"calling tue-install-pip-now {self._pips}") + if not self.tue_install_pip_now(self._pips): + self.tue_install_error(f"Failed to install pip packages: {self._pips}") + # ToDo: This depends on behaviour of tue-install-error + return False + + if self._snaps: + with self._set_target("SNAP"): + self.tue_install_debug(f"calling tue-install-snap-now {self._snaps}") + if not self.tue_install_snap_now(self._snaps): + self.tue_install_error(f"Failed to install snap packages: {self._snaps}") + # ToDo: This depends on behaviour of tue-install-error + return False + + if self._gems: + with self._set_target("GEM"): + self.tue_install_debug(f"calling tue-install-gem-now {self._gems}") + if not self.tue_install_gem_now(self._gems): + self.tue_install_error(f"Failed to install gem packages: {self._gems}") + # ToDo: This depends on behaviour of tue-install-error + return False + + return True + + +if __name__ == "__main__": + import argparse + import sys + + ros_test_depends = os.environ.get("TUE_INSTALL_TEST_DEPENDS", False) + ros_doc_depends = os.environ.get("TUE_INSTALL_DOC_DEPENDS", False) + + parser = argparse.ArgumentParser(prog="tue-get", description="Installs all your (ROS) dependencies") + parser.add_argument("--branch", "-b", help="Branch to checkout", default=None) + parser.add_argument("--debug", "-d", action="store_true", help="Enable debug output", default=False) + parser.add_argument("--no-ros-deps", action="store_true", help="Skip resolving of ROS dependencies", default=False) + parser.add_argument( + "mode", + choices=["install", "update"], + type=str, + help="Install OR update the targets", + ) + m = parser.add_mutually_exclusive_group(required=False) + m.add_argument( + "--test-depends", + dest="test_depends", + action="store_true", + help="Also resolve ROS test dependencies", + default=ros_test_depends, + ) + m.add_argument( + "--no-test-depends", + dest="test_depends", + action="store_false", + help="Do not resolve ROS test dependencies", + default=ros_test_depends, + ) + m = parser.add_mutually_exclusive_group(required=False) + m.add_argument( + "--doc-depends", + dest="doc_depends", + action="store_true", + help="Also resolve ROS doc dependencies", + default=ros_doc_depends, + ) + m.add_argument( + "--no-doc-depends", + dest="doc_depends", + action="store_false", + help="Do not resolve ROS doc dependencies", + default=ros_doc_depends, + ) + parser.add_argument("targets", help="Targets to install", nargs=argparse.REMAINDER) + + args = parser.parse_args() + if args.mode == "install" and not args.targets: + parser.error("Minimal one target should be specified, when installing") + + installer = InstallerImpl( + branch=args.branch, + debug=args.debug, + skip_ros_deps=args.no_ros_deps, + ros_test_deps=args.test_depends, + ros_doc_deps=args.doc_depends, + ) + + targets = sorted(args.targets) + if args.mode == "install": + if not installer.install(targets): + installer.tue_install_error(f"Failed to install targets: {targets}") + sys.exit(1) + elif args.mode == "update": + if not installer.update(targets): + installer.tue_install_error(f"Failed to update targets: {targets}") + sys.exit(1) + + if not installer.print_queued_logs(): + pass + + if not installer.install_queued_pkgs(): + sys.exit(1) + + installer.tue_install_echo("Installer completed successfully") + sys.exit(0) diff --git a/src/tue_get/resources/installer_impl.bash b/src/tue_get/resources/installer_impl.bash new file mode 100644 index 000000000..af4632dc9 --- /dev/null +++ b/src/tue_get/resources/installer_impl.bash @@ -0,0 +1,260 @@ +#! /usr/bin/env bash + +function tue-install-error +{ + echo -e "tue-install-error: $(echo -e "$*" | tr '\n' '^')" + local return_value + read -r return_value + return $(("$return_value")) +} + +function tue-install-warning +{ + echo -e "tue-install-warning: $(echo -e "$*" | tr '\n' '^')" + local return_value + read -r return_value + return $(("$return_value")) +} + +function tue-install-info +{ + echo -e "tue-install-info: $(echo -e "$*" | tr '\n' '^')" + local return_value + read -r return_value + return $(("$return_value")) +} + +function tue-install-debug +{ + echo -e "tue-install-debug: $(echo -e "$*" | tr '\n' '^')" + local return_value + read -r return_value + return $(("$return_value")) +} + +function tue-install-echo +{ + echo -e "tue-install-echo: $(echo -e "$*" | tr '\n' '^')" + local return_value + read -r return_value + return $(("$return_value")) +} + +function tue-install-tee +{ + echo -e "tue-install-tee: $(echo -e "$*" | tr '\n' '^')}" + local return_value + read -r return_value + return $(("$return_value")) +} + +function tue-install-pipe +{ + local return_code + local pipefail_old return_code + pipefail_old=$(set -o | grep pipefail | awk '{printf $2}') + [ "$pipefail_old" != "on" ] && set -o pipefail # set pipefail if not yet set + tue-install-echo "$*" + # Executes the command (all arguments), catch stdout and stderr, red styled, print them directly and to file + { + IFS=$'\n' read -r -d '' CAPTURED_STDERR; + IFS=$'\n' read -r -d '' CAPTURED_STDOUT; + } < <((printf '\0%s\0' "$("$@")" 1>&2) 2>&1) + return_code=$? + [ "$pipefail_old" != "on" ] && set +o pipefail # restore old pipefail setting + # shellcheck disable=SC2034 + TUE_INSTALL_PIPE_STDOUT=$CAPTURED_STDOUT + + CAPTURED_STDOUT=$(echo -e "$CAPTURED_STDOUT" | tr '\n' '^') + CAPTURED_STDERR=$(echo -e "$CAPTURED_STDERR" | tr '\n' '^') + echo -e "tue-install-pipe: ${CAPTURED_STDOUT}^^^${CAPTURED_STDERR}" + read -r return_value + if [ "$return_value" != "0" ] + then + return $(("$return_value")) + fi + return $return_code +} + +function tue-install-target-now +{ + echo -e "tue-install-target-now: $*" + local return_value + read -r return_value + return $(("$return_value")) +} + +function tue-install-target +{ + echo -e "tue-install-target: $*" + local return_value + read -r return_value + return $(("$return_value")) +} + +function tue-install-git +{ + local url targetdir version + url=$1 + shift + for i in "$@" + do + case $i in + --target-dir=* ) + targetdir="${i#*=}" + ;; + --version=* ) + version="${i#*=}" ;; + * ) + tue-install-error "Unknown input variable ${i}" ;; + esac + done + echo -e "tue-install-git: ${url} ${targetdir} ${version}" + local return_value + read -r return_value + return $(("$return_value")) +} + +function tue-install-apply-patch +{ + echo -e "tue-install-apply-patch: $*" + local return_value + read -r return_value + return $(("$return_value")) +} + +function tue-install-cp +{ + echo -e "tue-install-cp: $*" + local return_value + read -r return_value + return $(("$return_value")) +} + +function tue-install-add-text +{ + echo -e "tue-install-add-text: $*" + local return_value + read -r return_value + return $(("$return_value")) +} + +function tue-install-get-releases +{ + echo -e "tue-install-get-releases: $*" + local return_value + read -r return_value + return $(("$return_value")) +} + +function tue-install-system +{ + echo -e "tue-install-system: $*" + local return_value + read -r return_value + return $(("$return_value")) +} + +function tue-install-system-now +{ + echo -e "tue-install-system-now: $*" + local return_value + read -r return_value + return $(("$return_value")) +} + +function tue-install-apt-get-update +{ + echo -e "tue-install-apt-get-update: $*" + local return_value + read -r return_value + return $(("$return_value")) +} + +function tue-install-ppa +{ + local ppa_list + for ppa in "$@" + do + ppa_list="${ppa_list:+${ppa_list} }${ppa// /^}" + done + echo -e "tue-install-ppa: ${ppa_list}" + local return_value + read -r return_value + return $(("$return_value")) +} + +function tue-install-ppa-now +{ + local ppa_list + for ppa in "$@" + do + ppa_list="${ppa_list:+${ppa_list} }${ppa// /^}" + done + echo -e "tue-install-ppa-now: $ppa_list" + local return_value + read -r return_value + return $(("$return_value")) +} + +function tue-install-pip +{ + echo -e "tue-install-pip: $*" + local return_value + read -r return_value + return $(("$return_value")) +} + +# TEMP for backward compatibility +function tue-install-pip3 +{ + tue-install-pip "$*" + return $? +} + +function tue-install-pip-now +{ + echo -e "tue-install-pip-now: $*" + local return_value + read -r return_value + return $(("$return_value")) +} + +# TEMP for backward compatibility +function tue-install-pip3-now +{ + tue-install-pip-now "$*" + return $? +} + +function tue-install-snap +{ + echo -e "tue-install-snap: $*" + local return_value + read -r return_value + return $(("$return_value")) +} + +function tue-install-snap-now +{ + echo -e "tue-install-snap-now: $*" + local return_value + read -r return_value + return $(("$return_value")) +} + +function tue-install-gem +{ + echo -e "tue-install-gem: $*" + local return_value + read -r return_value + return $(("$return_value")) +} + +function tue-install-gem-now +{ + echo -e "tue-install-gem-now: $*" + local return_value + read -r return_value + return $(("$return_value")) +} diff --git a/src/tue_get/util/__init__.py b/src/tue_get/util/__init__.py new file mode 100644 index 000000000..1259abb53 --- /dev/null +++ b/src/tue_get/util/__init__.py @@ -0,0 +1,2 @@ +from . import background_popen +from . import grep diff --git a/src/tue_get/util/background_popen.py b/src/tue_get/util/background_popen.py new file mode 100644 index 000000000..340841eea --- /dev/null +++ b/src/tue_get/util/background_popen.py @@ -0,0 +1,44 @@ +from typing import Callable, IO, Optional +import subprocess +from threading import Thread + + +class BackgroundPopen(subprocess.Popen): + """ + Inspired by https://stackoverflow.com/a/42352331 + """ + + def _proxy_lines(self, pipe: IO, handler: Callable): + with pipe: + for line in pipe: + handler(self, line) + + def __init__(self, out_handler: Optional[Callable] = None, err_handler: Optional[Callable] = None, *args, **kwargs): + if out_handler is not None: + kwargs["stdout"] = subprocess.PIPE + if err_handler is not None: + kwargs["stderr"] = subprocess.PIPE + super().__init__(*args, **kwargs) + if out_handler is not None: + Thread(target=self._proxy_lines, args=[self.stdout, out_handler]).start() + if err_handler is not None: + Thread(target=self._proxy_lines, args=[self.stderr, err_handler]).start() + + +if __name__ == "__main__": + import shlex + from termcolor import cprint + + cmd = """ + bash -c ' + for i in {1..100} + do + echo -e "${i}" + done' + """ + + def cyan_handler(sub: BackgroundPopen, line: str): + cprint(line.strip(), color="cyan") + + bla = BackgroundPopen(out_handler=cyan_handler, args=shlex.split(cmd), text=True) + bla.wait() diff --git a/src/tue_get/util/grep.py b/src/tue_get/util/grep.py new file mode 100644 index 000000000..34d7aee84 --- /dev/null +++ b/src/tue_get/util/grep.py @@ -0,0 +1,75 @@ +from typing import List +import os +import re + +""" +Text functions inspired by https://eli.thegreenplace.net/2011/10/19/perls-guess-if-file-is-text-or-binary-implemented-in-python +""" + +text_characters = b"".join(map(lambda x: bytes((x,)), range(32, 127))) + b"\n\r\t\f\b" +_null_trans = bytes.maketrans(b"", b"") + + +def istextfile(filename: str, blocksize: int = 512) -> bool: + return istext(open(filename, "rb").read(blocksize)) + + +def istext(b: bytes) -> bool: + if b"\0" in b: + return False + + if not b: # Empty files are considered text + return True + + # Get the non-text characters (maps a character to itself then + # use the 'remove' option to get rid of the text characters.) + t = b.translate(_null_trans, text_characters) + + # If more than 30% non-text characters, then + # this is considered a binary file + if len(t) / len(b) > 0.30: + return False + return True + + +def grep_directory( + directory: str, pattern: re.Pattern, recursive: bool = False, include_binary: bool = False +) -> List[str]: + """ + Searches for a regex in a directory. + + :param directory: The directory to search in. + :param pattern: The regex to search with. + :param recursive: (optional) Whether to search recursively. Defaults to False. + :param include_binary: (optional) Whether to include binary files. Defaults to False. + + :return: A list of files that match the regex. + """ + files = [] + for root, dirs, filenames in os.walk(directory): + for filename in filenames: + full_path = os.path.join(root, filename) + if not include_binary and not istextfile(full_path): + continue + if grep_file(full_path, pattern): + files.append(full_path) + if not recursive: + break + return files + + +def grep_file(file: str, pattern: re.Pattern) -> bool: + """ + Searches for a regex in a file. + + :param file: The file to search in. + :param pattern: The regex to search with. + + :return: Whether the regex was found in the file. + """ + with open(file, "r") as f: + lines = f.readlines() + for line in lines: + if pattern.search(line): + return True + return False diff --git a/test/test_tue_get/__init__.py b/test/test_tue_get/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/test/test_tue_get/test_install_yaml_parser.py b/test/test_tue_get/test_install_yaml_parser.py new file mode 100644 index 000000000..11a7abc03 --- /dev/null +++ b/test/test_tue_get/test_install_yaml_parser.py @@ -0,0 +1,119 @@ +import os +from typing import List, Optional + +import tempfile +import yaml + +from tue_get import install_yaml_parser + + +class MockInstaller: + def __init__(self): + pass + + def tue_install_target_now(self, target: str): + pass + + def tue_install_target(self, target: str, now: bool = False): + pass + + def tue_install_git(self, url: str, target_dir: Optional[str] = None, version: Optional[str] = None): + pass + + def tue_install_apply_patch(self, patch_file: str, target_dir: str): + pass + + def tue_install_cp(self, source_file: str, target_file: str): + pass + + def tue_install_add_text(self, source_file: str, target_file: str): + pass + + def tue_install_get_releases(self, url: str, filename: str, output_dir: str, tag: Optional[str] = None): + pass + + def tue_install_system(self, pkgs: List[str]): + pass + + def tue_install_system_now(self, pkgs: List[str]): + pass + + def tue_install_apt_get_update(self): + pass + + def tue_install_ppa(self, ppa: List[str]): + pass + + def tue_install_ppa_now(self, ppa: List[str]): + pass + + def tue_install_pip(self, pkgs: List[str]): + pass + + def tue_install_pip_now(self, pkgs: List[str]): + pass + + def tue_install_snap(self, pkgs: List[str]): + pass + + def tue_install_snap_now(self, pkgs: List[str]): + pass + + def tue_install_gem(self, pkgs: List[str]): + pass + + def tue_install_gem_now(self, pkgs: List[str]): + pass + + def tue_install_dpkg(self, pkg_file: str): + pass + + def tue_install_dpkg_now(self, pkg_file: str): + pass + + def tue_install_ros(self, source_type: str, **kwargs): + pass + + +def parse_target(yaml_seq: List, now: bool = False): + install_file = tempfile.NamedTemporaryFile(mode="w", delete=False) + try: + dumper = yaml.CSafeDumper + except AttributeError: + dumper = yaml.SafeDumper + + yaml.dump(yaml_seq, install_file, dumper) + + install_file.close() + + installer = MockInstaller() + try: + cmds = install_yaml_parser.installyaml_parser(installer, install_file.name, now)["commands"] + except Exception: + raise + finally: + os.unlink(install_file.name) + + return cmds + + +def test_empty_target(): + target = [{"type": "empty"}] + cmds = parse_target(target, False) + assert not cmds + cmds = parse_target(target, True) + assert not cmds + + +def test_system_target(): + target = [{"type": "system", "name": "pkg_name"}] + cmds = parse_target(target, False) + assert cmds + assert len(cmds) == 1 + for cmd in cmds: + cmd() + # assert isinstance(cmds[0].func, MockInstaller.tue_install_system) + cmds = parse_target(target, True) + assert cmds + assert len(cmds) == 1 + # assert isinstance(cmds[0].func, MockInstaller.tue_install_system_now)