diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index 562bf29e1..db98ad069 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -2,3 +2,7 @@ d71897d20d9816b2c86bb23d2d8ba78a24622a8d # (bootstrap) use curly brackets for vars 5c2e670137b009ec55eff5f101575bfc3652ed6c +# (tue-get) consistent case styling +ef656c09024fbdb65b56a80bc40e0fa15d7b8a2d +# (bootstrap) consistent case styling +1a66fd24500c4b9d8252eba22f7fe7fb277cefa0 diff --git a/ci/build-package.sh b/ci/build-package.sh index a4d9cf6ae..9c3cd34f1 100755 --- a/ci/build-package.sh +++ b/ci/build-package.sh @@ -50,24 +50,24 @@ fi if [[ "${ROS_VERSION}" == 1 ]] then echo -e "\e[35m\e[1mCompile the package (catkin build -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCATKIN_ENABLE_TESTING=OFF)\e[0m" - docker exec -t tue-env bash -c 'source ~/.bashrc; cd "${TUE_SYSTEM_DIR}"/src/"${PACKAGE}" && /usr/bin/python3 "$(command -v catkin)" build --this --no-status -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCATKIN_ENABLE_TESTING=OFF' + docker exec -t tue-env bash -c 'source ~/.bashrc; cd "${TUE_SYSTEM_DIR}"/src/"${PACKAGE}" && (deactivate &>/dev/null; /usr/bin/python3 "$(command -v catkin)" build --this --no-status -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCATKIN_ENABLE_TESTING=OFF)' else echo -e "\e[35m\e[1mCheck for default mixin repo (colcon mixin list)\e[0m" - MIXIN_REPOS=$(docker exec -t tue-env bash -c 'source ~/.bashrc; cd "${TUE_SYSTEM_DIR}" && colcon mixin list | grep -v "^- "' | tr -d '\r' | awk -F ": " '{print $1}') + MIXIN_REPOS=$(docker exec -t tue-env bash -c 'source ~/.bashrc; cd "${TUE_SYSTEM_DIR}" && (deactivate &>/dev/null; /usr/bin/python3 -m colcon mixin list) | grep -v "^- "' | tr -d '\r' | awk -F ": " '{print $1}') if ! echo -e "${MIXIN_REPOS}" | grep "^default$" -q then echo -e "\e[35m\e[1mAdd the default mixin repo (colcon mixin add default https://raw.githubusercontent.com/colcon/colcon-mixin-repository/master/index.yaml)\e[0m" - docker exec -t tue-env bash -c 'source ~/.bashrc; cd "${TUE_SYSTEM_DIR}" && colcon mixin add default https://raw.githubusercontent.com/colcon/colcon-mixin-repository/master/index.yaml' + docker exec -t tue-env bash -c 'source ~/.bashrc; cd "${TUE_SYSTEM_DIR}" && (deactivate &>/dev/null; /usr/bin/python3 -m colcon mixin add default https://raw.githubusercontent.com/colcon/colcon-mixin-repository/master/index.yaml)' else echo -e "\e[35m\e[1mDefault mixin repo already exists\e[0m" fi echo -e "\e[35m\e[1mUpdate colcon mixins (colcon mixin update)\e[0m" - docker exec -t tue-env bash -c 'source ~/.bashrc; cd "${TUE_SYSTEM_DIR}" && colcon mixin update' + docker exec -t tue-env bash -c 'source ~/.bashrc; cd "${TUE_SYSTEM_DIR}" && (deactivate &>/dev/null; /usr/bin/python3 -m colcon mixin update)' echo -e "\e[35m\e[1mDeleting the merged install directory\e[0m" docker exec -t tue-env bash -c 'source ~/.bashrc; cd "${TUE_SYSTEM_DIR}" && rm -rf install' echo -e "\e[35m\e[1mCompile the package (colcon build --mixin rel-with-deb-info build-testing-off)\e[0m" - docker exec -t tue-env bash -c 'source ~/.bashrc; cd "${TUE_SYSTEM_DIR}" && colcon build --packages-up-to "${PACKAGE}" --mixin rel-with-deb-info build-testing-off --event-handlers desktop_notification- status- terminal_title-' + docker exec -t tue-env bash -c 'source ~/.bashrc; cd "${TUE_SYSTEM_DIR}" && (deactivate &>/dev/null; /usr/bin/python3 -m colcon build --packages-up-to "${PACKAGE}" --mixin rel-with-deb-info build-testing-off --event-handlers desktop_notification- status- terminal_title-)' fi diff --git a/ci/delete-docker-tags.py b/ci/delete-docker-tags.py index 537ef0a30..9293ed694 100755 --- a/ci/delete-docker-tags.py +++ b/ci/delete-docker-tags.py @@ -43,7 +43,7 @@ def _get_authorization_token(self, username, password): Actually gets the authentication token Raises: - AuthenticationError: didn't login right + AuthenticationError: didn't log in correctly """ resp = self._session.post( f"{DH_API}/users/login/", diff --git a/ci/test-package.sh b/ci/test-package.sh index 925e18042..e5e827626 100755 --- a/ci/test-package.sh +++ b/ci/test-package.sh @@ -50,21 +50,21 @@ if [[ "${ROS_VERSION}" == 1 ]] then # Build test targets echo -e "\e[35m\e[1mBuild test targets of this package (catkin build --this --no-deps -DCATKIN_ENABLE_TESTING=ON)\e[0m" - docker exec -t tue-env bash -c 'source ~/.bashrc; cd "${TUE_SYSTEM_DIR}"/src/"${PACKAGE}" && /usr/bin/python3 "$(command -v catkin)" build --this --no-status --no-deps -DCATKIN_ENABLE_TESTING=ON' + docker exec -t tue-env bash -c 'source ~/.bashrc; cd "${TUE_SYSTEM_DIR}"/src/"${PACKAGE}" && (deactivate &>/dev/null; /usr/bin/python3 "$(command -v catkin)" build --this --no-status --no-deps -DCATKIN_ENABLE_TESTING=ON)' # Run unit tests echo -e "\e[35m\e[1mRun tests on this package (catkin test --this --no-deps -DCATKIN_ENABLE_TESTING=ON)\e[0m" - docker exec -t tue-env bash -c 'source ~/.bashrc; cd "${TUE_SYSTEM_DIR}"/src/"${PACKAGE}" && /usr/bin/python3 "$(command -v catkin)" test --this --no-status --no-deps -DCATKIN_ENABLE_TESTING=ON' + docker exec -t tue-env bash -c 'source ~/.bashrc; cd "${TUE_SYSTEM_DIR}"/src/"${PACKAGE}" && (deactivate &>/dev/null; /usr/bin/python3 "$(command -v catkin)" test --this --no-status --no-deps -DCATKIN_ENABLE_TESTING=ON)' else # Build test targets echo -e "\e[35m\e[1mBuild test targets of this package (colcon build --packages-select ${PACKAGE} --mixin rel-with-deb-info build-testing-on)\e[0m" - docker exec -t tue-env bash -c 'source ~/.bashrc; cd "${TUE_SYSTEM_DIR}" && colcon build --packages-select "${PACKAGE}" --mixin rel-with-deb-info build-testing-on --event-handlers desktop_notification- status- terminal_title-' + docker exec -t tue-env bash -c 'source ~/.bashrc; cd "${TUE_SYSTEM_DIR}" && (deactivate &>/dev/null; /usr/bin/python3 -m colcon build --packages-select "${PACKAGE}" --mixin rel-with-deb-info build-testing-on --event-handlers desktop_notification- status- terminal_title-)' # Run unit tests echo -e "\e[35m\e[1mRun tests on this package (colcon test --packages-select ${PACKAGE} --executor sequential)\e[0m" - docker exec -t tue-env bash -c 'source ~/.bashrc; cd "${TUE_SYSTEM_DIR}" && colcon test --packages-select "${PACKAGE}" --executor sequential --event-handlers desktop_notification- status- terminal_title- console_cohesion+' + docker exec -t tue-env bash -c 'source ~/.bashrc; cd "${TUE_SYSTEM_DIR}" && (deactivate &>/dev/null; /usr/bin/python3 -m colcon test --packages-select "${PACKAGE}" --executor sequential --event-handlers desktop_notification- status- terminal_title- console_cohesion+)' # Check test results echo -e "\e[35m\e[1mCheck test results (colcon test-result --verbose)\e[0m" - docker exec -t tue-env bash -c 'source ~/.bashrc; cd "${TUE_SYSTEM_DIR}" && colcon test-result --verbose' + docker exec -t tue-env bash -c 'source ~/.bashrc; cd "${TUE_SYSTEM_DIR}" && (deactivate &>/dev/null; /usr/bin/python3 -m colcon test-result --verbose)' fi diff --git a/dockerfiles/tue-env.Dockerfile b/dockerfiles/tue-env.Dockerfile index 3c8b93552..aae7d295a 100644 --- a/dockerfiles/tue-env.Dockerfile +++ b/dockerfiles/tue-env.Dockerfile @@ -15,17 +15,18 @@ FROM base as builder # Build time arguments # BRANCH is the target branch if in PULL_REQUEST mode else it is the test branch ARG CI=false -ARG BRANCH= +ARG BRANCH ARG PULL_REQUEST=false -ARG COMMIT= -ARG REF_NAME= +ARG COMMIT +ARG REF_NAME # Default is empty and gives ROS1, for ROS2 use --build-arg ROS_VERSION=2 -ARG ROS_VERSION= -ARG ROS_DISTRO= +ARG ROS_VERSION +ARG ROS_DISTRO # Default is empty and will result in the default targets repo -ARG TARGETS_REPO= +ARG TARGETS_REPO ARG CREATE_VENV=false -ARG OAUTH2_TOKEN= +ARG VENV_INCLUDE_SYSTEM_SITE_PACKAGES=false +ARG OAUTH2_TOKEN # Inform scripts that no questions should be asked and set some environment # variables to prevent warnings and errors @@ -79,7 +80,12 @@ RUN --mount=type=ssh,uid=1000 sed -e s/return//g -i ~/.bashrc && \ export COMMIT=$COMMIT && \ export REF_NAME=$REF_NAME && \ # Run the standard installation script - source bootstrap.bash --ros-version="${ROS_VERSION}" --ros-distro="${ROS_DISTRO}" --create-virtualenv="${CREATE_VENV}"--targets-repo="${TARGETS_REPO}" && \ + source bootstrap.bash \ + --ros-version="${ROS_VERSION}" \ + --ros-distro="${ROS_DISTRO}" \ + --create-virtualenv="${CREATE_VENV}" \ + --virtualenv-include-system-site-packages="${VENV_INCLUDE_SYSTEM_SITE_PACKAGES}" \ + --targets-repo="${TARGETS_REPO}" && \ # Make tue-env to be available to the environment source ~/.bashrc && \ # Install target ros diff --git a/docs/CI_Travis_Setup.md b/docs/CI_Travis_Setup.md index b8ab6d822..b80fb1628 100644 --- a/docs/CI_Travis_Setup.md +++ b/docs/CI_Travis_Setup.md @@ -1,42 +1,52 @@ # Setting up SSH support on Travis CI -The travis configuration needs changes to recurse submodules from private gitlab repository. + +The travis configuration needs changes to recurse submodules from private gitlab repository. Steps to enable ssh support with travis and gitlab are as follows: ## On the local PC -1. Generate a ssh keypair to be used with CI - ``` + +1. Generate an ssh keypair to be used with CI + + ```bash mkdir -p ~/travis ssh-keygen -t rsa -b 4096 -N "" -f ~/travis/deploy_key ``` -1. Add the ssh key to travis - ``` +2. Add the ssh key to travis + + ```bash gem install travis cd travis encrypt-file ~/travis/deploy_key --pro --add ``` + The last command will automatically generate and set two environment variables `encrypted__key` and `encrypted__iv`in the repository settings of travis. They are not to be removed. -1. Change the value of `-out` option in `before_install: - openssl` to `./deploy_key` +3. Change the value of `-out` option in `before_install: - openssl` to `./deploy_key` -1. Commit the updated .travis.yml file and the generated file `deploy_key.enc` +4. Commit the updated .travis.yml file and the generated file `deploy_key.enc` ### For targets repository + 1. Append the following lines to .travis.yml under `before_install` - ``` + + ```bash - eval "$(ssh-agent -s)" - chmod 600 ./deploy_key - ssh-add ./deploy_key ``` ### For package repository + 1. Add the following lines to .travis.yml under `install` - ``` + + ```bash - SSH_KEY_PRIVATE=$(cat deploy_key) - bash install-package.sh --package=$PACKAGE --branch=$TRAVIS_BRANCH --commit=$TRAVIS_COMMIT --pullrequest=$TRAVIS_PULL_REQUEST --ssh --ssh-key="$SSH_KEY_PRIVATE" ``` ## On GITLAB + 1. In the repository Settings > CI/CD > Deploy Keys, add the public key from the file ~/travis/deploy_key.pub diff --git a/installer/bootstrap.bash b/installer/bootstrap.bash index a06221d36..ce72a1ad0 100755 --- a/installer/bootstrap.bash +++ b/installer/bootstrap.bash @@ -1,13 +1,17 @@ #! /usr/bin/env bash +# local variables need to be declared before they are used and need to be lowercase +# global variables which needs to be defined outside this script are in uppercase + function conditional_apt_update { - TUE_APT_GET_UPDATED_FILE=/tmp/tue_get_apt_get_updated - if [[ ! -f ${TUE_APT_GET_UPDATED_FILE} ]] + local tue_get_apt_get_updated_file + tue_get_apt_get_updated_file=/tmp/tue_get_apt_get_updated + if [[ ! -f ${tue_get_apt_get_updated_file} ]] then echo "[tue-env](bootstrap) sudo apt-get update -qq" sudo apt-get update -qq || return 1 - touch ${TUE_APT_GET_UPDATED_FILE} + touch ${tue_get_apt_get_updated_file} fi return 0 } @@ -45,34 +49,41 @@ function main installed_or_install virtualenv python3-virtualenv # Check if OS is Ubuntu - DISTRIB_ID="$(lsb_release -si)" - DISTRIB_RELEASE="$(lsb_release -sr)" + local distrib_id distrib_release + distrib_id="$(lsb_release -si)" + distrib_release="$(lsb_release -sr)" - if [[ "${DISTRIB_ID}" != "Ubuntu" ]] + if [[ "${distrib_id}" != "Ubuntu" ]] then - echo "[tue-env](bootstrap) Unsupported OS ${DISTRIB_ID}. Use Ubuntu." + echo "[tue-env](bootstrap) Unsupported OS ${distrib_id}. Use Ubuntu." return 1 fi - # Set ROS version - TUE_ROS_DISTRO= - TUE_ROS_VERSION= + # Initialize variables + local create_virtualenv targets_repo tue_ppm tue_ros_distro tue_ros_version virtualenv_include_system_site_packages + + # Default values + tue_ppm="pip" + create_virtualenv="true" + virtualenv_include_system_site_packages="false" for i in "$@" do case $i in --ros-version=* ) - ros_version="${i#*=}" - ;; + ros_version="${i#*=}" ;; --ros-distro=* ) - ros_distro="${i#*=}" - ;; + ros_distro="${i#*=}" ;; --targets-repo=* ) - targets_repo="${i#*=}" - ;; + targets_repo="${i#*=}" ;; --create-virtualenv=* ) - create_virtualenv="${i#*=}" - ;; + create_virtualenv="${i#*=}" ;; + --virtualenv-include-system-site-packages=* ) + virtualenv_include_system_site_packages="${i#*=}" ;; + --pip ) + tue_ppm="pip" ;; + --poetry ) + tue_ppm="poetry" ;; * ) echo "[tue-env](bootstrap) Error! Unknown argument '${i}' provided to bootstrap script." return 1 @@ -80,40 +91,47 @@ function main esac done - case ${DISTRIB_RELEASE} in + # Poetry should only be used in combination with a virtualenv + if [[ "${tue_ppm}" == "poetry" && "${create_virtualenv}" == "false" ]] + then + echo "[tue-env](bootstrap) Error! Poetry should only be used in combination with a virtualenv." + return 1 + fi + + case ${distrib_release} in "20.04") if [[ "${ros_version}" -eq 2 ]] then - TUE_ROS_VERSION=2 + tue_ros_version=2 if [[ "${ros_distro}" == "foxy" ]] then - TUE_ROS_DISTRO="foxy" + tue_ros_distro="foxy" elif [[ "${ros_distro}" == "galactic" ]] then - TUE_ROS_DISTRO="galactic" + tue_ros_distro="galactic" elif [[ "${ros_distro}" == "rolling" ]] then - TUE_ROS_DISTRO="rolling" + tue_ros_distro="rolling" elif [[ -n "${ros_distro}" ]] then echo "[tue-env](bootstrap) Error! ROS ${ros_distro} is unsupported with tue-env." return 1 else - TUE_ROS_DISTRO="galactic" - echo "[tue-env](bootstrap) Using default ROS_DISTRO '${TUE_ROS_DISTRO}' with ROS_VERSION '${TUE_ROS_VERSION}'" + tue_ros_distro="galactic" + echo "[tue-env](bootstrap) Using default ROS_DISTRO '${tue_ros_distro}' with ROS_VERSION '${tue_ros_version}'" fi elif [[ "${ros_version}" -eq 1 ]] then - TUE_ROS_DISTRO="noetic" - TUE_ROS_VERSION=1 + tue_ros_distro="noetic" + tue_ros_version=1 elif [[ -n "${ros_version}" ]] then echo "[tue-env](bootstrap) Error! ROS ${ros_version} is unsupported with tue-env." return 1 else - TUE_ROS_DISTRO="noetic" - TUE_ROS_VERSION=1 - echo "[tue-env](bootstrap) Using default ROS_DISTRO '${TUE_ROS_DISTRO}' with ROS_VERSION '${TUE_ROS_VERSION}'" + tue_ros_distro="noetic" + tue_ros_version=1 + echo "[tue-env](bootstrap) Using default ROS_DISTRO '${tue_ros_distro}' with ROS_VERSION '${tue_ros_version}'" fi ;; "22.04") @@ -122,43 +140,45 @@ function main echo "[tue-env](bootstrap) Error! Only ROS version 2 is supported with ubuntu 22.04 and newer" return 1 fi - TUE_ROS_VERSION=2 + tue_ros_version=2 if [[ "${ros_distro}" == "humble" ]] then - TUE_ROS_DISTRO="humble" + tue_ros_distro="humble" elif [[ "${ros_distro}" == "rolling" ]] then - TUE_ROS_DISTRO="rolling" + tue_ros_distro="rolling" elif [[ -n "${ros_distro}" ]] then echo "[tue-env](bootstrap) Error! ROS ${ros_distro} is unsupported with tue-env." return 1 else - TUE_ROS_DISTRO="humble" - echo "[tue-env](bootstrap) Using default ROS_DISTRO '${TUE_ROS_DISTRO}' with ROS_VERSION '${TUE_ROS_VERSION}'" + tue_ros_distro="humble" + echo "[tue-env](bootstrap) Using default ROS_DISTRO '${tue_ros_distro}' with ROS_VERSION '${tue_ros_version}'" fi ;; *) - echo "[tue-env](bootstrap) Ubuntu ${DISTRIB_RELEASE} is unsupported. Please use one of Ubuntu 20.04 or 22.04." + echo "[tue-env](bootstrap) Ubuntu ${distrib_release} is unsupported. Please use one of Ubuntu 20.04 or 22.04." return 1 ;; esac # Script variables + local env_url env_targets_url env_dir workspace workspace_dir env_url="https://github.com/tue-robotics/tue-env.git" { [[ -n "${targets_repo}" ]] && env_targets_url="${targets_repo}"; } || env_targets_url="https://github.com/tue-robotics/tue-env-targets.git" [[ -n "${create_virtualenv}" ]] || create_virtualenv="true" env_dir="${HOME}/.tue" - workspace="ros-${TUE_ROS_DISTRO}" - workspace_dir="${HOME}/ros/${TUE_ROS_DISTRO}" + workspace="ros-${tue_ros_distro}" + workspace_dir="${HOME}/ros/${tue_ros_distro}" # Move old environments and installer if [[ -d "${env_dir}" ]] && [[ -z "${CI}" ]] then - FILES=$(find "${env_dir}"/user/envs -maxdepth 1 -type f) + local files date_now + files=$(find "${env_dir}"/user/envs -maxdepth 1 -type f) date_now=$(date +%F_%R) - for env in ${FILES} + for env in ${files} do mv -f "$(cat "${env}")" "$(cat "${env}")"."${date_now}" done @@ -175,19 +195,19 @@ function main then if [[ -n "${BRANCH}" ]] then - echo -e "[tue-env](bootstrap) Cloning tue-env repository with branch: ${BRANCH} at commit: ${COMMIT}" + echo "[tue-env](bootstrap) Cloning tue-env repository with branch: ${BRANCH} at commit: ${COMMIT}" git clone -q --single-branch --branch "${BRANCH}" "${env_url}" "${env_dir}" else - echo -e "[tue-env](bootstrap) Cloning tue-env repository with default branch at commit: ${COMMIT}" + echo "[tue-env](bootstrap) Cloning tue-env repository with default branch at commit: ${COMMIT}" git clone -q --single-branch "${env_url}" "${env_dir}" fi git -C "${env_dir}" reset --hard "${COMMIT}" else - echo -e "[tue-env](bootstrap) Error! CI branch or commit is unset" + echo "[tue-env](bootstrap) Error! CI branch or commit is unset" return 1 fi else - echo -e "[tue-env](bootstrap) Testing Pull Request" + echo "[tue-env](bootstrap) Testing Pull Request" [[ -z "${REF_NAME}" ]] && { echo "[tue-env](bootstrap) Error! Environment variable REF_NAME is not set."; return 1; } git clone -q --depth=10 "${env_url}" "${env_dir}" @@ -196,10 +216,17 @@ function main fi else # Update installer - echo -e "[tue-env](bootstrap) Cloning tue-env repository" + echo "[tue-env](bootstrap) Cloning tue-env repository" git clone "${env_url}" "${env_dir}" fi + # Install Poetry when needed + if [[ "${tue_ppm}" == "poetry" ]] + then + echo "[tue-env](bootstrap) Installing Poetry" + curl -sSL https://install.python-poetry.org | /usr/bin/python3 - + fi + # Source the installer commands # No need to follow to a file which is already checked by CI # shellcheck disable=SC1090 @@ -209,15 +236,19 @@ function main mkdir -p "${workspace_dir}" # Initialize ros environment directory incl. targets - tue-env init "${workspace}" "${workspace_dir}" "--create-virtualenv=${create_virtualenv}" "--targets-url=${env_targets_url}" + tue-env init "${workspace}" "${workspace_dir}" \ + "--create-virtualenv=${create_virtualenv}" \ + "--virtualenv-include-system-site-packages=${virtualenv_include_system_site_packages}" \ + "--targets-url=${env_targets_url}" # Configure environment - tue-env config "${workspace}" set "TUE_ROS_DISTRO" "${TUE_ROS_DISTRO}" - tue-env config "${workspace}" set "TUE_ROS_VERSION" "${TUE_ROS_VERSION}" + tue-env config "${workspace}" set "TUE_ROS_DISTRO" "${tue_ros_distro}" + tue-env config "${workspace}" set "TUE_ROS_VERSION" "${tue_ros_version}" + tue-env config "${workspace}" set "TUE_PPM" "${tue_ppm}" # Add loading of TU/e tools (tue-env, tue-get, etc) to bashrc # shellcheck disable=SC2088 - if ! grep -q "${env_dir}/setup.bash" ~/.bashrc; + if ! grep -q "${env_dir}/setup.bash" ~/.bashrc then echo " # Load TU/e tools diff --git a/installer/check-pip-pkg-installed-version.py b/installer/check-pip-pkg-installed-version.py index bc6789e50..a26693143 100755 --- a/installer/check-pip-pkg-installed-version.py +++ b/installer/check-pip-pkg-installed-version.py @@ -1,21 +1,18 @@ #! /usr/bin/env python3 -import sys +import site +from typing import List from pip._internal.req.constructors import install_req_from_line from pip._internal.utils.virtualenv import running_under_virtualenv -def main() -> int: - if len(sys.argv) < 2: - print("Usage: check-pip-pkg-installed-version.py requirement [requirements]") - return 2 - +def main(req_strs: List[str]) -> int: return_code = 0 pkg_installed = [] try: - for arg in sys.argv[1:]: - req = install_req_from_line(arg) + for req_str in req_strs: + req = install_req_from_line(req_str) req.check_if_exists(not running_under_virtualenv()) @@ -34,4 +31,14 @@ def main() -> int: if __name__ == "__main__": - sys.exit(main()) + import argparse + import sys + + parser = argparse.ArgumentParser( + description="Check if a set of pip package is installed, meeting a requirement string." + ) + parser.add_argument("req_strs", nargs="+") + + args = parser.parse_args() + + sys.exit(main(**vars(args))) diff --git a/installer/github-releases.py b/installer/github-releases.py index eef91be2a..944432924 100755 --- a/installer/github-releases.py +++ b/installer/github-releases.py @@ -34,7 +34,7 @@ def get_release(url, filename, output) -> int: def download_url(url, root, filename=None, md5=None) -> None: - """Download a file from a url and place it in root. + """Download a file from an url and place it in root. Args: url (str): URL to download file from diff --git a/installer/tue-install-impl.bash b/installer/tue-install-impl.bash index 659e81544..639554ac8 100755 --- a/installer/tue-install-impl.bash +++ b/installer/tue-install-impl.bash @@ -168,7 +168,7 @@ function _remove_old_target_dep_recursively old_dep_dep_file="${TUE_INSTALL_DEPENDENCIES_DIR}"/"${old_dep_target}" if [[ -f "${old_dep_dep_file}" ]] then - # Iterate over all depencies of old_dep_target, which is removed. + # Iterate over all dependencies of old_dep_target, which is removed. while read -r dep_of_old_dep do # Actually remove the deps @@ -183,11 +183,11 @@ function _remove_old_target_dep_recursively done < "${old_dep_dep_file}" rm -f "${old_dep_dep_file}" else - tue-install-debug "[remove_old_dep] No depencies file exist for target: ${old_dep_target}" + tue-install-debug "[remove_old_dep] No dependencies file exist for target: ${old_dep_target}" fi tue-install-debug "[remove_old_dep] Uninstalled '${old_dep_target}' as a dependency of '${parent_target}'" - tue-install-info "[remove_old_dep] '${old_dep_target}' has been uninstalled, you can remove it from the workspace or deinstall it in another way" + tue-install-info "[remove_old_dep] '${old_dep_target}' has been uninstalled, you can remove it from the workspace or uninstall it in another way" return ${error_code} } @@ -309,7 +309,7 @@ function tue-install-target [ "$now" == "true" ] && now_cmd="--now" # Do not use 'local cmds=' because it does not preserve command output status ($?) local cmds - if cmds=$("$TUE_INSTALL_SCRIPTS_DIR"/parse_install_yaml.py "$install_file".yaml $now_cmd) + if cmds=$(/usr/bin/python3 "${TUE_INSTALL_SCRIPTS_DIR}"/parse_install_yaml.py "${install_file}".yaml ${now_cmd}) then for cmd in $cmds do @@ -352,7 +352,7 @@ function tue-install-target old_deps_removed=$(comm -23 <(echo "${old_deps}") <(echo "${new_deps}")) if [[ -n ${old_deps_removed} ]] then - tue-install-debug "Following dropped depedencies need to be removed:\n${old_deps_removed}" + tue-install-debug "Following dropped dependencies need to be removed:\n${old_deps_removed}" else tue-install-debug "No dropped dependencies to be removed" fi @@ -361,15 +361,17 @@ function tue-install-target do # Remove this target from dep-on file of dep # When the dep-on file is now empty, remove it - # Recurisvely -> Remove it from the dep-on files of its deps + # Recursively -> Remove it from the dep-on files of its deps tue-install-debug "Going to remove '${dep}' as a dependency" _remove_old_target_dep_recursively "${target}" "${dep}" || tue-install-error "Something went wrong while removing '${dep}' as a dependency" done if [ "$now" == "true" ] then + tue-install-debug "Execution finished with now enabled. Creating state file ${state_file_now}" touch "$state_file_now" else + tue-install-debug "Execution finished. Creating state file ${state_file}" touch "$state_file" fi @@ -459,8 +461,8 @@ function tue-install-git "The problem will probably be solved by resourcing the setup" fi - local targetdir version - targetdir=$(_git_url_to_repos_dir "${repo_pre}") + local target_dir version + target_dir=$(_git_url_to_repos_dir "${repo_pre}") # The shift here is to ensure that all options are explicitly checked as they can be at most 2 # and are optional @@ -476,8 +478,8 @@ function tue-install-git do case $i in --target-dir=* ) - targetdir="${i#*=}" - targetdir="${targetdir/#\~/$HOME}" + target_dir="${i#*=}" + target_dir="${target_dir/#\~/$HOME}" ;; --version=* ) version="${i#*=}" ;; @@ -487,53 +489,54 @@ function tue-install-git done fi - if [[ -z "${targetdir}" ]] + if [[ -z "${target_dir}" ]] then tue-install-error "Target directory path cannot be empty" fi local res - if [ ! -d "$targetdir" ] + if [ ! -d "${target_dir}" ] then - tue-install-debug "git clone --recursive $repo $targetdir" - res=$(git clone --recursive "$repo" "$targetdir" 2>&1) - TUE_INSTALL_GIT_PULL_Q+=:$targetdir: + tue-install-debug "git clone --recursive ${repo} ${target_dir}" + res=$(git clone --recursive "${repo}" "${target_dir}" 2>&1) + TUE_INSTALL_GIT_PULL_Q+=("${target_dir}") else # Check if we have already pulled the repo - if [[ $TUE_INSTALL_GIT_PULL_Q == *:$targetdir:* ]] + if [[ ${TUE_INSTALL_GIT_PULL_Q[*]} == "${target_dir}" ]] then tue-install-debug "Repo previously pulled, skipping" # We have already pulled this repo, skip it res= else + tue-install-debug "Repo not yet pulled, pulling" # Switch url of origin to use https/ssh if different # Get current remote url local current_url - current_url=$(git -C "$targetdir" config --get remote.origin.url) + current_url=$(git -C "${target_dir}" config --get remote.origin.url) # If different, switch url if [ "$current_url" != "$repo" ] then - tue-install-pipe git -C "$targetdir" remote set-url origin "$repo" || tue-install-error "Could not change git url of '$targetdir' to '$repo'" + tue-install-pipe git -C "${target_dir}" remote set-url origin "${repo}" || tue-install-error "Could not change git url of '${target_dir}' to '${repo}'" tue-install-info "URL has switched to $repo" fi - tue-install-debug "git -C $targetdir pull --ff-only --prune" - res=$(git -C "$targetdir" pull --ff-only --prune 2>&1) + tue-install-debug "git -C ${target_dir} pull --ff-only --prune" + res=$(git -C "${target_dir}" pull --ff-only --prune 2>&1) tue-install-debug "res: $res" - TUE_INSTALL_GIT_PULL_Q+=:$targetdir: + TUE_INSTALL_GIT_PULL_Q+=("${target_dir}") local submodule_sync_res submodule_sync_error_code - tue-install-debug "git -C $targetdir submodule sync --recursive" - submodule_sync_res=$(git -C "$targetdir" submodule sync --recursive) + tue-install-debug "git -C ${target_dir} submodule sync --recursive" + submodule_sync_res=$(git -C "${target_dir}" submodule sync --recursive) submodule_sync_error_code=$? tue-install-debug "submodule_sync_res(${submodule_sync_error_code}): ${submodule_sync_res}" [ "${submodule_sync_error_code}" -gt 0 ] && [ -n "${submodule_sync_res}" ] && res="${res:+${res}\n}${submodule_sync_res}" local submodule_res - tue-install-debug "git -C $targetdir submodule update --init --recursive" - submodule_res=$(git -C "$targetdir" submodule update --init --recursive 2>&1) + tue-install-debug "git -C ${target_dir} submodule update --init --recursive" + submodule_res=$(git -C "${target_dir}" submodule update --init --recursive 2>&1) tue-install-debug "submodule_res: $submodule_res" [ -n "$submodule_res" ] && res="${res:+${res}\n}$submodule_res" @@ -546,13 +549,13 @@ function tue-install-git tue-install-debug "Desired version: $version" local _try_branch_res # Will be used in _try_branch_git - local version_cache_file="$TUE_ENV_DIR/.env/version_cache/$targetdir" + local version_cache_file="${TUE_ENV_DIR}/.env/version_cache/${target_dir}" if [ -n "$version" ] then mkdir -p "$(dirname "$version_cache_file")" echo "$version" > "$version_cache_file" _try_branch_res="" - _try_branch_git "$targetdir" "$version" + _try_branch_git "${target_dir}" "${version}" [ -n "$_try_branch_res" ] && res="${res:+${res}\n}$_try_branch_res" else rm "$version_cache_file" 2>/dev/null @@ -564,7 +567,7 @@ function tue-install-git do tue-install-debug "Parsed branch '${branch}'" _try_branch_res="" - _try_branch_git "${targetdir}" "${branch}" + _try_branch_git "${target_dir}" "${branch}" _try_branch_error_code=$? [ -n "${_try_branch_res}" ] && res="${res:+${res}\n}${_try_branch_res}" [ "${_try_branch_error_code}" -eq 0 ] && break @@ -1060,23 +1063,40 @@ function tue-install-ppa-now function _tue-install-pip { - local pv name + local pv pv=$1 shift tue-install-debug "tue-install-pip${pv} $*" - name="$*" - name="${name// /^}" - if [ -z "$1" ] + local pkg_req pkg_reqs python_site + { [[ -n ${VIRTUAL_ENV} ]] && python_site="venv"; } || python_site="user" + if [[ -n $1 ]] then - tue-install-error "Invalid tue-install-pip${pv} call: needs package as argument." + for i in "$@" + do + case $i in + --python-site=* ) + python_site="${i#*=}" ;; + --* ) + tue-install-error "tue-install-pip${pv}-now: Unknown input variable ${i}" ;; + * ) + pkg_reqs+=("$i") ;; + esac + done + fi + pkg_req=${pkg_reqs[*]} + pkg_req=${pkg_req// /^} + + if [[ -n ${VIRTUAL_ENV} && ${TUE_PPM} == "poetry" && ${python_site} == "venv" ]] + then + tue-install-error "tue-install-pip${pv}: You are using poetry as package manager, but you are trying to install '${pkg_req}' in the virtual environment site-packages. This is not allowed as poetry manages the packages in the virtual environment." fi - tue-install-debug "Adding $name to pip${pv} list" + tue-install-debug "Adding ${pkg_req} to pip${pv} ${python_site} list" local list - list=TUE_INSTALL_PIP"${pv}"S + list=TUE_INSTALL_"${python_site^^}"_PIP"${pv}"S # shellcheck disable=SC2140 - declare -g "$list"="$name ${!list}" + declare -g "${list}"="${pkg_req} ${!list}" } # Needed for backward compatibility @@ -1092,6 +1112,48 @@ function tue-install-pip3 # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +function _handle_python_sites +{ + local pv + pv=$1 + shift + # requires the parent function to have declared the following local variables: + # python_exec, site_arg, sudo_cmd, user_arg + tue-install-debug "_handle_python${pv}_sites $*" + + local python_site + python_site=$1 + + case ${python_site} in + "system" ) + python_exec=/usr/bin/python"${pv}" + site_arg="-s" + sudo_cmd="sudo -H" + user_arg="" + ;; + "user" ) + python_exec=/usr/bin/python"${pv}" + site_arg="" + sudo_cmd="" + user_arg="--user" + ;; + "venv" ) + python_exec=python"${pv}" + site_arg="-s" + sudo_cmd="" + user_arg="" + ;; + * ) + tue-install-error "_handle_python${pv}_sites: Unknown input python_site: ${python_site}" ;; + esac + + tue-install-debug "python_site: ${python_site}" + tue-install-debug "python_exec: ${python_exec}" + tue-install-debug "site_arg: ${site_arg}" + tue-install-debug "sudo_cmd: ${sudo_cmd}" + tue-install-debug "user_arg: ${user_arg}" +} + function _tue-install-pip-now { local pv @@ -1104,25 +1166,41 @@ function _tue-install-pip-now tue-install-error "Invalid tue-install-pip${pv}-now call: needs package as argument." fi - local user_arg - [[ -z "${VIRTUAL_ENV}" ]] && user_arg="--user" + local pips python_site + { [[ -n ${VIRTUAL_ENV} ]] && python_site="venv"; } || python_site="user" + if [[ -n $1 ]] + then + for i in "$@" + do + case $i in + --python-site=* ) + python_site="${i#*=}" ;; + --* ) + tue-install-error "tue-install-pip${pv}-now: Unknown input variable ${i}" ;; + * ) + pips+=("$i") ;; + esac + done + fi + + local python_exec site_arg sudo_cmd user_arg + _handle_python_sites "${pv}" "${python_site}" # Make sure pip is up-to-date before checking version and installing local pip_version desired_pip_version - pip_version=$(python"${pv}" -m pip --version | awk '{print $2}') - desired_pip_version="22" + pip_version=$(${python_exec} ${site_arg:+${site_arg} }-m pip --version | awk '{print $2}') + desired_pip_version="24" if version_gt "$desired_pip_version" "$pip_version" then tue-install-debug "pip${pv} not yet version >=$desired_pip_version, but $pip_version" - tue-install-pipe python"${pv}" -m pip install ${user_arg} --upgrade pip + tue-install-pipe ${sudo_cmd:+${sudo_cmd} }"${python_exec}" ${site_arg:+${site_arg} }-m pip install ${user_arg:+${user_arg} }--upgrade pip hash -r else tue-install-debug "Already pip${pv}>=$desired_pip_version" fi local pips_to_check pips_to_check_w_options pips_to_install pips_to_install_w_options git_pips_to_install - # shellcheck disable=SC2048 - for pkg in $* + for pkg in "${pips[@]}" do if [[ "$pkg" == "git+"* ]] then @@ -1142,7 +1220,7 @@ function _tue-install-pip-now then local indexes_to_install # shellcheck disable=SC2086 - _tue-install-pip-check "$pv" $pips_to_check + _tue-install-pip-check "${pv}" --python-site=${python_site} ${pips_to_check} read -r -a pips_to_check <<< "$pips_to_check" @@ -1164,7 +1242,7 @@ function _tue-install-pip-now pips_to_check_options_removed="$pips_to_check_options_removed ${pkg_split[0]}" done # shellcheck disable=SC2086 - _tue-install-pip-check "$pv" $pips_to_check_options_removed + _tue-install-pip-check "${pv}" --python-site=${python_site} ${pips_to_check_options_removed} read -r -a pips_to_check_w_options <<< "$pips_to_check_w_options" @@ -1180,7 +1258,7 @@ function _tue-install-pip-now if [ -n "$pips_to_install" ] then # shellcheck disable=SC2048,SC2086 - tue-install-pipe python"${pv}" -m pip install ${user_arg} $pips_to_install <<< yes || tue-install-error "An error occurred while installing pip${pv} packages." + tue-install-pipe ${sudo_cmd:+${sudo_cmd} }"${python_exec}" ${site_arg:+${site_arg} }-m pip install ${user_arg:+${user_arg} }$pips_to_install <<< yes || tue-install-error "An error occurred while installing pip${pv} packages." fi if [ -n "$pips_to_install_w_options" ] @@ -1188,7 +1266,7 @@ function _tue-install-pip-now for pkg in $pips_to_install_w_options do # shellcheck disable=SC2048,SC2086 - tue-install-pipe python"${pv}" -m pip install ${user_arg} ${pkg//^/ } <<< yes || tue-install-error "An error occurred while installing pip${pv} packages with options." + tue-install-pipe ${sudo_cmd:+${sudo_cmd} }"${python_exec}" ${site_arg:+${site_arg} }-m pip install ${user_arg:+${user_arg} }${pkg//^/ } <<< yes || tue-install-error "An error occurred while installing pip${pv} packages with options." done fi @@ -1197,7 +1275,7 @@ function _tue-install-pip-now for pkg in $git_pips_to_install do # shellcheck disable=SC2048,SC2086 - tue-install-pipe python"${pv}" -m pip install ${user_arg} ${pkg} <<< yes || tue-install-error "An error occurred while installing pip${pv} git packages." + tue-install-pipe ${sudo_cmd:+${sudo_cmd} }"${python_exec}"${site_arg:+${site_arg} }-m pip install ${user_arg:+${user_arg} }${pkg} <<< yes || tue-install-error "An error occurred while installing pip${pv} git packages." done fi } @@ -1209,12 +1287,35 @@ function _tue-install-pip-check pv=$1 shift - local pips_to_check installed_versions - pips_to_check=("$@") + local installed_versions python_site pips_to_check + if [[ -n $1 ]] + then + for i in "$@" + do + case $i in + --python-site=* ) + python_site="${i#*=}" ;; + --* ) + tue-install-error "Unknown input variable ${i}" ;; + * ) + pips_to_check+=("$i") ;; + esac + done + fi + + if [[ -z "${VIRTUAL_ENV}" && "${python_site}" == "venv" ]] + then + tue-install-error "Trying to check pip packages in a virtualenv, but no virtualenv is activated" + fi + + # Set by _handle_python_sites + local python_exec sudo_cmd site_arg user_arg + _handle_python_sites "${pv}" "${python_site}" + if [ ${#pips_to_check[@]} -gt 0 ] then local error_code - installed_versions=$(python"${pv}" "$TUE_INSTALL_SCRIPTS_DIR"/check-pip-pkg-installed-version.py "${pips_to_check[@]}") + installed_versions=$(${python_exec} ${site_arg:+${site_arg} }"${TUE_INSTALL_SCRIPTS_DIR}"/check-pip-pkg-installed-version.py "${pips_to_check[@]}") error_code=$? if [ "$error_code" -gt 1 ] then @@ -1256,6 +1357,38 @@ function tue-install-pip3-now # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +function tue-install-poetry-now +{ + tue-install-debug "tue-install-poetry-now $*" + + if [[ -z "$1" ]] + then + tue-install-error "Invalid tue-install-poetry-now call: needs project as argument." + fi + + local poetry_self_updated_file + poetry_self_updated_file=/tmp/tue_get_poetry_self_updated + if [[ ! -f ${poetry_self_updated_file} ]] + then + tue-install-pipe poetry self update + touch ${poetry_self_updated_file} + fi + + local return_code sync_arg venv_include_system_site_packages + venv_include_system_site_packages=$(python3 -c "from pip._internal.utils.virtualenv import virtualenv_no_global; print(str(not virtualenv_no_global()).lower())") + return_code=$? + [[ "${return_code}" -ne 0 ]] && tue-install-warning "An error occurred while checking if the virtual environment includes the system site packages. Assuming it does." && venv_include_system_site_packages="true" + [[ "${venv_include_system_site_packages}" == "false" ]] && sync_arg="--sync" + + local project_dir + project_dir=${TUE_WS_DIR}/src/${TUE_POETRY_PROJECT//-/_} + [[ ! -d ${project_dir} ]] && tue-install-error "tue-install-poetry-now: The project ${TUE_POETRY_PROJECT} does not exist in the workspace directory $(dirname "${project_dir}")" + + tue-install-pipe poetry -C "${project_dir}" install ${sync_arg:+${sync_arg} } ${TUE_GET_POETRY_INSTALL_ARGS:+${TUE_GET_POETRY_INSTALL_ARGS} } || tue-install-error "An error occurred while installing the project ${TUE_POETRY_PROJECT} with poetry." +} + +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # + function tue-install-snap { tue-install-debug "tue-install-snap $*" @@ -1372,7 +1505,7 @@ function tue-install-dpkg tue-install-error "Invalid tue-install-dpkg call: needs package as argument." fi tue-install-pipe sudo dpkg --install "$1" - tue-install-pipe sudo apt-get --fix-broken --assume-yes -q install || tue-install-error "An error occured while fixing dpkg install" + tue-install-pipe sudo apt-get --fix-broken --assume-yes -q install || tue-install-error "An error occurred while fixing dpkg install" } # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # @@ -1392,11 +1525,7 @@ function tue-install-ros local ros_pkg_name ros_pkg_name=${TUE_INSTALL_CURRENT_TARGET#ros-} - if [[ $ros_pkg_name == *-* ]] - then - tue-install-error "A ROS package cannot contain dashes (${ros_pkg_name}), make sure the package is named '${ros_pkg_name//-/_}' and rename the target to 'ros-${ros_pkg_name//-/_}'" - return 1 - fi + [[ $ros_pkg_name == *-* ]] && tue-install-error "A ROS package cannot contain dashes (${ros_pkg_name}), make sure the package is named '${ros_pkg_name//-/_}' and rename the target to 'ros-${ros_pkg_name//-/_}'" # First of all, make sure ROS itself is installed tue-install-target ros"${TUE_ROS_VERSION}" || tue-install-error "Failed to install target 'ros${TUE_ROS_VERSION}'" @@ -1420,20 +1549,13 @@ function tue-install-ros do case $i in --target-dir=* ) - repos_dir="${i#*=}" - ;; - + repos_dir="${i#*=}" ;; --version=* ) - version="${i#*=}" - ;; - + version="${i#*=}" ;; --sub-dir=* ) - sub_dir="${i#*=}" - ;; - + sub_dir="${i#*=}" ;; * ) - tue-install-error "Unknown input variable ${i}" - ;; + tue-install-error "Unknown input variable ${i}" ;; esac done fi @@ -1492,10 +1614,11 @@ function tue-install-ros local pkg_xml="$ros_pkg_dir"/package.xml if [ -f "$pkg_xml" ] then - # Catkin + # catkin/ament/colcon tue-install-debug "Parsing $pkg_xml" + local deps - deps=$("$TUE_INSTALL_SCRIPTS_DIR"/parse_package_xml.py "$pkg_xml") + deps=$(/usr/bin/python3 "${TUE_INSTALL_SCRIPTS_DIR}"/parse_package_xml.py "${pkg_xml}") tue-install-debug "Parsed package.xml\n$deps" for dep in $deps @@ -1562,38 +1685,31 @@ while test $# -gt 0 do case "$1" in --debug) - DEBUG="true" - ;; + DEBUG="true" ;; --no-ros-deps) - export TUE_INSTALL_SKIP_ROS_DEPS="all" - ;; + export TUE_INSTALL_SKIP_ROS_DEPS="all" ;; --doc-depend) [[ "$TUE_INSTALL_SKIP_ROS_DEPS" == "all" ]] && export TUE_INSTALL_SKIP_ROS_DEPS="normal" export TUE_INSTALL_DOC_DEPEND="true" ;; --no-doc-depend) - export TUE_INSTALL_DOC_DEPEND="false" - ;; + export TUE_INSTALL_DOC_DEPEND="false" ;; --test-depend) [[ "$TUE_INSTALL_SKIP_ROS_DEPS" == "all" ]] && export TUE_INSTALL_SKIP_ROS_DEPS="normal" export TUE_INSTALL_TEST_DEPEND="true" ;; --no-test-depend) - export TUE_INSTALL_TEST_DEPEND="false" - ;; + export TUE_INSTALL_TEST_DEPEND="false" ;; --branch*) - echo "Usage of --branch is deprecated, switch to --try-branch" - ;;& + echo "Usage of --branch is deprecated, switch to --try-branch" ;;& --try-branch* | --branch*) # shellcheck disable=SC2001 BRANCH="$(echo "$1" | sed -e 's/^[^=]*=//g')${BRANCH:+ ${BRANCH}}" # Reverse order, so we try last one first ;; --*) - echo "unknown option $1" - ;; + echo "unknown option $1" ;; *) - targets="${targets:+${targets} }$1" - ;; + targets="${targets:+${targets} }$1" ;; esac shift done @@ -1625,7 +1741,9 @@ TUE_INSTALL_GIT_PULL_Q=() TUE_INSTALL_SYSTEMS= TUE_INSTALL_PPA= -TUE_INSTALL_PIP3S= +TUE_INSTALL_SYSTEM_PIP3S= +TUE_INSTALL_USER_PIP3S= +TUE_INSTALL_VENV_PIP3S= TUE_INSTALL_SNAPS= TUE_INSTALL_GEMS= @@ -1635,7 +1753,8 @@ TUE_INSTALL_INFOS= # Make sure tools used by this installer are installed tue-install-system-now curl git jq python-is-python3 python3-pip wget -tue-install-pip3-now catkin-pkg PyYAML +# Install in user or system site-packages +tue-install-pip3-now --python-site="user" catkin-pkg 'PyYAML>=6' # Handling of targets @@ -1643,23 +1762,33 @@ if [[ -z "${targets// }" ]] #If only whitespace then # If no targets are provided, update all installed targets targets=$(ls "$TUE_INSTALL_INSTALLED_DIR") + tue-install-debug "No targets provided, updating all installed targets:\n${targets}" else - raw_targets=$targets - targets="" + raw_targets=${targets} + tue-install-debug "Following raw target names provided: ${raw_targets}" + targets_arr=() for target in $raw_targets do - resolved_targets="$(find "$TUE_INSTALL_TARGETS_DIR" -maxdepth 1 -name "$target" -type d -printf "%P ")" - if [ -z "$resolved_targets" ] # So the missing target is handled by _missing_targets_check + resolved_targets="$(find "${TUE_INSTALL_TARGETS_DIR}" -maxdepth 1 -name "${target}" -type d -printf "%P ")" + if [[ -z "${resolved_targets}" ]] # So the missing target is handled by _missing_targets_check then - resolved_targets="$target" + tue-install-debug "Target regex didn't match any target, just adding the raw target name '${target}', so it can be handled by _missing_targets_check" + resolved_targets="${target}" + else + tue-install-debug "Target regex '${target}' matched the following targets: ${resolved_targets}" fi - targets="${targets:+$targets }$resolved_targets" + + # Unpack the resolved targets + for resolved_target in ${resolved_targets} + do + targets_arr+=("${resolved_target}") + done done + targets=${targets_arr[*]} fi - # Check if all installed targets exist in the targets repo -_missing_targets_check "$targets" +_missing_targets_check "${targets}" for target in $targets do @@ -1673,7 +1802,7 @@ do tue-install-debug "[$target] marked as installed after a successful install" touch "$TUE_INSTALL_INSTALLED_DIR"/"$target" else - tue-install-debug "[$target] succesfully updated" + tue-install-debug "[$target] successfully updated" fi done @@ -1714,16 +1843,44 @@ then tue-install-system-now "$TUE_INSTALL_SYSTEMS" fi - # Installing all python3 (pip3) targets, which are collected during the install -if [ -n "$TUE_INSTALL_PIP3S" ] +if [[ -n "${TUE_INSTALL_SYSTEM_PIP3S}" ]] +then + TUE_INSTALL_CURRENT_TARGET="PIP3(SYSTEM)" + + # No need to check for poetry, as you can always install in the system site-packages + tue-install-debug "calling: tue-install-pip3-now --python-site=\"system\" ${TUE_INSTALL_SYSTEM_PIP3S}" + tue-install-pip3-now --python-site="system" "${TUE_INSTALL_SYSTEM_PIP3S}" +fi + +if [[ -n "${TUE_INSTALL_USER_PIP3S}" ]] then - TUE_INSTALL_CURRENT_TARGET="PIP3" + TUE_INSTALL_CURRENT_TARGET="PIP3(USER)" - tue-install-debug "calling: tue-install-pip3-now $TUE_INSTALL_PIP3S" - tue-install-pip3-now "$TUE_INSTALL_PIP3S" + # No need to check for poetry, as you can always install in the user site-packages + tue-install-debug "calling: tue-install-pip3-now --python-site=\"user\" ${TUE_INSTALL_USER_PIP3S}" + tue-install-pip3-now --python-site="user" "${TUE_INSTALL_USER_PIP3S}" fi +if [[ -n "${TUE_INSTALL_VENV_PIP3S}" ]] +then + TUE_INSTALL_CURRENT_TARGET="PIP3(VENV)" + + if [[ -z ${VIRTUAL_ENV} ]] + then + # This should not be reachable, but just in case + tue-install-error "You can't install pip3 packages in the virtualenv, when no virtualenv is activated" + fi + + if [[ ${TUE_PPM} == "poetry" ]] + then + # This should not be reachable, but just in case + tue-install-error "You can't install pip3 packages in the virtualenv, when using poetry as python package manager" + fi + + tue-install-debug "calling: tue-install-pip3-now --python-site=\"venv\" ${TUE_INSTALL_VENV_PIP3S}" + tue-install-pip3-now --python-site="venv" "${TUE_INSTALL_VENV_PIP3S}" +fi # Installing all snap targets, which are collected during the install if [ -n "$TUE_INSTALL_SNAPS" ] @@ -1743,8 +1900,30 @@ then tue-install-gem-now "$TUE_INSTALL_GEMS" fi +if [[ -n ${TUE_POETRY_PROJECT} ]] +then + TUE_INSTALL_CURRENT_TARGET="POETRY" + + poetry_project_target="ros-${TUE_POETRY_PROJECT//-/_}" + state_file=${TUE_INSTALL_STATE_DIR}/${poetry_project_target} + state_file_now=${state_file}-now + if [[ ! -f ${state_file} && ! -f ${state_file_now} ]] + then + tue-install-echo "The target (${poetry_project_target}) of the poetry root project (${TUE_POETRY_PROJECT}) is not processed in this run of tue-get. Therefore, the poetry root project will not be installed." + else + if [[ ${TUE_PPM} != "poetry" ]] + then + tue-install-error "You can't install poetry project '${TUE_POETRY_PROJECT}' when not using poetry as python package manager" + fi + + tue-install-debug "calling: tue-install-poetry-now ${TUE_POETRY_PROJECT}" + tue-install-poetry-now "${TUE_POETRY_PROJECT}" + fi + +fi + TUE_INSTALL_CURRENT_TARGET="main-loop" -tue-install-echo "Installer completed succesfully" +tue-install-echo "Installer completed successfully" return 0 diff --git a/setup.bash b/setup.bash index ad81bd0b4..5c77e70ca 100644 --- a/setup.bash +++ b/setup.bash @@ -58,6 +58,16 @@ then source "$TUE_ENV_DIR"/.env/setup/user_setup.bash fi +if [[ ${TUE_PPM} == "poetry" ]] +then + if ! command -v poetry &> /dev/null + then + echo "[tue] Loading an environment with Poetry, but Poetry is not installed." + echo -e "[tue] Please install Poetry and try again.\n\ncurl -sSL https://install.python-poetry.org | /usr/bin/python3 -\n" + return 1 + fi +fi + # ----------------------------------------- # Load all the bash functions # shellcheck disable=SC1091 diff --git a/setup/tue-env.bash b/setup/tue-env.bash index 08cf85eff..622a86f93 100644 --- a/setup/tue-env.bash +++ b/setup/tue-env.bash @@ -38,9 +38,10 @@ function tue-env # Make sure the correct directories are there mkdir -p "$TUE_DIR"/user/envs - local create_venv dir env_name targets_url show_help + local create_venv dir env_name show_help targets_url venv_include_system_site_packages create_venv="false" show_help="false" + venv_include_system_site_packages="false" if [[ $cmd == "init" ]] then @@ -55,6 +56,8 @@ function tue-env targets_url="${i#*=}" ;; --create-virtualenv=* ) create_venv="${i#*=}" ;; + --virtualenv-include-system-site-packages=* ) + venv_include_system_site_packages="${i#*=}" ;; --help ) show_help="true" ;; * ) @@ -105,7 +108,7 @@ function tue-env if [[ "${create_venv}" == "true" ]] then - tue-env init-venv "${env_name}" + tue-env init-venv "${env_name}" "${venv_include_system_site_packages}" fi elif [[ $cmd == "remove" ]] @@ -281,33 +284,35 @@ Environment directory '${dir}' didn't exist (anymore)""" elif [[ $cmd == "init-venv" ]] then - local env - env=$1 - [ -n "${env}" ] || env=${TUE_ENV} - - if [[ -z "${env}" ]] + if [ -z "$1" ] || { [ -z "${TUE_ENV}" ] && [ -z "$2" ]; } then - echo "[tue-env](init-venv) no environment set or provided" - echo "Usage: tue-env init-venv [ NAME ]" + echo "Usage: tue-env init-venv [ENVIRONMENT] INCLUDE_SYSTEM_SITE_PACKAGES" return 1 fi - python3 -c "import virtualenv" 2>/dev/null || - { echo -e "[tue-env](init-venv) 'virtualenv' module is not found. Make sure you install it 'sudo apt-get install python3-virtualenv'"; return 1; } + local env venv_include_system_site_packages + env=$1 + venv_include_system_site_packages=$2 + if [ -z "${venv_include_system_site_packages}" ] + then + env=${TUE_ENV} + if [ -z "${env}" ] + then + # This shouldn't be possible logical, should have exited after printing usage + echo "[tue-env](init-venv) no environment set or provided" + return 1 + fi + venv_include_system_site_packages=$1 + fi + + /usr/bin/python3 -c "import virtualenv" 2>/dev/null || + { echo "[tue-env](init-venv) 'virtualenv' module is not found. Make sure you install it 'sudo apt-get install python3-virtualenv'"; return 1; } local tue_env_dir tue_env_dir=$(cat "${TUE_DIR}"/user/envs/"${env}") local venv_dir venv_dir=${tue_env_dir}/.venv/${env} - if [ -d "$tue_env_targets_dir" ] - then - local targets_dir_moved - targets_dir_moved=$tue_env_targets_dir.$(date +%F_%R) - mv -f "$tue_env_targets_dir" "$targets_dir_moved" - echo "[tue-env] Moved old targets of environment '$env' to $targets_dir_moved" - fi - if [[ -d "${venv_dir}" ]] then local venv_dir_moved @@ -322,7 +327,12 @@ Environment directory '${dir}' didn't exist (anymore)""" echo "Don't use it anymore as its old path is hardcoded in the virtualenv" fi - python3 -m virtualenv "${venv_dir}" -q --system-site-packages --symlinks 2>/dev/null + local system_site_args + if [[ "${venv_include_system_site_packages}" == "true" ]] + then + system_site_args="--system-site-packages" + fi + /usr/bin/python3 -m virtualenv "${venv_dir}" -q "${system_site_args}" --symlinks 2>/dev/null echo "[tue-env] Initialized virtualenv of environment '${env}'" if [ "${env}" == "${TUE_ENV}" ] diff --git a/setup/tue-functions.bash b/setup/tue-functions.bash index 4b31c74ac..694d2b33c 100644 --- a/setup/tue-functions.bash +++ b/setup/tue-functions.bash @@ -18,7 +18,7 @@ export TUE_RELEASE_DIR # HELPER FUNCTIONS # ---------------------------------------------------------------------------------------------------- -function _list_subdirs +function _list_sub_dirs { fs=$(ls "$1") for f in $fs @@ -84,11 +84,13 @@ function _tue-git-get-default-branch export -f _tue-git-get-default-branch +# shellcheck disable=SC2120 function __tue-git-checkout-default-branch { - local default_branch - default_branch=$(_tue-git-get-default-branch "$1") - _git_remote_checkout "$1" origin "$default_branch" + local default_branch repo + repo=$1 + default_branch=$(_tue-git-get-default-branch "${repo}") + _git_remote_checkout "${repo}" origin "${default_branch}" } export -f __tue-git-checkout-default-branch @@ -158,7 +160,7 @@ function _tue-git-clean-local git branch -d "$stale_branch" > /dev/null 2>&1 error_code=$? - # If an error occured in safe deletion of a stale branch, add it to the + # If an error occurred in safe deletion of a stale branch, add it to the # list of unmerged stale branches which are to be forcefully removed # upon confirmation by the user if [ ! $error_code -eq 0 ] @@ -412,7 +414,7 @@ export -f tue-git-deep-fetch # TUE-MAKE # ---------------------------------------------------------------------------------------------------- -# catkin build/test worflow +# catkin build/test workflow # catkin build -DCATKIN_ENABLE_TESTING=OFF # As we don't install test dependencies by default. Test shouldn't be build # When you want to test, make sure the test dependencies are installed # catkin build -DCATKIN_ENABLE_TESTING=ON # This will trigger cmake and will create the targets and build the tests @@ -437,12 +439,12 @@ function tue-make fi case $build_tool in 'catkin build') - /usr/bin/python3 "$(command -v catkin)" build --workspace "$TUE_SYSTEM_DIR" "$@" + (deactivate &>/dev/null; /usr/bin/python3 "$(command -v catkin)" build --workspace "${TUE_SYSTEM_DIR}" "$@") return $? ;; '') - /usr/bin/python3 "$(command -v catkin)" config --init --mkdirs --workspace "$TUE_SYSTEM_DIR" --extend /opt/ros/"$TUE_ROS_DISTRO" -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCATKIN_ENABLE_TESTING=OFF - /usr/bin/python3 "$(command -v catkin)" build --workspace "$TUE_SYSTEM_DIR" "$@" + (deactivate &>/dev/null; /usr/bin/python3 "$(command -v catkin)" config --init --mkdirs --workspace "${TUE_SYSTEM_DIR}" --extend /opt/ros/"${TUE_ROS_DISTRO}" -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCATKIN_ENABLE_TESTING=OFF) + (deactivate &>/dev/null; /usr/bin/python3 "$(command -v catkin)" build --workspace "${TUE_SYSTEM_DIR}" "$@") touch "$TUE_SYSTEM_DIR"/devel/.catkin # hack to allow overlaying to this ws while being empty ;; *) @@ -457,10 +459,10 @@ function tue-make # Disable symlink install for production if [ "${CI_INSTALL}" == "true" ] then - rm -rf "$TUE_SYSTEM_DIR"/install - python3 -m colcon --log-base "$TUE_SYSTEM_DIR"/log build --base-paths "$TUE_SYSTEM_DIR"/src --build-base "$TUE_SYSTEM_DIR"/build --install-base "$TUE_SYSTEM_DIR"/install "$@" + rm -rf "${TUE_SYSTEM_DIR}"/install + (deactivate &>/dev/null; /usr/bin/python3 -m colcon --log-base "${TUE_SYSTEM_DIR}"/log build --base-paths "${TUE_SYSTEM_DIR}"/src --build-base "${TUE_SYSTEM_DIR}"/build --install-base "${TUE_SYSTEM_DIR}"/install "$@") else - python3 -m colcon --log-base "$TUE_SYSTEM_DIR"/log build --merge-install --symlink-install --base-paths "$TUE_SYSTEM_DIR"/src --build-base "$TUE_SYSTEM_DIR"/build --install-base "$TUE_SYSTEM_DIR"/install "$@" + (deactivate &>/dev/null; /usr/bin/python3 -m colcon --log-base "${TUE_SYSTEM_DIR}"/log build --merge-install --symlink-install --base-paths "${TUE_SYSTEM_DIR}"/src --build-base "${TUE_SYSTEM_DIR}"/build --install-base "${TUE_SYSTEM_DIR}"/install "$@") fi return $? else @@ -488,7 +490,7 @@ function tue-make-test fi case ${build_tool} in 'catkin build') - /usr/bin/python3 "$(command -v catkin)" test --workspace "${TUE_SYSTEM_DIR}" "$@" + (deactivate &>/dev/null; /usr/bin/python3 "$(command -v catkin)" test --workspace "${TUE_SYSTEM_DIR}" "$@") return $? ;; '') @@ -517,9 +519,9 @@ function tue-make-test # Disable symlink install for production if [ "${CI_INSTALL}" == "true" ] then - python3 -m colcon --log-base "${TUE_SYSTEM_DIR}"/log test --base-paths "${TUE_SYSTEM_DIR}"/src --build-base "${TUE_SYSTEM_DIR}"/build --install-base "${TUE_SYSTEM_DIR}"/install --executor sequential --event-handlers console_cohesion+ "$@" + (deactivate &>/dev/null; /usr/bin/python3 -m colcon --log-base "${TUE_SYSTEM_DIR}"/log test --base-paths "${TUE_SYSTEM_DIR}"/src --build-base "${TUE_SYSTEM_DIR}"/build --install-base "${TUE_SYSTEM_DIR}"/install --executor sequential --event-handlers console_cohesion+ "$@") else - python3 -m colcon --log-base "${TUE_SYSTEM_DIR}"/log test --merge-install --base-paths "${TUE_SYSTEM_DIR}"/src --build-base "${TUE_SYSTEM_DIR}"/build --install-base "${TUE_SYSTEM_DIR}"/install --executor sequential --event-handlers console_cohesion+ "$@" + (deactivate &>/dev/null; /usr/bin/python3 -m colcon --log-base "${TUE_SYSTEM_DIR}"/log test --merge-install --base-paths "${TUE_SYSTEM_DIR}"/src --build-base "${TUE_SYSTEM_DIR}"/build --install-base "${TUE_SYSTEM_DIR}"/install --executor sequential --event-handlers console_cohesion+ "$@") fi return $? else @@ -547,7 +549,7 @@ function tue-make-test-result fi case $build_tool in 'catkin build') - python3 "$(command -v catkin)" test_results "${TUE_SYSTEM_DIR}"/build "$@" + (deactivate &>/dev/null; /usr/bin/python3 "$(command -v catkin)" test_results "${TUE_SYSTEM_DIR}"/build "$@") return $? ;; '') @@ -573,7 +575,7 @@ function tue-make-test-result return 1 fi - python3 -m colcon --log-base "${TUE_SYSTEM_DIR}"/log test-result --test-result-base "${TUE_SYSTEM_DIR}"/build "$@" + (deactivate &>/dev/null; /usr/bin/python3 -m colcon --log-base "${TUE_SYSTEM_DIR}"/log test-result --test-result-base "${TUE_SYSTEM_DIR}"/build "$@") return $? else echo -e "\e[31;1mError! ROS_VERSION '${TUE_ROS_VERSION}' is not supported by tue-env.\e[0m" @@ -589,7 +591,7 @@ function _tue-make local options [[ "${TUE_ROS_VERSION}" -eq 2 ]] && options="${options} --packages-select" - mapfile -t COMPREPLY < <(compgen -W "$(_list_subdirs "${TUE_SYSTEM_DIR}"/src) ${options}" -- "${cur}") + mapfile -t COMPREPLY < <(compgen -W "$(_list_sub_dirs "${TUE_SYSTEM_DIR}"/src) ${options}" -- "${cur}") } complete -F _tue-make tue-make @@ -761,18 +763,18 @@ function tue-revert new_hash=$(git -C "$pkg_dir" rev-list -1 --before="$human_time" "$branch") current_hash=$(git -C "$pkg_dir" rev-parse HEAD) - local newtime + local new_time if git -C "$pkg_dir" diff -s --exit-code "$new_hash" "$current_hash" then - newtime=$(git -C "$pkg_dir" show -s --format=%ci) - printf "\e[0;36m%-20s\e[0m %-15s \e[1m%s\e[0m %s\n" "$branch is fine" "$new_hash" "$newtime" "$pkg" + new_time=$(git -C "${pkg_dir}" show -s --format=%ci) + printf "\e[0;36m%-20s\e[0m %-15s \e[1m%s\e[0m %s\n" "${branch} is fine" "${new_hash}" "${new_time}" "${pkg}" else - local newbranch + local new_branch git -C "$pkg_dir" checkout -q "$new_hash" -- - newbranch=$(git -C "$pkg_dir" rev-parse --abbrev-ref HEAD 2>&1) - newtime=$(git -C "$pkg_dir" show -s --format=%ci) + new_branch=$(git -C "${pkg_dir}" rev-parse --abbrev-ref HEAD 2>&1) + new_time=$(git -C "${pkg_dir}" show -s --format=%ci) echo "$branch" > "$pkg_dir/.do_not_commit_this" - printf "\e[0;36m%-20s\e[0m %-15s \e[1m%s\e[0m %s\n" "$newbranch based on $branch" "$new_hash" "$newtime" "$pkg" + printf "\e[0;36m%-20s\e[0m %-15s \e[1m%s\e[0m %s\n" "${new_branch} based on ${branch}" "${new_hash}" "${new_time}" "${pkg}" fi else echo "Package $pkg could not be reverted, current state: $branch" @@ -859,10 +861,10 @@ function _remove_recursively fi fi - # If no packages depend on this target, remove it and its dependcies. + # If no packages depend on this target, remove it and its dependencies. if [ -f "$tue_dependencies_dir"/"$target" ] then - # Iterate over all depencies of target, which is removed. + # Iterate over all dependencies of target, which is removed. while read -r dep do # Target is removed, so remove yourself from depend-on files of deps @@ -894,7 +896,7 @@ function _remove_recursively done < "$tue_dependencies_dir"/"$target" rm -f "$tue_dependencies_dir"/"$target" else - echo "[tue-get] No depencies file exist for target: $target" + echo "[tue-get] No dependencies file exist for target: $target" fi echo "[tue-get] Fully uninstalled $target and its dependencies" @@ -922,10 +924,10 @@ function tue-get Possible options: --debug - Shows more debugging information --no-ros-deps - Do not install ROS dependencies (Breaks the dependency tree, not all setup files will be sourced) - --doc-depend - Do install doc dependencies, overules config and --no-ros-deps - --no-doc-depend - Do not install doc dependencies, overules config - --test-depend - Do install test dependencies, overules config and --no-ros-deps - --no-test-depend - Do not install test dependencies, overules config + --doc-depend - Do install doc dependencies, overrules config and --no-ros-deps + --no-doc-depend - Do not install doc dependencies, overrules config + --test-depend - Do install test dependencies, overrules config and --no-ros-deps + --no-test-depend - Do not install test dependencies, overrules config --try-branch=name - Try to checkout the branch (or tag) 'name'. This argument can be specified multiple times and all the --try-branch arguments are processed in the reverse order of their declaration, with the last one being the first. 'name' must only be an one word value, not a list or any @@ -946,7 +948,7 @@ function tue-get cmd=$1 shift - #Create btrfs snapshot if possible and usefull: + #Create btrfs snapshot if possible and useful: if [[ -n "$BTRFS_SNAPSHOT" && "$cmd" =~ ^(install|update|remove)$ ]] && { df --print-type / | grep -q btrfs; } then echo "[tue-get] Creating btrfs snapshot" @@ -1013,14 +1015,16 @@ function tue-get return $error_code; fi - if [ -f /tmp/tue_get_remove_lock ] + local remove_lock_file + remove_lock_file=/tmp/tue_get_remove_lock + if [[ -f ${remove_lock_file} ]] then echo "[tue-get] Can't execute 'remove' as an other run is still busy" - echo "[tue-get] If this keeps happening, excute: rm /tmp/tue_get_remove_lock" + echo "[tue-get] If this keeps happening, execute: 'rm ${remove_lock_file}'" return 1 fi - touch /tmp/tue_get_remove_lock + touch ${remove_lock_file} for target in $targets_to_remove do local target_error @@ -1033,7 +1037,7 @@ function tue-get echo "[tue-get] Problems during uninstalling $target" else rm "$tue_installed_dir"/"$target" - echo "[tue-get] Succesfully uninstalled: $target" + echo "[tue-get] Successfully uninstalled: ${target}" fi done @@ -1043,7 +1047,7 @@ function tue-get _generate_setup_file fi - rm /tmp/tue_get_remove_lock + rm ${remove_lock_file} echo "" if [ -n "$2" ] @@ -1067,24 +1071,24 @@ function tue-get echo "[tue-get](show) Provide at least one target name" return 1 fi - local firsttarget - firsttarget=true + local first_target + first_target=true for target in "$@" do - if [[ $firsttarget == false ]] + if [[ ${first_target} == false ]] then echo "" fi if [ ! -d "$TUE_ENV_TARGETS_DIR"/"$target" ] then echo "[tue-get](show) '$target' is not a valid target" - firsttarget=false + first_target=false continue fi - local firstfile + local first_file local -a files - firstfile=true + first_file=true mapfile -t files < <(find "$TUE_ENV_TARGETS_DIR"/"$target" -type f) # First show the common target files @@ -1096,12 +1100,12 @@ function tue-get do if [ "${files[$key]}" == "$TUE_ENV_TARGETS_DIR"/"$target"/"$file" ] then - if [[ $firstfile == false ]] + if [[ ${first_file} == false ]] then echo "" fi _show_file "$target" "$file" - firstfile=false + first_file=false unset "files[$key]" files=("${files[@]}") break @@ -1112,14 +1116,14 @@ function tue-get # Show all remaining files for file in "${files[@]}" do - if [[ $firstfile == false ]] + if [[ ${first_file} == false ]] then echo "" fi _show_file "$target" "${file#*"${TUE_ENV_TARGETS_DIR}/${target}/"}" - firstfile=false + first_file=false done - firsttarget=false + first_target=false done elif [[ $cmd == "dep" ]]