Merge remote-tracking branch 'origin' into jrivero/remove_indexes_dup…

…_abichecker

check_releasepy.bash

@@ -25,6 +25,16 @@ exec_ignition_releasepy_test()
     ign-foo 1.2.3 token ${test_params}""
 }
 
+exec_ignition_gazebo_releasepy_test()
+{
+  test_params=${1}
+
+    ./release.py \
+      --dry-run \
+      --no-sanity-checks \
+    ign-gazebo 1.2.3 token ${test_params}""
+}
+
 expect_job_run()
 {
   output="${1}" job="${2}"
@@ -66,10 +76,11 @@ expect_param()
 
 }
 
-source_repo_uri_test=$(exec_releasepy_test "--source-repo-uri https://github.org/gazebosim/gz-foo")
+source_repo_uri_test=$(exec_releasepy_test "--source-repo-uri https://github.com/gazebosim/gz-foo.git")
 expect_job_run "${source_repo_uri_test}" "gz-foo-source"
 expect_job_not_run "${source_repo_uri_test}" "gz-foo-debbuilder"
 expect_number_of_jobs "${source_repo_uri_test}" "1"
+expect_param "${source_repo_uri_test}" "SOURCE_REPO_URI=https%3A%2F%2Fgithub.com%2Fgazebosim%2Fgz-foo.git"
 
 source_tarball_uri_test=$(exec_releasepy_test "--source-tarball-uri https://gazebosim/gz-foo-1.2.3.tar.gz")
 expect_job_run "${source_tarball_uri_test}" "gz-foo-debbuilder"
@@ -92,14 +103,28 @@ expect_job_not_run "${bump_linux_test}" "gz-foo-source"
 expect_number_of_jobs "${bump_linux_test}" "6"
 expect_param "${bump_linux_test}" "RELEASE_VERSION=2"
 
-ignition_test=$(exec_ignition_releasepy_test "--source-repo-uri https://github.org/gazebosim/gz-foo")
+ignition_test=$(exec_ignition_releasepy_test "--source-repo-uri https://github.com/gazebosim/gz-foo.git")
 expect_job_run "${ignition_test}" "gz-foo-source"
 expect_job_not_run "${ignition_test}" "ignition-foo-source"
 expect_number_of_jobs "${ignition_test}" "1"
+expect_param "${ignition_test}" "PACKAGE=ign-foo"
+expect_param "${ignition_test}" "PACKAGE_ALIAS=ignition-foo"
+expect_param "${ignition_test}" "SOURCE_REPO_URI=https%3A%2F%2Fgithub.com%2Fgazebosim%2Fgz-foo.git"
 
 ignition_source_tarball_uri_test=$(exec_ignition_releasepy_test "--source-tarball-uri https://gazebosim/gz-foo-1.2.3.tar.gz")
 expect_job_run "${ignition_source_tarball_uri_test}" "gz-foo-debbuilder"
 expect_job_run "${ignition_source_tarball_uri_test}" "generic-release-homebrew_pull_request_updater"
 expect_job_not_run "${ignition_source_tarball_uri_test}" "gz-foo-source"
 expect_number_of_jobs "${ignition_source_tarball_uri_test}" "7"
 expect_param "${ignition_source_tarball_uri_test}" "SOURCE_TARBALL_URI=https%3A%2F%2Fgazebosim%2Fgz-foo-1.2.3.tar.gz"
+expect_param "${ignition_source_tarball_uri_test}" "PACKAGE=ign-foo"
+expect_param "${ignition_source_tarball_uri_test}" "PACKAGE_ALIAS=ignition-foo"
+
+ign_gazebo_source_tarball_uri_test=$(exec_ignition_gazebo_releasepy_test "--source-tarball-uri https://gazebosim/ign-gazebo-1.2.3.tar.gz")
+expect_job_run "${ign_gazebo_source_tarball_uri_test}" "gz-sim-debbuilder"
+expect_job_run "${ign_gazebo_source_tarball_uri_test}" "generic-release-homebrew_pull_request_updater"
+expect_job_not_run "${ign_gazebo_source_tarball_uri_test}" "gz-sim-source"
+expect_number_of_jobs "${ign_gazebo_source_tarball_uri_test}" "7"
+expect_param "${ign_gazebo_source_tarball_uri_test}" "SOURCE_TARBALL_URI=https%3A%2F%2Fgazebosim%2Fign-gazebo-1.2.3.tar.gz"
+expect_param "${ign_gazebo_source_tarball_uri_test}" "PACKAGE=ign-gazebo"
+expect_param "${ign_gazebo_source_tarball_uri_test}" "PACKAGE_ALIAS=ignition-gazebo"

jenkins-scripts/docker/lib/_generic_linux_compilation_build.sh.bash

@@ -96,7 +96,10 @@ if $GENERIC_ENABLE_TESTS; then
   echo '# BEGIN SECTION: running tests'
   init_stopwatch TEST
   mkdir -p \$HOME
-  make test ARGS="-VV ${BUILDING_EXTRA_MAKETEST_PARAMS}" || true
+  make test ARGS="-VV ${BUILDING_EXTRA_MAKETEST_PARAMS} --output-junit cmake_junit_output.xml" || true
+  if [ -f cmake_junit_output.xml ]; then
+    python3 $WORKSPACE/scripts/jenkins-scripts/tools/cmake_to_gtest_junit_output.py cmake_junit_output.xml test_results  || true
+  fi
   stop_stopwatch TEST
   echo '# END SECTION'
 else

jenkins-scripts/docker/lib/_install_nvidia_docker.sh

@@ -1,22 +1,7 @@
-INSTALL_NVIDIA_DOCKER1="""
-echo '# BEGIN SECTION: install docker (in docker)'
-curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
-sudo add-apt-repository \"deb [arch=amd64] https://download.docker.com/linux/ubuntu \$(lsb_release -cs) stable\"
-sudo apt-get update
-sudo apt-get install -y docker-ce
-echo '# END SECTION'
-
-echo '# BEGIN SECTION: install nvidia-docker1 (in docker)'
-sudo apt-get install -y wget nvidia-340 nvidia-modprobe
-wget -P /tmp https://github.com/NVIDIA/nvidia-docker/releases/download/v1.0.1/nvidia-docker_1.0.1-1_amd64.deb
-sudo dpkg -i /tmp/nvidia-docker*.deb && rm /tmp/nvidia-docker*.deb
-echo '# END SECTION'
-"""
-
 INSTALL_NVIDIA_DOCKER2="""
 echo '# BEGIN SECTION: install docker (in docker)'
 curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
-sudo add-apt-repository \"deb [arch=amd64] https://download.docker.com/linux/ubuntu \$(lsb_release -cs) stable\"
+sudo add-apt-repository -y \"deb [arch=amd64] https://download.docker.com/linux/ubuntu \$(lsb_release -cs) stable\"
 sudo apt-get update
 # sudo apt-get install -y docker-ce
 echo '# END SECTION'
@@ -29,11 +14,8 @@ curl -s -L https://nvidia.github.io/nvidia-docker/\$distribution/nvidia-docker.l
   sudo tee /etc/apt/sources.list.d/nvidia-docker.list
 sudo apt-get update
 sudo apt-get install -y nvidia-docker2
-# systemctl daemon-reload
-# systemctl restart docker
+sudo docker info
 echo '# END SECTION'
 """
 
-DOCKER2_CMD="sudo docker run --privileged -e DISPLAY=unix:0 -v /sys:/sys:ro -v /var/run/docker.sock:/var/run/docker.sock -v /tmp/.X11-unix:/tmp/.X11-unix:rw --runtime=nvidia -e DOCKER_FIX= -v /dev/log:/dev/log:ro -v /run/log:/run/log:ro -v /sys/fs/cgroup:/sys/fs/cgroup:ro --device /dev/snd --tty --rm"
-
 DEPENDENCY_PKGS="${DEPENDENCY_PKGS} curl software-properties-common"

jenkins-scripts/docker/lib/docker_generate_dockerfile.bash

@@ -45,7 +45,7 @@ GZDEV_BRANCH=${GZDEV_BRANCH:-master}
 if python3 ${SCRIPT_DIR}/../tools/detect_ci_matching_branch.py "${ghprbSourceBranch}"; then
   GZDEV_TRY_BRANCH=$ghprbSourceBranch
 fi
-
+ 
 KEYSERVER="keyserver.ubuntu.com"
 if [[ "${DISTRO}" == 'bionic' || "${DISTRO}" == 'focal' ]]; then
   KEYSERVER="hkps://pgp.surf.nl"
@@ -62,23 +62,26 @@ RUN if [ -n $GZDEV_TRY_BRANCH ]; then \
     fi || true
 # print branch for informational purposes
 RUN git -C ${GZDEV_DIR} branch
+# clean all _gzdev_ repository installations from the system before handling the configuration
+# otherwise the docker cache could contain unexpected repositories
+RUN ${GZDEV_DIR}/gzdev.py repository purge
 DELIM_OSRF_REPO_GIT
 if [[ -n ${GZDEV_PROJECT_NAME} ]]; then
 # debian sid docker images does not return correct name so we need to use
 # force-linux-distro
 cat >> Dockerfile << DELIM_OSRF_REPO_GZDEV
-RUN ${GZDEV_DIR}/gzdev.py repository enable --project=${GZDEV_PROJECT_NAME} --force-linux-distro=${DISTRO} --keyserver ${KEYSERVER} || ( git -C ${GZDEV_DIR} pull origin ${GZDEV_BRANCH} && \
+RUN ${GZDEV_DIR}/gzdev.py repository enable --project=${GZDEV_PROJECT_NAME} --force-linux-distro=${DISTRO} || ( git -C ${GZDEV_DIR} pull origin ${GZDEV_BRANCH} && \
     if [ -n $GZDEV_TRY_BRANCH ]; then git -C ${GZDEV_DIR} checkout $GZDEV_TRY_BRANCH; fi || true && \
-    ${GZDEV_DIR}/gzdev.py repository enable --project=${GZDEV_PROJECT_NAME} --force-linux-distro=${DISTRO} --keyserver ${KEYSERVER})
+    ${GZDEV_DIR}/gzdev.py repository enable --project=${GZDEV_PROJECT_NAME} --force-linux-distro=${DISTRO} )
 DELIM_OSRF_REPO_GZDEV
 fi
 
 # This could duplicate repositories enabled in the step above. gzdev should warn about it without failing.
 for repo in ${OSRF_REPOS_TO_USE}; do
 cat >> Dockerfile << DELIM_OSRF_REPO
-RUN ${GZDEV_DIR}/gzdev.py repository enable osrf ${repo} --force-linux-distro=${DISTRO} --keyserver ${KEYSERVER} || ( git -C ${GZDEV_DIR} pull origin ${GZDEV_BRANCH} && \
+RUN ${GZDEV_DIR}/gzdev.py repository enable osrf ${repo} --force-linux-distro=${DISTRO} || ( git -C ${GZDEV_DIR} pull origin ${GZDEV_BRANCH} && \
     if [ -n $GZDEV_TRY_BRANCH ]; then git -C ${GZDEV_DIR} checkout $GZDEV_TRY_BRANCH; fi || true && \
-    ${GZDEV_DIR}/gzdev.py repository enable osrf ${repo} --force-linux-distro=${DISTRO} --keyserver ${KEYSERVER})
+    ${GZDEV_DIR}/gzdev.py repository enable osrf ${repo} --force-linux-distro=${DISTRO} )
 DELIM_OSRF_REPO
 done
 }
@@ -113,12 +116,10 @@ case ${ARCH} in
      fi
      ;;
    'armhf')
-     # There is no osrf/jammy_armhf image. Trying new
-     # platform support in docker
-     if [[ ${DISTRO} == 'jammy' ]]; then
-      FROM_VALUE=${LINUX_DISTRO}:${DISTRO}
-     else
+     if [[ ${DISTRO} == 'focal' ]]; then
       FROM_VALUE=osrf/${LINUX_DISTRO}_${ARCH}:${DISTRO}
+     else
+      FROM_VALUE=${LINUX_DISTRO}:${DISTRO}
      fi
      ;;
   'arm64')
@@ -413,6 +414,13 @@ DELIM_DISPLAY
   fi
 fi
 
+if ${USE_DOCKER_IN_DOCKER}; then
+cat >> Dockerfile << DELIM_WORKAROUND_DOCKER_IN_DOCKER_HOOK
+# Avoid ERROR: invoke-rc.d: policy-rc.d denied execution of start.
+RUN sed -i "s/^exit 101$/exit 0/" /usr/sbin/policy-rc.d
+DELIM_WORKAROUND_DOCKER_IN_DOCKER_HOOK
+fi
+
 if [ `expr length "${DOCKER_POSTINSTALL_HOOK}"` -gt 1 ]; then
 cat >> Dockerfile << DELIM_WORKAROUND_POST_HOOK
 RUN ${DOCKER_POSTINSTALL_HOOK}
@@ -468,11 +476,16 @@ RUN adduser --uid \$USERID --gid \$GID --gecos "Developer" --disabled-password \
 RUN adduser \$USER sudo
 RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
 RUN chown -R \$USER:\$USER /home/\$USER
+# Needed if USE_DOCKER_IN_DOCKER is active. Harmless to be here
+RUN groupadd docker
+RUN gpasswd -a \$USER docker
+RUN newgrp docker
 
 # permit access to USER variable inside docker
 ENV USER \$USER
 USER \$USER
 # Must use sudo where necessary from this point on
+
 DELIM_DOCKER_USER
 
 if [[ -n ${SOFTWARE_DIR} ]]; then

jenkins-scripts/docker/lib/docker_run.bash

@@ -26,7 +26,9 @@ USER=$(whoami)
 # platform support starts on versions greater than 17.07
 PLAFTORM_PARAM=
 DOCKER_CLI_PLUGIN=
-if [[ ${DISTRO} == 'jammy' && ${ARCH} == 'armhf' ]]; then
+if [[ ${LINUX_DISTRO} == 'ubuntu' && \
+      ${DISTRO} != 'focal' && \
+      ${ARCH} == 'armhf' ]]; then
   PLAFTORM_PARAM="--platform=linux/armhf"
   DOCKER_CLI_PLUGIN="buildx"
 fi

jenkins-scripts/docker/lib/gzdev-base-linux.bash

@@ -13,49 +13,49 @@ set -ex
 export MAKE_JOBS=${MAKE_JOBS}
 export DISPLAY=${DISPLAY}
 
-${INSTALL_NVIDIA_DOCKER1}
+${INSTALL_NVIDIA_DOCKER2}
 
 echo '# BEGIN SECTION: install pip requirements'
 cd ${WORKSPACE}/gzdev
-pip3 install -r requirements.txt
+# pip3 install -r requirements.txt
 echo '# END SECTION'
 
-echo '# BEGIN SECTION: run gzdev for gazebo8 with nvidia'
-cd ${WORKSPACE}/gzdev
-./gzdev.py spawn --gzv=8 --nvidia
-echo '# END SECTION'
+echo '# BEGIN SECTION: smoke tests for ign-docker-env'
+sudo pip3 install git+https://github.com/adlarkin/ign-rocker.git
 
-echo '# BEGIN SECTION: Disply log file.'
-cat ./gz8.log
-echo '# END SECTION'
+# TODO: rocker can not play well docker-in-docker installations
+# out of the box. Needs more work.
+
+# xvfb :1 -ac -noreset -core -screen 0 1280x1024x24 &
+# export display=:1.0
+# export mesa_gl_version_override=3.3
+
+# test_timeout=300
+# test_start=\`date +%s\`
+# sudo bash -c "timeout --preserve-status \$test_timeout ./gzdev.py ign-docker-env dome --linux-distro ubuntu:bionic"
+# test_end=\`date +%s\`
+# diff=\$(expr \$test_end - \$test_start)
 
-echo '# BEGIN SECTION: check that gazebo is running'
-gazebo_detection=false
-seconds_waiting=0
-cat gz8.log | grep "Connected to gazebo master" && gazebo_detection=true
-while (! \$gazebo_detection); do
-   sleep 1
-   docker top gz8 | grep gazebo && gazebo_detection=true
-   docker top gz8 | grep gzserver && gazebo_detection=true
-   seconds_waiting=\$((seconds_waiting+1))
-   [ \$seconds_waiting -gt 30 ] && break
-done
-# clean up gazebo instances
-docker rm -f gz8 || true
-killall -9 gazebo gzserver gzclient || true
-! \${gazebo_detection} && exit 1
+# if [ \$diff -lt \$test_timeout ]; then
+#    echo 'the test took less than \$test_timeout. something bad happened'
+#    exit 1
+# fi
 echo '# END SECTION'
 DELIM
 
 export USE_DOCKER_IN_DOCKER=true
 export OSRF_REPOS_TO_USE="stable"
+export USE_ROS_REPO=true
+export ROS2=true
 export DEPENDENCY_PKGS="python3-pip \
                  bash \
                  apt-transport-https \
                  ca-certificates \
                  curl \
                  software-properties-common \
-		 psmisc" # killall
+                 python3-rocker \
+                 psmisc \
+                 xvfb"
 
 . "${SCRIPT_DIR}/lib/docker_generate_dockerfile.bash"
 . "${SCRIPT_DIR}/lib/docker_run.bash"

jenkins-scripts/docker/ros_gz-install-test-job.bash

@@ -18,12 +18,15 @@ export INSTALL_JOB_POSTINSTALL_HOOK="""
 
 echo '# BEGIN SECTION: ros_gz talker/listener'
 ros2 run ros_gz_bridge parameter_bridge /chatter@std_msgs/msg/[email protected] 2>/dev/null &
-sleep 1
+ros2 topic info /chatter
+# ros2 listener
 ros2 topic echo /chatter > /tmp/echo_chatter &
-sleep 1
+# print debug info the chatter topic
+gz topic -i -t /chatter
+# gz transport talker
 gz topic -t /chatter -m gz.msgs.StringMsg -p 'data:\"Hello\"' &
-sleep 1
-if ! grep -v Hello /tmp/echo_chatter; then
+sleep 2
+if ! grep -q Hello /tmp/echo_chatter; then
   echo 'chatter log file does not contain the expected Hello string'
   exit 1
 fi

jenkins-scripts/dsl/_configs_/OSRFSourceCreation.groovy

@@ -33,9 +33,11 @@ class OSRFSourceCreation
         stringParam("DISTRO",
                     default_params.find{ it.key == "DISTRO"}?.value,
                     "Linux release inside LINUX_DISTRO to generate sources on")
-        choiceParam('PACKAGE',
-                    [default_params.find{ it.key == "PACKAGE"}?.value],
-                    "For downstream use: Package name (can not be modified)")
+        // Not using choiceParam here to support Citadel/Fortress ign-* packages not only
+        // gz-* packages
+        stringParam('PACKAGE',
+                    default_params.find{ it.key == "PACKAGE"}?.value,
+                    "For downstream use: Package name")
         stringParam("RELEASE_VERSION",
                     default_params.find{ it.key == "RELEASE_VERSION"}?.value,
                     "For downstream jobs: Packages release version")
@@ -70,8 +72,6 @@ class OSRFSourceCreation
         priority 100
       }
 
-      def canonical_package_name = Globals.get_canonical_package_name(
-        default_params.find{ it.key == "PACKAGE"}.value)
       def s3_download_url_basedir = Globals.s3_download_url_basedir(
         default_params.find{ it.key == "PACKAGE"}?.value)
 
@@ -94,14 +94,22 @@ class OSRFSourceCreation
           /bin/bash -x ./scripts/jenkins-scripts/docker/gz-source-generation.bash
           """.stripIndent()
         )
+
+        // This can be enabled after the complete deprecation of ign- names
+        // that uses ignition aliases.
+        // def canonical_package_name = Globals.get_canonical_package_name(
+        //  default_params.find{ it.key == "PACKAGE"}.value)
+        //
+        // then improve the find command below with:
+        //  -name ${canonical_package_name}-\${VERSION}.tar.* \
         shell("""\
           #!/bin/bash -xe
 
           # Export information from the build in properties_files. The tarball extraction helps to
           # deal with changes in the compression of the tarballs.
           tarball=\$(find \${WORKSPACE}/${pkg_sources_dir} \
                        -type f \
-                       -name ${canonical_package_name}-\${VERSION}.tar.* \
+                       -name *-\${VERSION}.tar.* \
                        -printf "%f\\n")
           if [[ -z \${tarball} ]] || [[ \$(wc -w <<< \${tarball}) != 1 ]]; then
             # There is one use case that can be valid but failed to find canonical_package_name
@@ -148,6 +156,7 @@ class OSRFSourceCreation
                     parameters {
                       currentBuild()
                       predefinedProps([PROJECT_NAME_TO_COPY_ARTIFACTS: '${JOB_NAME}',
+                                       PACKAGE_ALIAS: '${PACKAGE_ALIAS}',
                                        S3_UPLOAD_PATH: "${Globals.s3_releases_dir(package_name)}/"])  // relative path with a final /
                       propertiesFile(properties_file)  // S3_FILES_TO_UPLOAD
                     }