Make release.py to automatically deal with ign- Citadel packages (#1109)

* Make release.py to automatically deal with ign- Citadel packages Signed-off-by: Jose Luis Rivero <[email protected]> * Use gz- in SOURCE_REPO_URI Signed-off-by: Jose Luis Rivero <[email protected]> * Be sure of not using github.org Signed-off-by: Jose Luis Rivero <[email protected]> * Do not enforce the PACKAGE name in OSRFSourceCreation Signed-off-by: Jose Luis Rivero <[email protected]> * Add test to check SOURCE_REPO_URI Signed-off-by: Jose Luis Rivero <[email protected]> * Disable safety check on package-name tarball Signed-off-by: Jose Luis Rivero <[email protected]> * repository_uploader uses PACKAGE_ALIAS not PACKAGE Signed-off-by: Jose Luis Rivero <[email protected]> * PACKAGE needs to be in OSRFSourceCreation Signed-off-by: Jose Luis Rivero <[email protected]> * Fix dealing with ign-gazebo to be gz-sim Signed-off-by: Jose Luis Rivero <[email protected]> --------- Signed-off-by: Jose Luis Rivero <[email protected]>
gazebo-tooling · May 7, 2024 · 42a19d4 · 42a19d4
1 parent b9dd06a
commit 42a19d4
Show file tree

Hide file tree

Showing 4 changed files with 70 additions and 36 deletions.
diff --git a/check_releasepy.bash b/check_releasepy.bash
@@ -25,6 +25,16 @@ exec_ignition_releasepy_test()
     ign-foo 1.2.3 token ${test_params}""
 }
 
+exec_ignition_gazebo_releasepy_test()
+{
+  test_params=${1}
+
+    ./release.py \
+      --dry-run \
+      --no-sanity-checks \
+    ign-gazebo 1.2.3 token ${test_params}""
+}
+
 expect_job_run()
 {
   output="${1}" job="${2}"
@@ -66,10 +76,11 @@ expect_param()
 
 }
 
-source_repo_uri_test=$(exec_releasepy_test "--source-repo-uri https://github.org/gazebosim/gz-foo")
+source_repo_uri_test=$(exec_releasepy_test "--source-repo-uri https://github.com/gazebosim/gz-foo.git")
 expect_job_run "${source_repo_uri_test}" "gz-foo-source"
 expect_job_not_run "${source_repo_uri_test}" "gz-foo-debbuilder"
 expect_number_of_jobs "${source_repo_uri_test}" "1"
+expect_param "${source_repo_uri_test}" "SOURCE_REPO_URI=https%3A%2F%2Fgithub.com%2Fgazebosim%2Fgz-foo.git"
 
 source_tarball_uri_test=$(exec_releasepy_test "--source-tarball-uri https://gazebosim/gz-foo-1.2.3.tar.gz")
 expect_job_run "${source_tarball_uri_test}" "gz-foo-debbuilder"
@@ -92,14 +103,28 @@ expect_job_not_run "${bump_linux_test}" "gz-foo-source"
 expect_number_of_jobs "${bump_linux_test}" "6"
 expect_param "${bump_linux_test}" "RELEASE_VERSION=2"
 
-ignition_test=$(exec_ignition_releasepy_test "--source-repo-uri https://github.org/gazebosim/gz-foo")
+ignition_test=$(exec_ignition_releasepy_test "--source-repo-uri https://github.com/gazebosim/gz-foo.git")
 expect_job_run "${ignition_test}" "gz-foo-source"
 expect_job_not_run "${ignition_test}" "ignition-foo-source"
 expect_number_of_jobs "${ignition_test}" "1"
+expect_param "${ignition_test}" "PACKAGE=ign-foo"
+expect_param "${ignition_test}" "PACKAGE_ALIAS=ignition-foo"
+expect_param "${ignition_test}" "SOURCE_REPO_URI=https%3A%2F%2Fgithub.com%2Fgazebosim%2Fgz-foo.git"
 
 ignition_source_tarball_uri_test=$(exec_ignition_releasepy_test "--source-tarball-uri https://gazebosim/gz-foo-1.2.3.tar.gz")
 expect_job_run "${ignition_source_tarball_uri_test}" "gz-foo-debbuilder"
 expect_job_run "${ignition_source_tarball_uri_test}" "generic-release-homebrew_pull_request_updater"
 expect_job_not_run "${ignition_source_tarball_uri_test}" "gz-foo-source"
 expect_number_of_jobs "${ignition_source_tarball_uri_test}" "7"
 expect_param "${ignition_source_tarball_uri_test}" "SOURCE_TARBALL_URI=https%3A%2F%2Fgazebosim%2Fgz-foo-1.2.3.tar.gz"
+expect_param "${ignition_source_tarball_uri_test}" "PACKAGE=ign-foo"
+expect_param "${ignition_source_tarball_uri_test}" "PACKAGE_ALIAS=ignition-foo"
+
+ign_gazebo_source_tarball_uri_test=$(exec_ignition_gazebo_releasepy_test "--source-tarball-uri https://gazebosim/ign-gazebo-1.2.3.tar.gz")
+expect_job_run "${ign_gazebo_source_tarball_uri_test}" "gz-sim-debbuilder"
+expect_job_run "${ign_gazebo_source_tarball_uri_test}" "generic-release-homebrew_pull_request_updater"
+expect_job_not_run "${ign_gazebo_source_tarball_uri_test}" "gz-sim-source"
+expect_number_of_jobs "${ign_gazebo_source_tarball_uri_test}" "7"
+expect_param "${ign_gazebo_source_tarball_uri_test}" "SOURCE_TARBALL_URI=https%3A%2F%2Fgazebosim%2Fign-gazebo-1.2.3.tar.gz"
+expect_param "${ign_gazebo_source_tarball_uri_test}" "PACKAGE=ign-gazebo"
+expect_param "${ign_gazebo_source_tarball_uri_test}" "PACKAGE_ALIAS=ignition-gazebo"
diff --git a/jenkins-scripts/dsl/_configs_/OSRFSourceCreation.groovy b/jenkins-scripts/dsl/_configs_/OSRFSourceCreation.groovy
@@ -33,9 +33,11 @@ class OSRFSourceCreation
         stringParam("DISTRO",
                     default_params.find{ it.key == "DISTRO"}?.value,
                     "Linux release inside LINUX_DISTRO to generate sources on")
-        choiceParam('PACKAGE',
-                    [default_params.find{ it.key == "PACKAGE"}?.value],
-                    "For downstream use: Package name (can not be modified)")
+        // Not using choiceParam here to support Citadel/Fortress ign-* packages not only
+        // gz-* packages
+        stringParam('PACKAGE',
+                    default_params.find{ it.key == "PACKAGE"}?.value,
+                    "For downstream use: Package name")
         stringParam("RELEASE_VERSION",
                     default_params.find{ it.key == "RELEASE_VERSION"}?.value,
                     "For downstream jobs: Packages release version")
@@ -70,8 +72,6 @@ class OSRFSourceCreation
         priority 100
       }
 
-      def canonical_package_name = Globals.get_canonical_package_name(
-        default_params.find{ it.key == "PACKAGE"}.value)
       def s3_download_url_basedir = Globals.s3_download_url_basedir(
         default_params.find{ it.key == "PACKAGE"}?.value)
 
@@ -94,14 +94,22 @@ class OSRFSourceCreation
           /bin/bash -x ./scripts/jenkins-scripts/docker/gz-source-generation.bash
           """.stripIndent()
         )
+
+        // This can be enabled after the complete deprecation of ign- names
+        // that uses ignition aliases.
+        // def canonical_package_name = Globals.get_canonical_package_name(
+        //  default_params.find{ it.key == "PACKAGE"}.value)
+        //
+        // then improve the find command below with:
+        //  -name ${canonical_package_name}-\${VERSION}.tar.* \
         shell("""\
           #!/bin/bash -xe
 
           # Export information from the build in properties_files. The tarball extraction helps to
           # deal with changes in the compression of the tarballs.
           tarball=\$(find \${WORKSPACE}/${pkg_sources_dir} \
                        -type f \
-                       -name ${canonical_package_name}-\${VERSION}.tar.* \
+                       -name *-\${VERSION}.tar.* \
                        -printf "%f\\n")
           if [[ -z \${tarball} ]] || [[ \$(wc -w <<< \${tarball}) != 1 ]]; then
             # There is one use case that can be valid but failed to find canonical_package_name
@@ -148,6 +156,7 @@ class OSRFSourceCreation
                     parameters {
                       currentBuild()
                       predefinedProps([PROJECT_NAME_TO_COPY_ARTIFACTS: '${JOB_NAME}',
+                                       PACKAGE_ALIAS: '${PACKAGE_ALIAS}',
                                        S3_UPLOAD_PATH: "${Globals.s3_releases_dir(package_name)}/"])  // relative path with a final /
                       propertiesFile(properties_file)  // S3_FILES_TO_UPLOAD
                     }

diff --git a/jenkins-scripts/dsl/test.dsl b/jenkins-scripts/dsl/test.dsl
@@ -30,6 +30,7 @@ releasepy_job.with {
 def gz_source_job = job("_test_gz_source")
 OSRFSourceCreation.create(gz_source_job, [
   PACKAGE: "gz-plugin2",
+  PACKAGE_ALIAS: "gz-plugin2",
   SOURCE_REPO_URI: "https://github.com/gazebosim/gz-plugin.git",
   SOURCE_REPO_REF: "gz-plugin2"])
 OSRFSourceCreation.call_uploader_and_releasepy(gz_source_job,
@@ -50,7 +51,7 @@ repo_uploader.with
 
   parameters
   {
-    stringParam('PACKAGE','','Package name')
+    stringParam('PACKAGE_ALIAS','','Used by real repository_uploader')
     stringParam('S3_UPLOAD_PATH','', 'S3 path to upload')
     stringParam('S3_FILES_TO_UPLOAD','', 'S3 file names to upload')
     stringParam('UPLOAD_TO_REPO','none','repo to upload')

diff --git a/release.py b/release.py
@@ -31,22 +31,6 @@
 
 IGNORE_DRY_RUN = True
 
-GARDEN_IGN_PACKAGES = ['ign-cmake3',
-                       'ign-common5',
-                       'ign-fuel-tools8',
-                       'ign-sim7',
-                       'ign-gui7',
-                       'ign-launch6',
-                       'ign-math7',
-                       'ign-msgs9',
-                       'ign-physics6',
-                       'ign-plugin2',
-                       'ign-rendering7',
-                       'ign-sensors7',
-                       'ign-tools2',
-                       'ign-transport12',
-                       'ign-utils2']
-
 
 class ErrorNoPermsRepo(Exception):
     pass
@@ -85,6 +69,12 @@ def get_canonical_package_name(pkg_name):
     return pkg_name.rstrip('1234567890')
 
 
+def replace_ignition_gz(pkg_name):
+    return pkg_name \
+        .replace('ignition-gazebo', 'gz-sim') \
+        .replace('ignition-', 'gz-')
+
+
 def github_repo_exists(url):
     try:
         check_call(['git', 'ls-remote', '-q', '--exit-code', url], IGNORE_DRY_RUN)
@@ -164,10 +154,6 @@ def parse_args(argv):
 
     args = parser.parse_args()
 
-    if args.package in GARDEN_IGN_PACKAGES:
-        print(f"Garden packages start with gz- prefix, changing {args.package} to {args.package.replace('ign-','gz-')}",)
-        args.package = args.package.replace('ign-', 'gz-')
-
     args.package_alias = args.package
     if args.package.startswith('ign-'):
         args.package_alias = args.package.replace('ign-', 'ignition-')
@@ -310,6 +296,11 @@ def sanity_check_source_repo_uri(source_repo_uri):
     if not parsed_uri.scheme == "https" or \
        not parsed_uri.path.endswith(".git"):
         error("--source-repo-uri parameter should start with https:// and end with .git")
+    # Needs to be fully in sync for SOURCE_REPO_URI values in
+    # OSRFSourceCreation
+    # https://github.com/gazebo-tooling/release-tools/blob/master/jenkins-scripts/dsl/gazebo_libs.dsl#L513
+    if parsed_uri.netloc.startswith("github.org"):
+        error("--source-repo-uri needs github.com instead of github.org")
 
 
 def sanity_check_bump_linux(source_tarball_uri):
@@ -445,12 +436,20 @@ def generate_source_repository_uri(args):
 
     git_remote = out.decode().split('\n')[0]
     if org_repo not in git_remote:
-        print(f""" !! Automatic calculation of source_repo_uri failed.\
-              \n   * git remote origin is: {git_remote}\
-              \n   * Package name generated org/repo: {org_repo}\
-              \n >> Please use --source-repo-uri parameter""")
-        sys.exit(1)
+        # Handle the special case for citadel ignition repositories
+        if replace_ignition_gz(org_repo) not in git_remote:
+            print(f""" !! Automatic calculation of the source repository URI\
+                  failed with different information:\
+                  \n   * git remote origin in the local direcotry is: {git_remote}\
+                  \n   * Package name generated org/repo: {org_repo}\
+                  \n >> Please use --source-repo-uri parameter""")
+            sys.exit(1)
+        else:
+            org_repo = replace_ignition_gz(org_repo)
+            print(' ~ Ignition found in generated org/repo assuming gz repo: '
+                  + org_repo)
 
+    # Always use github.com
     return f"https://github.com/{org_repo}.git"  # NOQA
 
 
@@ -506,7 +505,7 @@ def display_help_job_chain_for_source_calls(args):
         f'{JENKINS_URL}/job/repository_uploader_packages/?{url_search_params}'
     rel_search_params = urllib.parse.urlencode(
         {'search':
-            f'{args.package_alias}/{args.version}-{args.release_version}'})
+            f'{args.package}/{args.version}-{args.release_version}'})
     releasepy_check_url = \
         f'{JENKINS_URL}/job/_releasepy/?{rel_search_params}'
     print('\tINFO: After the source job finished, the release process will trigger:\n'
@@ -523,7 +522,7 @@ def go(argv):
     if not args.release_version:
         args.release_version = 1
 
-    package_alias_force_gz = args.package_alias.replace('ignition-','gz-')
+    package_alias_force_gz = replace_ignition_gz(args.package_alias)
 
     print(f"Downloading releasing info for {args.package}")
     # Sanity checks and dicover supported distributions before proceed.