From e624294b33f1e5a27f76d3c4ee5d299c1b73d181 Mon Sep 17 00:00:00 2001 From: Stewart X Addison Date: Mon, 2 Sep 2024 18:17:58 +0100 Subject: [PATCH 01/19] First pass of batOrSh and fixes for windows containers Signed-off-by: Stewart X Addison --- .../common/openjdk_build_pipeline.groovy | 142 +++++++++++++----- 1 file changed, 103 insertions(+), 39 deletions(-) diff --git a/pipelines/build/common/openjdk_build_pipeline.groovy b/pipelines/build/common/openjdk_build_pipeline.groovy index 613938fd1..494c2bd14 100644 --- a/pipelines/build/common/openjdk_build_pipeline.groovy +++ b/pipelines/build/common/openjdk_build_pipeline.groovy @@ -900,7 +900,7 @@ class Build { target: 'workspace/target/', flatten: true) // Check if JRE exists, if so, build another installer for it - if (listArchives().any { it =~ /-jre/ } ) { buildWindowsInstaller(versionData, '**/OpenJDK*jre_*_windows*.zip', 'jre') } + if (listArchives(false).any { it =~ /-jre/ } ) { buildWindowsInstaller(versionData, '**/OpenJDK*jre_*_windows*.zip', 'jre') } break default: break @@ -1057,18 +1057,27 @@ class Build { /* Lists and returns any compressed archived or sbom file contents of the top directory of the build node */ - List listArchives() { - def files = context.sh( + List listArchives(Boolean allowBat) { + context.println "SXA: battable and batted 1060 - windbld#273" + + def files + if ( allowBat ) { context.println("listArchives() invoked with allowBat = true") } + if ( !allowBat ) { context.println("listArchives() invoked with allowBat = false") } + if ( buildConfig.TARGET_OS == 'windows' && buildConfig.DOCKER_IMAGE && allowBat ) { + // The grep here removes anything that still contains "*" because nothing matched + files = context.bat( + script: 'dir/b/s workspace\\target\\*.zip workspace\\target\\*.msi workspace\\target\\*.-sbom_* workspace\\target\\*.json', + returnStdout: true, + returnStatus: false + ).trim().replaceAll('\\\\','/').replaceAll('\\r','').split('\n').toList().grep( ~/^[^\*]*$/ ) // grep needed extra script approval + } else { + files = context.sh( script: '''find workspace/target/ | egrep -e '(\\.tar\\.gz|\\.zip|\\.msi|\\.pkg|\\.deb|\\.rpm|-sbom_.*\\.json)$' ''', returnStdout: true, returnStatus: false - ) - .trim() - .split('\n') - .toList() - + ).trim().split('\n').toList() + } context.println "listArchives: ${files}" - return files } @@ -1318,7 +1327,12 @@ class Build { MetaData data = formMetadata(version, initialWrite) Boolean metaWrittenOut = false - listArchives().each({ file -> + /* + * This is causing a problem when set to false. Sometimes it seems ok + * with the windows one, and sometimes not e.g. + * windbld#473/475/476/477 + */ + listArchives(false).each({ file -> def type = 'jdk' if (file.contains('-jre')) { type = 'jre' @@ -1333,16 +1347,21 @@ class Build { } else if (file.contains('-sbom')) { type = 'sbom' } + context.println "(writeMetaData) Potentially battable assuming sha256sum on windows 1340 windbld#388 - No - fails #479" - String hash = context.sh(script: """\ + String hash + if ( buildConfig.TARGET_OS == 'windows' && buildConfig.DOCKER_IMAGE ) { + hash = context.sh(script: "sha256sum ${file} | cut -f1 -d' '") // .replaceAll('\n', '') + } else { + hash = context.sh(script: """\ if [ -x "\$(command -v shasum)" ]; then (shasum -a 256 | cut -f1 -d' ') <$file else sha256sum $file | cut -f1 -d' ' fi - """.stripIndent(), returnStdout: true, returnStatus: false) + """.stripIndent(), returnStdout: true, returnStatus: false).replaceAll('\n', '') - hash = hash.replaceAll('\n', '') + } data.binary_type = type data.sha256 = hash @@ -1479,27 +1498,44 @@ class Build { return context.readFile(crossCompileVersionPath) } + /* + * In Windows docker containers sh can be unreliable, so use context.bat + * in preference. https://github.com/adoptium/infrastructure/issues/3714 + */ + def batOrSh(command) + { + if ( context.isUnix() ) { + context.sh(command) + } else { + context.bat(command) + } + } + /* Display the current git repo information */ def printGitRepoInfo() { context.println 'Checked out repo:' - context.sh(script: 'git status') + context.println 'batable and batted 1487 windbld #286-288' + batOrSh('git status') context.println 'Checked out HEAD commit SHA:' - context.sh(script: 'git rev-parse HEAD') + // windbld#245 + batOrSh('git rev-parse HEAD') } - /* +/* Executed on a build node, the function checks out the repository and executes the build via ./make-adopt-build-farm.sh Once the build completes, it will calculate its version output, commit the first metadata writeout, and archive the build results. Running in downstream job jdk-*-*-* build stage, called by build() */ + def buildScripts( cleanWorkspace, cleanWorkspaceAfter, cleanWorkspaceBuildOutputAfter, filename, - useAdoptShellScripts + useAdoptShellScripts, + enableSigner ) { return context.stage('build') { // Create the repo handler with the user's defaults to ensure a temurin-build checkout is not null @@ -1537,6 +1573,7 @@ class Build { // Issue: https://issues.jenkins.io/browse/JENKINS-64779 if (context.WORKSPACE != null && !context.WORKSPACE.isEmpty()) { context.println 'Cleaning workspace non-hidden files: ' + context.WORKSPACE + '/*' + context.println 'SXA: batable 1540' context.sh(script: 'rm -rf ' + context.WORKSPACE + '/*') } else { context.println 'Warning: Unable to clean workspace as context.WORKSPACE is null/empty' @@ -1564,7 +1601,8 @@ class Build { context.timeout(time: buildTimeouts.NODE_CLEAN_TIMEOUT, unit: 'HOURS') { if (context.WORKSPACE != null && !context.WORKSPACE.isEmpty()) { context.println 'Removing workspace openjdk build directory: ' + openjdk_build_dir - context.sh(script: 'rm -rf ' + openjdk_build_dir) + context.println 'SXA: batable and batted 1568 windbld#261,262' + batOrSh('rm -rf ' + openjdk_build_dir) } else { context.println 'Warning: Unable to remove workspace openjdk build directory as context.WORKSPACE is null/empty' } @@ -1587,11 +1625,11 @@ class Build { // Perform a git clean outside of checkout to avoid the Jenkins enforced 10 minute timeout // https://github.com/adoptium/infrastucture/issues/1553 + if ( buildConfig.TARGET_OS == 'windows' && buildConfig.DOCKER_IMAGE ) { - context.sh(script: 'git config --global safe.directory $(cygpath ' + '\$' + '{WORKSPACE})') + context.bat(script: 'bash -c "git config --global safe.directory $(cygpath ' + '\$' + '{WORKSPACE})"') } - context.sh(script: 'git clean -fdx') - + batOrSh('git clean -fdx') printGitRepoInfo() } } catch (FlowInterruptedException e) { @@ -1621,12 +1659,14 @@ class Build { if (env.JOB_NAME.contains('pr-tester')) { updateGithubCommitStatus('PENDING', 'Build Started') } + context.println "SXAEC: Before UASS" if (useAdoptShellScripts) { + context.println "SXAEC: UASS=true" context.println '[CHECKOUT] Checking out to adoptium/temurin-build...' repoHandler.checkoutAdoptBuild(context) printGitRepoInfo() - if ((buildConfig.TARGET_OS == 'mac' || buildConfig.TARGET_OS == 'windows') && buildConfig.JAVA_TO_BUILD != 'jdk8u') { - context.println "Processing exploded build, sign JMODS, and assemble build, for platform ${buildConfig.TARGET_OS} version ${buildConfig.JAVA_TO_BUILD}" + if ((buildConfig.TARGET_OS == 'mac' || buildConfig.TARGET_OS == 'windows') && buildConfig.JAVA_TO_BUILD != 'jdk8u' && enableSigner) { + context.println "Generating exploded build, sign JMODS, and assemble build, for platform ${buildConfig.TARGET_OS} version ${buildConfig.JAVA_TO_BUILD}" def signBuildArgs if (env.BUILD_ARGS != null && !env.BUILD_ARGS.isEmpty()) { signBuildArgs = env.BUILD_ARGS + ' --make-exploded-image' + openjdk_build_dir_arg @@ -1635,11 +1675,15 @@ class Build { } context.withEnv(['BUILD_ARGS=' + signBuildArgs]) { context.println 'Building an exploded image for signing' - context.sh(script: "./${ADOPT_DEFAULTS_JSON['scriptDirectories']['buildfarm']}") + // Call make-adopt-build-farm.sh to do initial windows/mac build + // windbld#254 + context.bat(script: "bash -c 'curl https://ci.adoptium.net/userContent/windows/openjdk-cached-workspace.configANDtargetANDbuild.tar.gz | tar -C /cygdrive/c/workspace/openjdk-build -xpzf -'") + // context.sh(script: "./${ADOPT_DEFAULTS_JSON['scriptDirectories']['buildfarm']}") } def base_path = build_path if (openjdk_build_dir_arg == "") { // If not using a custom openjdk build dir, then query what autoconf created as the build sub-folder + context.println 'SXA: not batable 1648 - windbld#263' base_path = context.sh(script: "ls -d ${build_path}/* | tr -d '\\n'", returnStdout:true) } context.println "base build path for jmod signing = ${base_path}" @@ -1650,7 +1694,10 @@ class Build { // JDK 16 + jpackage needs to be signed as well stash the resources folder containing the executables "${base_path}/jdk/modules/jdk.jpackage/jdk/jpackage/internal/resources/*" + // Should this part be under "if (enableSigner)" instead + // of it being on the earlier "if" section? context.node('eclipse-codesign') { + context.println 'SXA: batable-ish 1660' context.sh "rm -rf ${base_path}/* || true" repoHandler.checkoutAdoptBuild(context) @@ -1660,6 +1707,7 @@ class Build { context.unstash 'jmods' def target_os = "${buildConfig.TARGET_OS}" context.withEnv(['base_os='+target_os, 'base_path='+base_path]) { + context.println 'SXA: not batable 1670' // groovylint-disable context.sh ''' #!/bin/bash @@ -1730,9 +1778,11 @@ class Build { // groovylint-enable } context.stash name: 'signed_jmods', includes: "${base_path}/**/*" - } + } // context.node ("eclipse-codesign") - joe thinks it matches with something else though ... + // } // if (ENABLE_SIGN == true) // Remove jmod directories to be replaced with the stash saved above + context.println 'SXA: batable 1744' context.sh "rm -rf ${base_path}/hotspot/variant-server || true" context.sh "rm -rf ${base_path}/support/modules_cmds || true" context.sh "rm -rf ${base_path}/support/modules_libs || true" @@ -1752,9 +1802,11 @@ class Build { } context.withEnv(['BUILD_ARGS=' + assembleBuildArgs]) { context.println 'Assembling the exploded image' + // Call make-adopt-build-farm.sh on windows/mac to create signed tarball + context.println 'SXA: probably batable 1764' context.sh(script: "./${ADOPT_DEFAULTS_JSON['scriptDirectories']['buildfarm']}") } - } else { + } else { // Not Windows/Mac JDK11+ (i.e. doesn't require signing) def buildArgs if (env.BUILD_ARGS != null && !env.BUILD_ARGS.isEmpty()) { buildArgs = env.BUILD_ARGS + openjdk_build_dir_arg @@ -1762,7 +1814,12 @@ class Build { buildArgs = openjdk_build_dir_arg } context.withEnv(['BUILD_ARGS=' + buildArgs]) { - context.sh(script: "./${ADOPT_DEFAULTS_JSON['scriptDirectories']['buildfarm']}") + context.println 'SXA: probably batable 1775' + // Call make-adopt-build-farm.sh to do one-step build (i.e. not signed) + // and when USEW_ADOPT_SHELL_SCRIPTS=false +// batOrSh("./${ADOPT_DEFAULTS_JSON['scriptDirectories']['buildfarm']}") +// context.sh(script: "./${ADOPT_DEFAULTS_JSON['scriptDirectories']['buildfarm']}") + context.bat(script: "bash -c 'curl https://ci.adoptium.net/userContent/windows/openjdk-cached-workspace.tar.gz | tar -C /cygdrive/c/workspace/openjdk-build -xpzf -'") } } context.println '[CHECKOUT] Reverting pre-build adoptium/temurin-build checkout...' @@ -1774,7 +1831,7 @@ class Build { repoHandler.checkoutUserPipelines(context) } printGitRepoInfo() - } else { + } else { // USE_ADOPT_SHELL_SCRIPTS == false context.println "[CHECKOUT] Checking out to the user's temurin-build..." repoHandler.setUserDefaultsJson(context, DEFAULTS_JSON) repoHandler.checkoutUserBuild(context) @@ -1786,7 +1843,10 @@ class Build { buildArgs = openjdk_build_dir_arg } context.withEnv(['BUILD_ARGS=' + buildArgs]) { + context.println 'SXA: probably batable 1783' + // Call make-adopt-build-farm.sh when USE_ADOPT_SHELL_SCRIPTS==false context.sh(script: "./${DEFAULTS_JSON['scriptDirectories']['buildfarm']}") +// context.bat(script: "bash -c 'curl https://ci.adoptium.net/userContent/windows/openjdk-cached-workspace.tar.gz | tar -C /cygdrive/c/workspace/openjdk-build -xpzf -'") } context.println '[CHECKOUT] Reverting pre-build user temurin-build checkout...' repoHandler.checkoutUserPipelines(context) @@ -1843,6 +1903,7 @@ class Build { if (context.WORKSPACE != null && !context.WORKSPACE.isEmpty()) { if (cleanWorkspaceAfter) { context.println 'Cleaning workspace non-hidden files: ' + context.WORKSPACE + '/*' + context.println 'SXA: batable 1858' context.sh(script: 'rm -rf ' + context.WORKSPACE + '/*') // Clean remaining hidden files using cleanWs @@ -1853,14 +1914,9 @@ class Build { context.println "Failed to clean ${e}" } } else if (cleanWorkspaceBuildOutputAfter) { + context.println 'SXA: batable and batted 1869 windbld 266' context.println 'Cleaning workspace build output files: ' + openjdk_build_dir - context.sh(script: 'rm -rf ' + openjdk_build_dir) - context.println 'Cleaning workspace build output files: ' + context.WORKSPACE + '/workspace/target' - context.sh(script: 'rm -rf ' + context.WORKSPACE + '/workspace/target') - context.println 'Cleaning workspace build output files: ' + context.WORKSPACE + '/workspace/build/devkit' - context.sh(script: 'rm -rf ' + context.WORKSPACE + '/workspace/build/devkit') - context.println 'Cleaning workspace build output files: ' + context.WORKSPACE + '/workspace/build/straceOutput' - context.sh(script: 'rm -rf ' + context.WORKSPACE + '/workspace/build/straceOutput') + batOrSh('rm -rf ' + openjdk_build_dir + ' ' + context.WORKSPACE + '/workspace/target ' + context.WORKSPACE + '/workspace/build/devkit ' + context.WORKSPACE + '/workspace/build/straceOutput') } } else { context.println 'Warning: Unable to clean workspace as context.WORKSPACE is null/empty' @@ -2024,6 +2080,10 @@ class Build { context.node(label) { addNodeToBuildDescription() // Cannot clean workspace from inside docker container + context.println 'SXA: batable and batted 2042 (rm cyclonedx-lib)' + if ( buildConfig.TARGET_OS == 'windows' && buildConfig.DOCKER_IMAGE ) { + context.bat('rm -rf c:/workspace/openjdk-build/cyclonedx-lib') + } if (cleanWorkspace) { try { context.timeout(time: buildTimeouts.CONTROLLER_CLEAN_TIMEOUT, unit: 'HOURS') { @@ -2102,7 +2162,8 @@ class Build { cleanWorkspaceAfter, cleanWorkspaceBuildOutputAfter, filename, - useAdoptShellScripts + useAdoptShellScripts, + enableSigner ) } } else { @@ -2126,7 +2187,8 @@ class Build { cleanWorkspaceAfter, cleanWorkspaceBuildOutputAfter, filename, - useAdoptShellScripts + useAdoptShellScripts, + enableSigner ) } } @@ -2199,7 +2261,9 @@ class Build { try { // Installer job timeout managed by Jenkins job config buildInstaller(versionInfo) - signInstaller(versionInfo) + if ( enableSigner) { + signInstaller(versionInfo) + } } catch (FlowInterruptedException e) { currentBuild.result = 'FAILURE' throw new Exception("[ERROR] Installer job timeout (${buildTimeouts.INSTALLER_JOBS_TIMEOUT} HOURS) has been reached OR the downstream installer job failed. Exiting...") @@ -2216,7 +2280,7 @@ class Build { if (!env.JOB_NAME.contains('pr-tester')) { // pr-tester does not sign the binaries // Verify Windows and Mac Signing for Temurin - if (buildConfig.VARIANT == 'temurin') { + if (buildConfig.VARIANT == 'temurin' && enableSigner) { try { verifySigning() } catch (Exception e) { From 8d13a79f2302693c99158c7fe4f84dc56c529844 Mon Sep 17 00:00:00 2001 From: Stewart X Addison Date: Mon, 9 Sep 2024 16:24:16 +0100 Subject: [PATCH 02/19] Refactoring signing code to allow operation in Windows containers Signed-off-by: Stewart X Addison --- .../common/openjdk_build_pipeline.groovy | 513 ++++++++++++------ 1 file changed, 341 insertions(+), 172 deletions(-) diff --git a/pipelines/build/common/openjdk_build_pipeline.groovy b/pipelines/build/common/openjdk_build_pipeline.groovy index 494c2bd14..78478ad04 100644 --- a/pipelines/build/common/openjdk_build_pipeline.groovy +++ b/pipelines/build/common/openjdk_build_pipeline.groovy @@ -745,7 +745,7 @@ class Build { if ( buildConfig.TARGET_OS == 'windows' || (buildConfig.TARGET_OS == 'mac') ) { - context.stage('sign') { + context.stage('sign zip/tgz') { def filter = '' def nodeFilter = 'eclipse-codesign' @@ -784,7 +784,6 @@ class Build { flatten: true) context.sh 'for file in $(ls workspace/target/*.tar.gz workspace/target/*.zip); do sha256sum "$file" > $file.sha256.txt ; done' - writeMetadata(versionInfo, false) context.archiveArtifacts artifacts: 'workspace/target/*' } @@ -900,7 +899,7 @@ class Build { target: 'workspace/target/', flatten: true) // Check if JRE exists, if so, build another installer for it - if (listArchives(false).any { it =~ /-jre/ } ) { buildWindowsInstaller(versionData, '**/OpenJDK*jre_*_windows*.zip', 'jre') } + if (listArchives().any { it =~ /-jre/ } ) { buildWindowsInstaller(versionData, '**/OpenJDK*jre_*_windows*.zip', 'jre') } break default: break @@ -1057,25 +1056,22 @@ class Build { /* Lists and returns any compressed archived or sbom file contents of the top directory of the build node */ - List listArchives(Boolean allowBat) { - context.println "SXA: battable and batted 1060 - windbld#273" + List listArchives() { def files - if ( allowBat ) { context.println("listArchives() invoked with allowBat = true") } - if ( !allowBat ) { context.println("listArchives() invoked with allowBat = false") } - if ( buildConfig.TARGET_OS == 'windows' && buildConfig.DOCKER_IMAGE && allowBat ) { + if ( context.isUnix() ) { + files = context.sh( + script: '''find workspace/target/ | egrep -e '(\\.tar\\.gz|\\.zip|\\.msi|\\.pkg|\\.deb|\\.rpm|-sbom_.*\\.json)$' ''', + returnStdout: true, + returnStatus: false + ).trim().split('\n').toList() + } else { // The grep here removes anything that still contains "*" because nothing matched files = context.bat( script: 'dir/b/s workspace\\target\\*.zip workspace\\target\\*.msi workspace\\target\\*.-sbom_* workspace\\target\\*.json', returnStdout: true, returnStatus: false ).trim().replaceAll('\\\\','/').replaceAll('\\r','').split('\n').toList().grep( ~/^[^\*]*$/ ) // grep needed extra script approval - } else { - files = context.sh( - script: '''find workspace/target/ | egrep -e '(\\.tar\\.gz|\\.zip|\\.msi|\\.pkg|\\.deb|\\.rpm|-sbom_.*\\.json)$' ''', - returnStdout: true, - returnStatus: false - ).trim().split('\n').toList() } context.println "listArchives: ${files}" return files @@ -1332,7 +1328,7 @@ class Build { * with the windows one, and sometimes not e.g. * windbld#473/475/476/477 */ - listArchives(false).each({ file -> + listArchives().each({ file -> def type = 'jdk' if (file.contains('-jre')) { type = 'jre' @@ -1347,12 +1343,11 @@ class Build { } else if (file.contains('-sbom')) { type = 'sbom' } - context.println "(writeMetaData) Potentially battable assuming sha256sum on windows 1340 windbld#388 - No - fails #479" + context.println "(writeMetaData for " + file + ") Batable and batted assuming sha256sum on windows 1340 windbld#388 - No - fails #479" String hash - if ( buildConfig.TARGET_OS == 'windows' && buildConfig.DOCKER_IMAGE ) { - hash = context.sh(script: "sha256sum ${file} | cut -f1 -d' '") // .replaceAll('\n', '') - } else { + if ( context.isUnix() ) { + context.println "Non-windows non-docker detected - running sh" hash = context.sh(script: """\ if [ -x "\$(command -v shasum)" ]; then (shasum -a 256 | cut -f1 -d' ') <$file @@ -1360,7 +1355,9 @@ class Build { sha256sum $file | cut -f1 -d' ' fi """.stripIndent(), returnStdout: true, returnStatus: false).replaceAll('\n', '') - + } else { + context.println "Windows detected - running bat" + hash = context.bat(script: "sha256sum ${file} | cut -f1 -d' '") // .replaceAll('\n', '') } data.binary_type = type @@ -1516,13 +1513,235 @@ class Build { */ def printGitRepoInfo() { context.println 'Checked out repo:' - context.println 'batable and batted 1487 windbld #286-288' batOrSh('git status') context.println 'Checked out HEAD commit SHA:' // windbld#245 batOrSh('git rev-parse HEAD') } + def buildScriptsEclipseSigner() { + def build_path + def openjdk_build_dir +// openjdk_build_dir_arg unused after https://github.com/adoptium/ci-jenkins-pipelines/pull/1084 +// Ref https://adoptium.slack.com/archives/C09NW3L2J/p1725957286184479?thread_ts=1725900849.766449&cid=C09NW3L2J +// def openjdk_build_dir_arg + + build_path = 'workspace/build/src/build' + openjdk_build_dir = context.WORKSPACE + '/' + build_path +// openjdk_build_dir_arg = "" + // SXA: This doesn't seem to set correctly with 'def base_path = build_path" + def base_path + base_path = build_path +// if (openjdk_build_dir_arg == "") { +// // If not using a custom openjdk build dir, then query what autoconf created as the build sub-folder +// context.println 'SXA: not batable 1648 - windbld#263' +// base_path = context.sh(script: "ls -d ${build_path}/* | tr -d '\\n'", returnStdout:true) +// } + def repoHandler = new RepoHandler(USER_REMOTE_CONFIGS, ADOPT_DEFAULTS_JSON, buildConfig.CI_REF, buildConfig.BUILD_REF) + context.stage('internal sign') { + // SXAEC: Should this part be under "if (enableSigner)" instead + // of it being on the earlier "if" section? + context.node('eclipse-codesign') { + // Safety first! + if (base_path != null && !base_path.isEmpty()) { + context.sh "rm -rf ${base_path}/* || true" + } + + repoHandler.checkoutAdoptBuild(context) +// SXA - skipping as it's batOrSh and we can't allow bat here + printGitRepoInfo() + + // Copy pre assembled binary ready for JMODs to be codesigned + context.unstash 'jmods' + def target_os = "${buildConfig.TARGET_OS}" + context.withEnv(['base_os='+target_os, 'base_path='+base_path]) { + // groovylint-disable + context.sh ''' + #!/bin/bash + set -eu + echo "Signing JMOD files under build path ${base_path} for base_os ${base_os}" + TMP_DIR="${base_path}/" + if [ "${base_os}" == "mac" ]; then + ENTITLEMENTS="$WORKSPACE/entitlements.plist" + FILES=$(find "${TMP_DIR}" -perm +111 -type f -o -name '*.dylib' -type f || find "${TMP_DIR}" -perm /111 -type f -o -name '*.dylib' -type f) + else + FILES=$(find "${TMP_DIR}" -type f -name '*.exe' -o -name '*.dll') + fi + for f in $FILES + do + echo "Signing $f using Eclipse Foundation codesign service" + dir=$(dirname "$f") + file=$(basename "$f") + mv "$f" "${dir}/unsigned_${file}" + success=false + if [ "${base_os}" == "mac" ]; then + if ! curl --fail --silent --show-error -o "$f" -F file="@${dir}/unsigned_${file}" -F entitlements="@$ENTITLEMENTS" https://cbi.eclipse.org/macos/codesign/sign; then + echo "curl command failed, sign of $f failed" + else + success=true + fi + else + if ! curl --fail --silent --show-error -o "$f" -F file="@${dir}/unsigned_${file}" https://cbi.eclipse.org/authenticode/sign; then + echo "curl command failed, sign of $f failed" + else + success=true + fi + fi + if [ $success == false ]; then + # Retry up to 20 times + max_iterations=20 + iteration=1 + echo "Code Not Signed For File $f" + while [ $iteration -le $max_iterations ] && [ $success = false ]; do + echo $iteration Of $max_iterations + sleep 1 + if [ "${base_os}" == "mac" ]; then + if curl --fail --silent --show-error -o "$f" -F file="@${dir}/unsigned_${file}" -F entitlements="@$ENTITLEMENTS" https://cbi.eclipse.org/macos/codesign/sign; then + success=true + fi + else + if curl --fail --silent --show-error -o "$f" -F file="@${dir}/unsigned_${file}" https://cbi.eclipse.org/authenticode/sign; then + success=true + fi + fi + + if [ $success = false ]; then + echo "curl command failed, $f Failed Signing On Attempt $iteration" + iteration=$((iteration+1)) + if [ $iteration -gt $max_iterations ] + then + echo "Errors Encountered During Signing" + exit 1 + fi + else + echo "$f Signed OK On Attempt $iteration" + fi + done + fi + chmod --reference="${dir}/unsigned_${file}" "$f" + rm -rf "${dir}/unsigned_${file}" + done + ''' + // groovylint-enable + } + context.sh(script: "ls -l ${base_path}/**/*") + context.stash name: 'signed_jmods', includes: "${base_path}/**/*" + } // context.node ("eclipse-codesign") - joe thinks it matches with something else though ... + } // context.stage +} + +def buildScriptsAssemble( + cleanWorkspaceAfter, + filename, + useAdoptShellScripts +) { + def build_path + def openjdk_build_dir + def openjdk_build_dir_arg + openjdk_build_dir_arg="" + + build_path = 'workspace/build/src/build' + openjdk_build_dir = context.WORKSPACE + '/' + build_path + // SXA: This doesn't seem to set correctly with 'def base_path = build_path" + def base_path + base_path = build_path + def assembleBuildArgs + // Remove jmod directories to be replaced with the stash saved above + batOrSh "rm -rf ${base_path}/hotspot/variant-server ${base_path}/support/modules_cmds ${base_path}/support/modules_libs" + // JDK 16 + jpackage executables need to be signed as well + if (buildConfig.JAVA_TO_BUILD != 'jdk11u') { + batOrSh "rm -rf ${base_path}/jdk/modules/jdk.jpackage/jdk/jpackage/internal/resources/*" + } + context.stage('assemble') { + if ( buildConfig.TARGET_OS == 'windows' && buildConfig.DOCKER_IMAGE ) { + // SXAC: Still TBC on this to determine if something fails without it + context.bat('chmod -R a+rwX /cygdrive/c/workspace/openjdk-build/workspace/build/src/build & echo Done & exit 0') + } + // Restore signed JMODs + context.unstash 'signed_jmods' + // Convert IndividualBuildConfig to jenkins env variables + List envVars = buildConfig.toEnvVars() + envVars.add("FILENAME=${filename}" as String) + // Use BUILD_REF override if specified + def adoptBranch = buildConfig.BUILD_REF ?: ADOPT_DEFAULTS_JSON['repository']['build_branch'] + // Add platform config path so it can be used if the user doesn't have one + def splitAdoptUrl = ((String)ADOPT_DEFAULTS_JSON['repository']['build_url']) - ('.git').split('/') + // e.g. https://github.com/adoptium/temurin-build.git will produce adoptium/temurin-build + String userOrgRepo = "${splitAdoptUrl[splitAdoptUrl.size() - 2]}/${splitAdoptUrl[splitAdoptUrl.size() - 1]}" + // e.g. adoptium/temurin-build/master/build-farm/platform-specific-configurations + envVars.add("ADOPT_PLATFORM_CONFIG_LOCATION=${userOrgRepo}/${adoptBranch}/${ADOPT_DEFAULTS_JSON['configDirectories']['platform']}" as String) + // SXAEC: temporary fudge as the container doesn't have ant in the path +// envVars.add("PATH=c:\\SXAEC;c:\\cygwin64\\bin;c:\\Windows\\System32;c:\\windows;c:\\Windows\\System32\\WindowsPowershell\\v1.0;c:\\Program Files\\Git\\bin;c:\\apache-ant\\apache-ant-1.10.5\\bin" as String) + context.withEnv(envVars) { + if (env.BUILD_ARGS != null && !env.BUILD_ARGS.isEmpty()) { + context.println "SXAEC: Adding " + env.BUILD_ARGS + " before starting secondary env context" + assembleBuildArgs = env.BUILD_ARGS + ' --assemble-exploded-image' + openjdk_build_dir_arg + } else { + context.println "SXAEC: Not adding additional build args before starting secondary env context" + assembleBuildArgs = '--assemble-exploded-image' + openjdk_build_dir_arg + } + context.withEnv(['BUILD_ARGS=' + assembleBuildArgs]) { + context.println "SXAEC: Running with BUILD_ARGS = " + env.BUILD_ARGS + context.println '[CHECKOUT] Checking out to adoptium/temurin-build...' + def repoHandler = new RepoHandler(USER_REMOTE_CONFIGS, ADOPT_DEFAULTS_JSON, buildConfig.CI_REF, buildConfig.BUILD_REF) + repoHandler.checkoutAdoptBuild(context) + if ( buildConfig.TARGET_OS == 'windows' && buildConfig.DOCKER_IMAGE ) { + context.bat(script: 'bash -c "git config --global safe.directory $(cygpath ' + '\$' + '{WORKSPACE})"') + } + printGitRepoInfo() + context.println 'openjdk_build_pipeline.groovy: Assembling the exploded image' + // Call make-adopt-build-farm.sh on windows/mac to create signed tarball + try { + context.timeout(time: buildTimeouts.BUILD_JDK_TIMEOUT, unit: 'HOURS') { + context.println "openjdk_build_pipeline: calling MABF to assemble on win/mac JDK11+" + // SXAEC: Running ls -l here generates the shortname links required + // by the build and create paths referenced in the config.status file + if ( !context.isUnix() ) { + context.bat(script: 'ls -l /cygdrive/c "/cygdrive/c/Program Files (x86)" "/cygdrive/c/Program Files (x86)/Microsoft Visual Studio/2022" "/cygdrive/c/Program Files (x86)/Microsoft Visual Studio/2022/BuildTools/VC/Redist/MSVC" "/cygdrive/c/Program Files (x86)/Windows Kits/10/bin" "/cygdrive/c/Program Files (x86)/Microsoft Visual Studio/2022/BuildTools/VC/Tools/MSVC" "/cygdrive/c/Program Files (x86)/Windows Kits/10/include" "/cygdrive/c/Program Files (x86)/Windows Kits/10/lib"') + } + batOrSh("env") + batOrSh("bash ${ADOPT_DEFAULTS_JSON['scriptDirectories']['buildfarm']} --assemble-exploded-image") + } + } catch (FlowInterruptedException e) { + // Set Github Commit Status + if (env.JOB_NAME.contains('pr-tester')) { + updateGithubCommitStatus('FAILED', 'Build FAILED') + } + throw new Exception("[ERROR] Build JDK timeout (${buildTimeouts.BUILD_JDK_TIMEOUT} HOURS) has been reached. Exiting...") + } + } // context.withEnv(buildargs) + } // context.withEnv(envVars) + String versionOut + if (buildConfig.BUILD_ARGS.contains('--cross-compile')) { + context.println "[WARNING] Don't read faked version.txt on cross compiled build! Archiving early and running downstream job to retrieve java version..." + versionOut = readCrossCompiledVersionString() + } else { + versionOut = context.readFile('workspace/target/metadata/version.txt') + } + versionInfo = parseVersionOutput(versionOut) + writeMetadata(versionInfo, true) +// END + // Always archive any artifacts including failed make logs.. + try { + context.timeout(time: buildTimeouts.BUILD_ARCHIVE_TIMEOUT, unit: 'HOURS') { + // We have already archived cross compiled artifacts, so only archive the metadata files + if (buildConfig.BUILD_ARGS.contains('--cross-compile')) { + context.println '[INFO] Archiving JSON Files...' + context.archiveArtifacts artifacts: 'workspace/target/*.json' + } else { + context.archiveArtifacts artifacts: 'workspace/target/*' + } + } + } catch (FlowInterruptedException e) { + // Set Github Commit Status + if (env.JOB_NAME.contains('pr-tester')) { + updateGithubCommitStatus('FAILED', 'Build FAILED') + } + throw new Exception("[ERROR] Build archive timeout (${buildTimeouts.BUILD_ARCHIVE_TIMEOUT} HOURS) has been reached. Exiting...") + } + } // context.stage('assemble') +} // End of buildScriptsAssemble() 1643-1765 + /* Executed on a build node, the function checks out the repository and executes the build via ./make-adopt-build-farm.sh Once the build completes, it will calculate its version output, commit the first metadata writeout, and archive the build results. @@ -1574,7 +1793,7 @@ class Build { if (context.WORKSPACE != null && !context.WORKSPACE.isEmpty()) { context.println 'Cleaning workspace non-hidden files: ' + context.WORKSPACE + '/*' context.println 'SXA: batable 1540' - context.sh(script: 'rm -rf ' + context.WORKSPACE + '/*') + batOrSh(script: 'rm -rf ' + context.WORKSPACE + '/*') } else { context.println 'Warning: Unable to clean workspace as context.WORKSPACE is null/empty' } @@ -1601,8 +1820,11 @@ class Build { context.timeout(time: buildTimeouts.NODE_CLEAN_TIMEOUT, unit: 'HOURS') { if (context.WORKSPACE != null && !context.WORKSPACE.isEmpty()) { context.println 'Removing workspace openjdk build directory: ' + openjdk_build_dir - context.println 'SXA: batable and batted 1568 windbld#261,262' batOrSh('rm -rf ' + openjdk_build_dir) + batOrSh('ls -l ' + context.WORKSPACE + '/workspace/target || true') + context.println 'SXAEC: Clearing output artefacts to avoid duplicates being passed to the installer #680' + // SXAEC: This was to handle using none of the clean options when extracting tarballs ... + batOrSh('rm -rf ' + context.WORKSPACE + '/workspace/target/*') } else { context.println 'Warning: Unable to remove workspace openjdk build directory as context.WORKSPACE is null/empty' } @@ -1659,15 +1881,14 @@ class Build { if (env.JOB_NAME.contains('pr-tester')) { updateGithubCommitStatus('PENDING', 'Build Started') } - context.println "SXAEC: Before UASS" if (useAdoptShellScripts) { - context.println "SXAEC: UASS=true" context.println '[CHECKOUT] Checking out to adoptium/temurin-build...' repoHandler.checkoutAdoptBuild(context) printGitRepoInfo() +// START if ((buildConfig.TARGET_OS == 'mac' || buildConfig.TARGET_OS == 'windows') && buildConfig.JAVA_TO_BUILD != 'jdk8u' && enableSigner) { - context.println "Generating exploded build, sign JMODS, and assemble build, for platform ${buildConfig.TARGET_OS} version ${buildConfig.JAVA_TO_BUILD}" - def signBuildArgs + context.println "Generating exploded build" // , sign JMODS, and assemble build, for platform ${buildConfig.TARGET_OS} version ${buildConfig.JAVA_TO_BUILD}" + def signBuildArgs // Build args for make-adopt-build-farm.sh if (env.BUILD_ARGS != null && !env.BUILD_ARGS.isEmpty()) { signBuildArgs = env.BUILD_ARGS + ' --make-exploded-image' + openjdk_build_dir_arg } else { @@ -1677,14 +1898,22 @@ class Build { context.println 'Building an exploded image for signing' // Call make-adopt-build-farm.sh to do initial windows/mac build // windbld#254 - context.bat(script: "bash -c 'curl https://ci.adoptium.net/userContent/windows/openjdk-cached-workspace.configANDtargetANDbuild.tar.gz | tar -C /cygdrive/c/workspace/openjdk-build -xpzf -'") - // context.sh(script: "./${ADOPT_DEFAULTS_JSON['scriptDirectories']['buildfarm']}") + context.println "openjdk_build_pipeline: Calling MABF on win/mac to build exploded image" +// batOrSh("bash ./${ADOPT_DEFAULTS_JSON['scriptDirectories']['buildfarm']}") + // Use cached version from an attempt at the first phase only + batOrSh("ant --version") + context.bat(script: "bash -c 'curl https://ci.adoptium.net/userContent/windows/openjdk-cached-workspace-phase1+8.tar.gz | tar -C /cygdrive/c/workspace/openjdk-build -xzf -'") } def base_path = build_path if (openjdk_build_dir_arg == "") { // If not using a custom openjdk build dir, then query what autoconf created as the build sub-folder - context.println 'SXA: not batable 1648 - windbld#263' - base_path = context.sh(script: "ls -d ${build_path}/* | tr -d '\\n'", returnStdout:true) + if ( context.isUnix() ) { + context.println "Setting base path via sh" + base_path = context.sh(script: "ls -d ${build_path}/* | tr -d '\\n'", returnStdout:true) + } else { + context.println "Setting fixed base_path for now on Windows" + base_path = "workspace/build/src/build/windows-x86_64-server-release" + } } context.println "base build path for jmod signing = ${base_path}" context.stash name: 'jmods', @@ -1694,119 +1923,9 @@ class Build { // JDK 16 + jpackage needs to be signed as well stash the resources folder containing the executables "${base_path}/jdk/modules/jdk.jpackage/jdk/jpackage/internal/resources/*" - // Should this part be under "if (enableSigner)" instead - // of it being on the earlier "if" section? - context.node('eclipse-codesign') { - context.println 'SXA: batable-ish 1660' - context.sh "rm -rf ${base_path}/* || true" - - repoHandler.checkoutAdoptBuild(context) - printGitRepoInfo() - - // Copy pre assembled binary ready for JMODs to be codesigned - context.unstash 'jmods' - def target_os = "${buildConfig.TARGET_OS}" - context.withEnv(['base_os='+target_os, 'base_path='+base_path]) { - context.println 'SXA: not batable 1670' - // groovylint-disable - context.sh ''' - #!/bin/bash - set -eu - echo "Signing JMOD files under build path ${base_path} for base_os ${base_os}" - TMP_DIR="${base_path}/" - if [ "${base_os}" == "mac" ]; then - ENTITLEMENTS="$WORKSPACE/entitlements.plist" - FILES=$(find "${TMP_DIR}" -perm +111 -type f -o -name '*.dylib' -type f || find "${TMP_DIR}" -perm /111 -type f -o -name '*.dylib' -type f) - else - FILES=$(find "${TMP_DIR}" -type f -name '*.exe' -o -name '*.dll') - fi - for f in $FILES - do - echo "Signing $f using Eclipse Foundation codesign service" - dir=$(dirname "$f") - file=$(basename "$f") - mv "$f" "${dir}/unsigned_${file}" - success=false - if [ "${base_os}" == "mac" ]; then - if ! curl --fail --silent --show-error -o "$f" -F file="@${dir}/unsigned_${file}" -F entitlements="@$ENTITLEMENTS" https://cbi.eclipse.org/macos/codesign/sign; then - echo "curl command failed, sign of $f failed" - else - success=true - fi - else - if ! curl --fail --silent --show-error -o "$f" -F file="@${dir}/unsigned_${file}" https://cbi.eclipse.org/authenticode/sign; then - echo "curl command failed, sign of $f failed" - else - success=true - fi - fi - if [ $success == false ]; then - # Retry up to 20 times - max_iterations=20 - iteration=1 - echo "Code Not Signed For File $f" - while [ $iteration -le $max_iterations ] && [ $success = false ]; do - echo $iteration Of $max_iterations - sleep 1 - if [ "${base_os}" == "mac" ]; then - if curl --fail --silent --show-error -o "$f" -F file="@${dir}/unsigned_${file}" -F entitlements="@$ENTITLEMENTS" https://cbi.eclipse.org/macos/codesign/sign; then - success=true - fi - else - if curl --fail --silent --show-error -o "$f" -F file="@${dir}/unsigned_${file}" https://cbi.eclipse.org/authenticode/sign; then - success=true - fi - fi + // SXAEC: eclipse-codesign and assemble sections were inlined here - if [ $success = false ]; then - echo "curl command failed, $f Failed Signing On Attempt $iteration" - iteration=$((iteration+1)) - if [ $iteration -gt $max_iterations ] - then - echo "Errors Encountered During Signing" - exit 1 - fi - else - echo "$f Signed OK On Attempt $iteration" - fi - done - fi - chmod --reference="${dir}/unsigned_${file}" "$f" - rm -rf "${dir}/unsigned_${file}" - done - ''' - // groovylint-enable - } - context.stash name: 'signed_jmods', includes: "${base_path}/**/*" - } // context.node ("eclipse-codesign") - joe thinks it matches with something else though ... - // } // if (ENABLE_SIGN == true) - - // Remove jmod directories to be replaced with the stash saved above - context.println 'SXA: batable 1744' - context.sh "rm -rf ${base_path}/hotspot/variant-server || true" - context.sh "rm -rf ${base_path}/support/modules_cmds || true" - context.sh "rm -rf ${base_path}/support/modules_libs || true" - // JDK 16 + jpackage executables need to be signed as well - if (buildConfig.JAVA_TO_BUILD != 'jdk11u') { - context.sh "rm -rf ${base_path}/jdk/modules/jdk.jpackage/jdk/jpackage/internal/resources/* || true" - } - - // Restore signed JMODs - context.unstash 'signed_jmods' - - def assembleBuildArgs - if (env.BUILD_ARGS != null && !env.BUILD_ARGS.isEmpty()) { - assembleBuildArgs = env.BUILD_ARGS + ' --assemble-exploded-image' + openjdk_build_dir_arg - } else { - assembleBuildArgs = '--assemble-exploded-image' + openjdk_build_dir_arg - } - context.withEnv(['BUILD_ARGS=' + assembleBuildArgs]) { - context.println 'Assembling the exploded image' - // Call make-adopt-build-farm.sh on windows/mac to create signed tarball - context.println 'SXA: probably batable 1764' - context.sh(script: "./${ADOPT_DEFAULTS_JSON['scriptDirectories']['buildfarm']}") - } - } else { // Not Windows/Mac JDK11+ (i.e. doesn't require signing) + } else { // Not Windows/Mac JDK11+ (i.e. doesn't require internal signing) def buildArgs if (env.BUILD_ARGS != null && !env.BUILD_ARGS.isEmpty()) { buildArgs = env.BUILD_ARGS + openjdk_build_dir_arg @@ -1814,12 +1933,8 @@ class Build { buildArgs = openjdk_build_dir_arg } context.withEnv(['BUILD_ARGS=' + buildArgs]) { - context.println 'SXA: probably batable 1775' - // Call make-adopt-build-farm.sh to do one-step build (i.e. not signed) - // and when USEW_ADOPT_SHELL_SCRIPTS=false -// batOrSh("./${ADOPT_DEFAULTS_JSON['scriptDirectories']['buildfarm']}") -// context.sh(script: "./${ADOPT_DEFAULTS_JSON['scriptDirectories']['buildfarm']}") - context.bat(script: "bash -c 'curl https://ci.adoptium.net/userContent/windows/openjdk-cached-workspace.tar.gz | tar -C /cygdrive/c/workspace/openjdk-build -xpzf -'") + context.println "openjdk_build_pipeline: Calling MABF when not win/mac JDK11+ to do single-pass build and UASS=false" + batOrSh("bash ./${ADOPT_DEFAULTS_JSON['scriptDirectories']['buildfarm']}") } } context.println '[CHECKOUT] Reverting pre-build adoptium/temurin-build checkout...' @@ -1843,9 +1958,8 @@ class Build { buildArgs = openjdk_build_dir_arg } context.withEnv(['BUILD_ARGS=' + buildArgs]) { - context.println 'SXA: probably batable 1783' - // Call make-adopt-build-farm.sh when USE_ADOPT_SHELL_SCRIPTS==false - context.sh(script: "./${DEFAULTS_JSON['scriptDirectories']['buildfarm']}") + context.println "openjdk_build_pipeline: calling MABF to do single pass build when USE_ADOPT_SHELL_SCRIPTS is false" + batOrSh("bash ./${DEFAULTS_JSON['scriptDirectories']['buildfarm']}") // context.bat(script: "bash -c 'curl https://ci.adoptium.net/userContent/windows/openjdk-cached-workspace.tar.gz | tar -C /cygdrive/c/workspace/openjdk-build -xpzf -'") } context.println '[CHECKOUT] Reverting pre-build user temurin-build checkout...' @@ -1860,21 +1974,30 @@ class Build { } throw new Exception("[ERROR] Build JDK timeout (${buildTimeouts.BUILD_JDK_TIMEOUT} HOURS) has been reached. Exiting...") } - - // Run a downstream job on riscv machine that returns the java version. Otherwise, just read the version.txt - String versionOut - if (buildConfig.BUILD_ARGS.contains('--cross-compile')) { - context.println "[WARNING] Don't read faked version.txt on cross compiled build! Archiving early and running downstream job to retrieve java version..." - versionOut = readCrossCompiledVersionString() +context.println "SXAEC: Likely problem section" + if ((buildConfig.TARGET_OS == 'mac' || buildConfig.TARGET_OS == 'windows') && buildConfig.JAVA_TO_BUILD != 'jdk8u' && enableSigner) { + context.println "Signing phase required - skipping metadata reading" } else { - versionOut = context.readFile('workspace/target/metadata/version.txt') +context.println "SXAEC: Definite problem section :-)" + // Run a downstream job on riscv machine that returns the java version. Otherwise, just read the version.txt + String versionOut + if (buildConfig.BUILD_ARGS.contains('--cross-compile')) { + context.println "[WARNING] Don't read faked version.txt on cross compiled build! Archiving early and running downstream job to retrieve java version..." + versionOut = readCrossCompiledVersionString() + } else { + versionOut = context.readFile('workspace/target/metadata/version.txt') + } + versionInfo = parseVersionOutput(versionOut) } - - versionInfo = parseVersionOutput(versionOut) } - - writeMetadata(versionInfo, true) + if (!((buildConfig.TARGET_OS == 'mac' || buildConfig.TARGET_OS == 'windows') && buildConfig.JAVA_TO_BUILD != 'jdk8u' && enableSigner)) { + writeMetadata(versionInfo, true) + } else { + context.println "SXAEC: Skipping writing incomplete metadata - needs to be added to second phase" + } + } finally { + // Always archive any artifacts including failed make logs.. try { context.timeout(time: buildTimeouts.BUILD_ARCHIVE_TIMEOUT, unit: 'HOURS') { @@ -1903,7 +2026,6 @@ class Build { if (context.WORKSPACE != null && !context.WORKSPACE.isEmpty()) { if (cleanWorkspaceAfter) { context.println 'Cleaning workspace non-hidden files: ' + context.WORKSPACE + '/*' - context.println 'SXA: batable 1858' context.sh(script: 'rm -rf ' + context.WORKSPACE + '/*') // Clean remaining hidden files using cleanWs @@ -1914,7 +2036,9 @@ class Build { context.println "Failed to clean ${e}" } } else if (cleanWorkspaceBuildOutputAfter) { - context.println 'SXA: batable and batted 1869 windbld 266' + if ( enableSigning ) { + context.println 'ERROR? ENABLE_SIGNER and CLEAN_WORKSPACE_AFTER_BUILD both set' + } context.println 'Cleaning workspace build output files: ' + openjdk_build_dir batOrSh('rm -rf ' + openjdk_build_dir + ' ' + context.WORKSPACE + '/workspace/target ' + context.WORKSPACE + '/workspace/build/devkit ' + context.WORKSPACE + '/workspace/build/straceOutput') } @@ -2022,7 +2146,7 @@ class Build { Main function. This is what is executed remotely via the helper file kick_off_build.groovy, which is in turn executed by the downstream jobs. Running in downstream build job jdk-*-*-* called by kick_off_build.groovy */ - @SuppressWarnings('unused') +//SXAEC// @SuppressWarnings('unused') def build() { context.timestamps { try { @@ -2061,6 +2185,7 @@ class Build { } if (buildConfig.DOCKER_IMAGE) { + context.println "openjdk_build_pipeline: preparing to use docker image" // Docker build environment def label = buildConfig.NODE_LABEL + '&&dockerBuild' if (buildConfig.DOCKER_NODE) { @@ -2080,9 +2205,9 @@ class Build { context.node(label) { addNodeToBuildDescription() // Cannot clean workspace from inside docker container - context.println 'SXA: batable and batted 2042 (rm cyclonedx-lib)' + context.println 'SXAEC: batable and batted 2042 (rm cyclonedx-lib and security)' if ( buildConfig.TARGET_OS == 'windows' && buildConfig.DOCKER_IMAGE ) { - context.bat('rm -rf c:/workspace/openjdk-build/cyclonedx-lib') + context.bat('rm -rf c:/workspace/openjdk-build/cyclonedx-lib c:/workspace/openjdk-build/security') } if (cleanWorkspace) { try { @@ -2178,6 +2303,7 @@ class Build { dockerRunArg += " --userns keep-id:uid=1002,gid=1003" } if (buildConfig.TARGET_OS == 'windows') { + context.println "openjdk_build_pipeline: running exploded build in docker on Windows" def workspace = 'C:/workspace/openjdk-build/' context.echo("Switched to using non-default workspace path ${workspace}") context.ws(workspace) { @@ -2193,6 +2319,7 @@ class Build { } } } else { + context.println "openjdk_build_pipeline: running initial build in docker on non-windows" context.docker.image(buildConfig.DOCKER_IMAGE).inside(buildConfig.DOCKER_ARGS+" "+dockerRunArg) { buildScripts( cleanWorkspace, @@ -2203,14 +2330,31 @@ class Build { ) } } + if ( enableSigner ) { + context.println "openjdk_build_pipeline: running eclipse signing phase" + buildScriptsEclipseSigner() + def workspace = 'C:/workspace/openjdk-build/' + context.ws(workspace) { + context.println "Signing with non-default workspace location ${workspace}" + context.println "openjdk_build_pipeline: running assemble phase (invocation 1)" + context.docker.image(buildConfig.DOCKER_IMAGE).inside(buildConfig.DOCKER_ARGS+" "+dockerRunArg) { + buildScriptsAssemble( + cleanWorkspaceAfter, + filename, + useAdoptShellScripts + ) + } + } + } } } context.println "[NODE SHIFT] OUT OF DOCKER NODE (LABELNAME ${label}!)" // Build the jdk outside of docker container... } else { + context.println "openjdk_build_pipeline: running build without docker" waitForANodeToBecomeActive(buildConfig.NODE_LABEL) - context.println "[NODE SHIFT] MOVING INTO JENKINS NODE MATCHING LABELNAME ${buildConfig.NODE_LABEL}..." + context.println "openjdk_build_pipeline: [NODE SHIFT] MOVING INTO NON-DOCKER JENKINS NODE MATCHING LABELNAME ${buildConfig.NODE_LABEL}..." context.node(buildConfig.NODE_LABEL) { addNodeToBuildDescription() nonDockerNodeName = context.NODE_NAME @@ -2231,8 +2375,17 @@ class Build { filename, useAdoptShellScripts ) + if ( enableSigner ) { + buildScriptsEclipseSigner() + context.println "openjdk_build_pipeline: running assemble phase (invocation 2)" + buildScriptsAssemble( + cleanWorkspaceAfter, + filename, + useAdoptShellScripts + ) + } } - } else { + } else { // Non-windows, non-docker buildScripts( cleanWorkspace, cleanWorkspaceAfter, @@ -2240,6 +2393,15 @@ class Build { filename, useAdoptShellScripts ) + if ( enableSigner ) { + buildScriptsEclipseSigner() + context.println "openjdk_build_pipeline: running assemble phase (invocation 3)" + buildScriptsAssemble( + cleanWorkspaceAfter, + filename, + useAdoptShellScripts + ) + } } } context.println "[NODE SHIFT] OUT OF JENKINS NODE (LABELNAME ${buildConfig.NODE_LABEL}!)" @@ -2250,6 +2412,8 @@ class Build { if (enableSigner) { try { // Sign job timeout managed by Jenkins job config + context.println "openjdk_build_pipeline: executing signing phase" + sign(versionInfo) } catch (FlowInterruptedException e) { throw new Exception("[ERROR] Sign job timeout (${buildTimeouts.SIGN_JOB_TIMEOUT} HOURS) has been reached OR the downstream sign job failed. Exiting...") @@ -2260,6 +2424,7 @@ class Build { if (enableInstallers) { try { // Installer job timeout managed by Jenkins job config + context.println "openjdk_build_pipeline: building installers" buildInstaller(versionInfo) if ( enableSigner) { signInstaller(versionInfo) @@ -2271,6 +2436,7 @@ class Build { } if (!env.JOB_NAME.contains('pr-tester') && context.JENKINS_URL.contains('adopt')) { try { + context.println "openjdk_build_pipeline: Running GPG signing process" gpgSign() } catch (Exception e) { context.println(e.message) @@ -2282,6 +2448,7 @@ class Build { // Verify Windows and Mac Signing for Temurin if (buildConfig.VARIANT == 'temurin' && enableSigner) { try { + context.println "openjdk_build_pipeline: Verifying signing" verifySigning() } catch (Exception e) { context.println(e.message) @@ -2297,7 +2464,9 @@ class Build { } else { try { //Only smoke tests succeed TCK and AQA tests will be triggerred. + context.println "openjdk_build_pipeline: running smoke tests" if (runSmokeTests() == 'SUCCESS') { + context.println "openjdk_build_pipeline: smoke tests OK - running full AQA suite" // Remote trigger Eclipse Temurin JCK tests if (buildConfig.VARIANT == 'temurin' && enableTCK) { def platform = '' @@ -2308,7 +2477,7 @@ class Build { } if ( !(platform =='aarch64_windows') ) { if ( !(buildConfig.JAVA_TO_BUILD == 'jdk8u' && platform == 's390x_linux') ) { - context.echo "Remote trigger Eclipse Temurin AQA_Test_Pipeline job with ${platform} ${buildConfig.JAVA_TO_BUILD}" + context.echo "openjdk_build_pipeline: Remote trigger Eclipse Temurin AQA_Test_Pipeline job with ${platform} ${buildConfig.JAVA_TO_BUILD}" def remoteTargets = remoteTriggerJckTests(platform, filename) context.parallel remoteTargets } @@ -2345,7 +2514,7 @@ class Build { } } } - +// } // assemble stage } return { From 3a0af97b11e06d38ea5afe234bd2af2d8a3cf303 Mon Sep 17 00:00:00 2001 From: Stewart X Addison Date: Thu, 26 Sep 2024 14:16:14 +0100 Subject: [PATCH 03/19] De-duplicate envVars setup Signed-off-by: Stewart X Addison --- .../common/openjdk_build_pipeline.groovy | 93 ++++++++----------- 1 file changed, 37 insertions(+), 56 deletions(-) diff --git a/pipelines/build/common/openjdk_build_pipeline.groovy b/pipelines/build/common/openjdk_build_pipeline.groovy index 78478ad04..85782ef7c 100644 --- a/pipelines/build/common/openjdk_build_pipeline.groovy +++ b/pipelines/build/common/openjdk_build_pipeline.groovy @@ -1632,8 +1632,8 @@ class Build { def buildScriptsAssemble( cleanWorkspaceAfter, - filename, - useAdoptShellScripts + useAdoptShellScripts, + buildConfigEnvVars ) { def build_path def openjdk_build_dir @@ -1660,19 +1660,8 @@ def buildScriptsAssemble( // Restore signed JMODs context.unstash 'signed_jmods' // Convert IndividualBuildConfig to jenkins env variables - List envVars = buildConfig.toEnvVars() - envVars.add("FILENAME=${filename}" as String) - // Use BUILD_REF override if specified - def adoptBranch = buildConfig.BUILD_REF ?: ADOPT_DEFAULTS_JSON['repository']['build_branch'] - // Add platform config path so it can be used if the user doesn't have one - def splitAdoptUrl = ((String)ADOPT_DEFAULTS_JSON['repository']['build_url']) - ('.git').split('/') - // e.g. https://github.com/adoptium/temurin-build.git will produce adoptium/temurin-build - String userOrgRepo = "${splitAdoptUrl[splitAdoptUrl.size() - 2]}/${splitAdoptUrl[splitAdoptUrl.size() - 1]}" - // e.g. adoptium/temurin-build/master/build-farm/platform-specific-configurations - envVars.add("ADOPT_PLATFORM_CONFIG_LOCATION=${userOrgRepo}/${adoptBranch}/${ADOPT_DEFAULTS_JSON['configDirectories']['platform']}" as String) // SXAEC: temporary fudge as the container doesn't have ant in the path -// envVars.add("PATH=c:\\SXAEC;c:\\cygwin64\\bin;c:\\Windows\\System32;c:\\windows;c:\\Windows\\System32\\WindowsPowershell\\v1.0;c:\\Program Files\\Git\\bin;c:\\apache-ant\\apache-ant-1.10.5\\bin" as String) - context.withEnv(envVars) { + context.withEnv(buildConfigEnvVars) { if (env.BUILD_ARGS != null && !env.BUILD_ARGS.isEmpty()) { context.println "SXAEC: Adding " + env.BUILD_ARGS + " before starting secondary env context" assembleBuildArgs = env.BUILD_ARGS + ' --assemble-exploded-image' + openjdk_build_dir_arg @@ -1710,7 +1699,7 @@ def buildScriptsAssemble( throw new Exception("[ERROR] Build JDK timeout (${buildTimeouts.BUILD_JDK_TIMEOUT} HOURS) has been reached. Exiting...") } } // context.withEnv(buildargs) - } // context.withEnv(envVars) + } // context.withEnv(buildConfigEnvVars) String versionOut if (buildConfig.BUILD_ARGS.contains('--cross-compile')) { context.println "[WARNING] Don't read faked version.txt on cross compiled build! Archiving early and running downstream job to retrieve java version..." @@ -1752,9 +1741,9 @@ def buildScriptsAssemble( cleanWorkspace, cleanWorkspaceAfter, cleanWorkspaceBuildOutputAfter, - filename, useAdoptShellScripts, - enableSigner + enableSigner, + buildConfigEnvVars ) { return context.stage('build') { // Create the repo handler with the user's defaults to ensure a temurin-build checkout is not null @@ -1792,7 +1781,6 @@ def buildScriptsAssemble( // Issue: https://issues.jenkins.io/browse/JENKINS-64779 if (context.WORKSPACE != null && !context.WORKSPACE.isEmpty()) { context.println 'Cleaning workspace non-hidden files: ' + context.WORKSPACE + '/*' - context.println 'SXA: batable 1540' batOrSh(script: 'rm -rf ' + context.WORKSPACE + '/*') } else { context.println 'Warning: Unable to clean workspace as context.WORKSPACE is null/empty' @@ -1821,10 +1809,8 @@ def buildScriptsAssemble( if (context.WORKSPACE != null && !context.WORKSPACE.isEmpty()) { context.println 'Removing workspace openjdk build directory: ' + openjdk_build_dir batOrSh('rm -rf ' + openjdk_build_dir) - batOrSh('ls -l ' + context.WORKSPACE + '/workspace/target || true') - context.println 'SXAEC: Clearing output artefacts to avoid duplicates being passed to the installer #680' - // SXAEC: This was to handle using none of the clean options when extracting tarballs ... - batOrSh('rm -rf ' + context.WORKSPACE + '/workspace/target/*') + // SXAEC: This handles when none of the clean options are used when extracting cached tarballs ... + batOrSh('rm -rf ' + context.WORKSPACE + '/workspace/target/*') } else { context.println 'Warning: Unable to remove workspace openjdk build directory as context.WORKSPACE is null/empty' } @@ -1860,21 +1846,8 @@ def buildScriptsAssemble( try { // Convert IndividualBuildConfig to jenkins env variables - List envVars = buildConfig.toEnvVars() - envVars.add("FILENAME=${filename}" as String) - - // Use BUILD_REF override if specified - def adoptBranch = buildConfig.BUILD_REF ?: ADOPT_DEFAULTS_JSON['repository']['build_branch'] - - // Add platform config path so it can be used if the user doesn't have one - def splitAdoptUrl = ((String)ADOPT_DEFAULTS_JSON['repository']['build_url']) - ('.git').split('/') - // e.g. https://github.com/adoptium/temurin-build.git will produce adoptium/temurin-build - String userOrgRepo = "${splitAdoptUrl[splitAdoptUrl.size() - 2]}/${splitAdoptUrl[splitAdoptUrl.size() - 1]}" - // e.g. adoptium/temurin-build/master/build-farm/platform-specific-configurations - envVars.add("ADOPT_PLATFORM_CONFIG_LOCATION=${userOrgRepo}/${adoptBranch}/${ADOPT_DEFAULTS_JSON['configDirectories']['platform']}" as String) - // Execute build - context.withEnv(envVars) { + context.withEnv(buildConfigEnvVars) { try { context.timeout(time: buildTimeouts.BUILD_JDK_TIMEOUT, unit: 'HOURS') { // Set Github Commit Status @@ -1885,7 +1858,6 @@ def buildScriptsAssemble( context.println '[CHECKOUT] Checking out to adoptium/temurin-build...' repoHandler.checkoutAdoptBuild(context) printGitRepoInfo() -// START if ((buildConfig.TARGET_OS == 'mac' || buildConfig.TARGET_OS == 'windows') && buildConfig.JAVA_TO_BUILD != 'jdk8u' && enableSigner) { context.println "Generating exploded build" // , sign JMODS, and assemble build, for platform ${buildConfig.TARGET_OS} version ${buildConfig.JAVA_TO_BUILD}" def signBuildArgs // Build args for make-adopt-build-farm.sh @@ -1901,7 +1873,6 @@ def buildScriptsAssemble( context.println "openjdk_build_pipeline: Calling MABF on win/mac to build exploded image" // batOrSh("bash ./${ADOPT_DEFAULTS_JSON['scriptDirectories']['buildfarm']}") // Use cached version from an attempt at the first phase only - batOrSh("ant --version") context.bat(script: "bash -c 'curl https://ci.adoptium.net/userContent/windows/openjdk-cached-workspace-phase1+8.tar.gz | tar -C /cygdrive/c/workspace/openjdk-build -xzf -'") } def base_path = build_path @@ -1974,11 +1945,9 @@ def buildScriptsAssemble( } throw new Exception("[ERROR] Build JDK timeout (${buildTimeouts.BUILD_JDK_TIMEOUT} HOURS) has been reached. Exiting...") } -context.println "SXAEC: Likely problem section" if ((buildConfig.TARGET_OS == 'mac' || buildConfig.TARGET_OS == 'windows') && buildConfig.JAVA_TO_BUILD != 'jdk8u' && enableSigner) { context.println "Signing phase required - skipping metadata reading" } else { -context.println "SXAEC: Definite problem section :-)" // Run a downstream job on riscv machine that returns the java version. Otherwise, just read the version.txt String versionOut if (buildConfig.BUILD_ARGS.contains('--cross-compile')) { @@ -2172,6 +2141,18 @@ context.println "SXAEC: Definite problem section :-)" def helperRef = buildConfig.HELPER_REF ?: DEFAULTS_JSON['repository']['helper_ref'] def nonDockerNodeName = '' + // Convert IndividualBuildConfig to jenkins env variables + List envVars = buildConfig.toEnvVars() + envVars.add("FILENAME=${filename}" as String) + // Use BUILD_REF override if specified + def adoptBranch = buildConfig.BUILD_REF ?: ADOPT_DEFAULTS_JSON['repository']['build_branch'] + // Add platform config path so it can be used if the user doesn't have one + def splitAdoptUrl = ((String)ADOPT_DEFAULTS_JSON['repository']['build_url']) - ('.git').split('/') + // e.g. https://github.com/adoptium/temurin-build.git will produce adoptium/temurin-build + String userOrgRepo = "${splitAdoptUrl[splitAdoptUrl.size() - 2]}/${splitAdoptUrl[splitAdoptUrl.size() - 1]}" + // e.g. adoptium/temurin-build/master/build-farm/platform-specific-configurations + envVars.add("ADOPT_PLATFORM_CONFIG_LOCATION=${userOrgRepo}/${adoptBranch}/${ADOPT_DEFAULTS_JSON['configDirectories']['platform']}" as String) + context.stage('queue') { /* This loads the library containing two Helper classes, and causes them to be imported/updated from their repo. Without the library being imported here, runTests method will fail to execute the post-build test jobs for reasons unknown.*/ @@ -2286,9 +2267,9 @@ context.println "SXAEC: Definite problem section :-)" cleanWorkspace, cleanWorkspaceAfter, cleanWorkspaceBuildOutputAfter, - filename, useAdoptShellScripts, - enableSigner + enableSigner, + envVars ) } } else { @@ -2312,9 +2293,9 @@ context.println "SXAEC: Definite problem section :-)" cleanWorkspace, cleanWorkspaceAfter, cleanWorkspaceBuildOutputAfter, - filename, useAdoptShellScripts, - enableSigner + enableSigner, + envVars ) } } @@ -2325,8 +2306,8 @@ context.println "SXAEC: Definite problem section :-)" cleanWorkspace, cleanWorkspaceAfter, cleanWorkspaceBuildOutputAfter, - filename, - useAdoptShellScripts + useAdoptShellScripts, + envVars ) } } @@ -2340,8 +2321,8 @@ context.println "SXAEC: Definite problem section :-)" context.docker.image(buildConfig.DOCKER_IMAGE).inside(buildConfig.DOCKER_ARGS+" "+dockerRunArg) { buildScriptsAssemble( cleanWorkspaceAfter, - filename, - useAdoptShellScripts + useAdoptShellScripts, + envVars ) } } @@ -2372,16 +2353,16 @@ context.println "SXAEC: Definite problem section :-)" cleanWorkspace, cleanWorkspaceAfter, cleanWorkspaceBuildOutputAfter, - filename, - useAdoptShellScripts + useAdoptShellScripts, + envVars ) if ( enableSigner ) { buildScriptsEclipseSigner() context.println "openjdk_build_pipeline: running assemble phase (invocation 2)" buildScriptsAssemble( cleanWorkspaceAfter, - filename, - useAdoptShellScripts + useAdoptShellScripts, + envVars ) } } @@ -2390,16 +2371,16 @@ context.println "SXAEC: Definite problem section :-)" cleanWorkspace, cleanWorkspaceAfter, cleanWorkspaceBuildOutputAfter, - filename, - useAdoptShellScripts + useAdoptShellScripts, + envVars ) if ( enableSigner ) { buildScriptsEclipseSigner() context.println "openjdk_build_pipeline: running assemble phase (invocation 3)" buildScriptsAssemble( cleanWorkspaceAfter, - filename, - useAdoptShellScripts + useAdoptShellScripts, + envVars ) } } From 9293f692b199470fa761e93a30a25d6df8f3b596 Mon Sep 17 00:00:00 2001 From: Stewart X Addison Date: Thu, 26 Sep 2024 16:02:30 +0100 Subject: [PATCH 04/19] Updates following PR creation (inc.linter fixes) Signed-off-by: Stewart X Addison --- .../common/openjdk_build_pipeline.groovy | 188 +++++++++--------- 1 file changed, 93 insertions(+), 95 deletions(-) diff --git a/pipelines/build/common/openjdk_build_pipeline.groovy b/pipelines/build/common/openjdk_build_pipeline.groovy index 85782ef7c..fcbd2ae4c 100644 --- a/pipelines/build/common/openjdk_build_pipeline.groovy +++ b/pipelines/build/common/openjdk_build_pipeline.groovy @@ -1521,15 +1521,13 @@ class Build { def buildScriptsEclipseSigner() { def build_path - def openjdk_build_dir // openjdk_build_dir_arg unused after https://github.com/adoptium/ci-jenkins-pipelines/pull/1084 // Ref https://adoptium.slack.com/archives/C09NW3L2J/p1725957286184479?thread_ts=1725900849.766449&cid=C09NW3L2J // def openjdk_build_dir_arg build_path = 'workspace/build/src/build' - openjdk_build_dir = context.WORKSPACE + '/' + build_path // openjdk_build_dir_arg = "" - // SXA: This doesn't seem to set correctly with 'def base_path = build_path" + // SXA: This did not seem to set correctly with 'def base_path = build_path" def base_path base_path = build_path // if (openjdk_build_dir_arg == "") { @@ -1539,8 +1537,6 @@ class Build { // } def repoHandler = new RepoHandler(USER_REMOTE_CONFIGS, ADOPT_DEFAULTS_JSON, buildConfig.CI_REF, buildConfig.BUILD_REF) context.stage('internal sign') { - // SXAEC: Should this part be under "if (enableSigner)" instead - // of it being on the earlier "if" section? context.node('eclipse-codesign') { // Safety first! if (base_path != null && !base_path.isEmpty()) { @@ -1548,7 +1544,6 @@ class Build { } repoHandler.checkoutAdoptBuild(context) -// SXA - skipping as it's batOrSh and we can't allow bat here printGitRepoInfo() // Copy pre assembled binary ready for JMODs to be codesigned @@ -1631,18 +1626,13 @@ class Build { } def buildScriptsAssemble( + buildConfigEnvVars, cleanWorkspaceAfter, - useAdoptShellScripts, - buildConfigEnvVars + cleanWorkspaceBuildOutputAfter ) { def build_path - def openjdk_build_dir - def openjdk_build_dir_arg - openjdk_build_dir_arg="" build_path = 'workspace/build/src/build' - openjdk_build_dir = context.WORKSPACE + '/' + build_path - // SXA: This doesn't seem to set correctly with 'def base_path = build_path" def base_path base_path = build_path def assembleBuildArgs @@ -1654,51 +1644,47 @@ def buildScriptsAssemble( } context.stage('assemble') { if ( buildConfig.TARGET_OS == 'windows' && buildConfig.DOCKER_IMAGE ) { - // SXAC: Still TBC on this to determine if something fails without it + // SXAEC: Still TBC on this to determine if something fails without it + // Ref https://github.com/adoptium/infrastructure/issues/3723 context.bat('chmod -R a+rwX /cygdrive/c/workspace/openjdk-build/workspace/build/src/build & echo Done & exit 0') } // Restore signed JMODs context.unstash 'signed_jmods' // Convert IndividualBuildConfig to jenkins env variables - // SXAEC: temporary fudge as the container doesn't have ant in the path context.withEnv(buildConfigEnvVars) { - if (env.BUILD_ARGS != null && !env.BUILD_ARGS.isEmpty()) { - context.println "SXAEC: Adding " + env.BUILD_ARGS + " before starting secondary env context" - assembleBuildArgs = env.BUILD_ARGS + ' --assemble-exploded-image' + openjdk_build_dir_arg - } else { - context.println "SXAEC: Not adding additional build args before starting secondary env context" - assembleBuildArgs = '--assemble-exploded-image' + openjdk_build_dir_arg - } - context.withEnv(['BUILD_ARGS=' + assembleBuildArgs]) { - context.println "SXAEC: Running with BUILD_ARGS = " + env.BUILD_ARGS - context.println '[CHECKOUT] Checking out to adoptium/temurin-build...' - def repoHandler = new RepoHandler(USER_REMOTE_CONFIGS, ADOPT_DEFAULTS_JSON, buildConfig.CI_REF, buildConfig.BUILD_REF) - repoHandler.checkoutAdoptBuild(context) - if ( buildConfig.TARGET_OS == 'windows' && buildConfig.DOCKER_IMAGE ) { - context.bat(script: 'bash -c "git config --global safe.directory $(cygpath ' + '\$' + '{WORKSPACE})"') + if (env.BUILD_ARGS != null && !env.BUILD_ARGS.isEmpty()) { + assembleBuildArgs = env.BUILD_ARGS + ' --assemble-exploded-image' + } else { + assembleBuildArgs = '--assemble-exploded-image' } - printGitRepoInfo() - context.println 'openjdk_build_pipeline.groovy: Assembling the exploded image' - // Call make-adopt-build-farm.sh on windows/mac to create signed tarball - try { - context.timeout(time: buildTimeouts.BUILD_JDK_TIMEOUT, unit: 'HOURS') { - context.println "openjdk_build_pipeline: calling MABF to assemble on win/mac JDK11+" - // SXAEC: Running ls -l here generates the shortname links required - // by the build and create paths referenced in the config.status file - if ( !context.isUnix() ) { - context.bat(script: 'ls -l /cygdrive/c "/cygdrive/c/Program Files (x86)" "/cygdrive/c/Program Files (x86)/Microsoft Visual Studio/2022" "/cygdrive/c/Program Files (x86)/Microsoft Visual Studio/2022/BuildTools/VC/Redist/MSVC" "/cygdrive/c/Program Files (x86)/Windows Kits/10/bin" "/cygdrive/c/Program Files (x86)/Microsoft Visual Studio/2022/BuildTools/VC/Tools/MSVC" "/cygdrive/c/Program Files (x86)/Windows Kits/10/include" "/cygdrive/c/Program Files (x86)/Windows Kits/10/lib"') - } - batOrSh("env") - batOrSh("bash ${ADOPT_DEFAULTS_JSON['scriptDirectories']['buildfarm']} --assemble-exploded-image") + context.withEnv(['BUILD_ARGS=' + assembleBuildArgs]) { + context.println '[CHECKOUT] Checking out to adoptium/temurin-build...' + def repoHandler = new RepoHandler(USER_REMOTE_CONFIGS, ADOPT_DEFAULTS_JSON, buildConfig.CI_REF, buildConfig.BUILD_REF) + repoHandler.checkoutAdoptBuild(context) + if ( buildConfig.TARGET_OS == 'windows' && buildConfig.DOCKER_IMAGE ) { + context.bat(script: 'bash -c "git config --global safe.directory $(cygpath ' + '\$' + '{WORKSPACE})"') } - } catch (FlowInterruptedException e) { - // Set Github Commit Status - if (env.JOB_NAME.contains('pr-tester')) { - updateGithubCommitStatus('FAILED', 'Build FAILED') + printGitRepoInfo() + context.println 'openjdk_build_pipeline.groovy: Assembling the exploded image' + // Call make-adopt-build-farm.sh on windows/mac to create signed tarball + try { + context.timeout(time: buildTimeouts.BUILD_JDK_TIMEOUT, unit: 'HOURS') { + context.println "openjdk_build_pipeline: calling MABF to assemble on win/mac JDK11+" + // SXAEC: Running ls -l here generates the shortname links required + // by the build and create paths referenced in the config.status file + if ( !context.isUnix() ) { + context.bat(script: 'ls -l /cygdrive/c "/cygdrive/c/Program Files (x86)" "/cygdrive/c/Program Files (x86)/Microsoft Visual Studio/2022" "/cygdrive/c/Program Files (x86)/Microsoft Visual Studio/2022/BuildTools/VC/Redist/MSVC" "/cygdrive/c/Program Files (x86)/Windows Kits/10/bin" "/cygdrive/c/Program Files (x86)/Microsoft Visual Studio/2022/BuildTools/VC/Tools/MSVC" "/cygdrive/c/Program Files (x86)/Windows Kits/10/include" "/cygdrive/c/Program Files (x86)/Windows Kits/10/lib"') + } + batOrSh("bash ${ADOPT_DEFAULTS_JSON['scriptDirectories']['buildfarm']} --assemble-exploded-image") + } + } catch (FlowInterruptedException e) { + // Set Github Commit Status + if (env.JOB_NAME.contains('pr-tester')) { + updateGithubCommitStatus('FAILED', 'Build FAILED') + } + throw new Exception("[ERROR] Build JDK timeout (${buildTimeouts.BUILD_JDK_TIMEOUT} HOURS) has been reached. Exiting...") } - throw new Exception("[ERROR] Build JDK timeout (${buildTimeouts.BUILD_JDK_TIMEOUT} HOURS) has been reached. Exiting...") - } - } // context.withEnv(buildargs) + } // context.withEnv(assembleBuildargs) } // context.withEnv(buildConfigEnvVars) String versionOut if (buildConfig.BUILD_ARGS.contains('--cross-compile')) { @@ -1709,7 +1695,6 @@ def buildScriptsAssemble( } versionInfo = parseVersionOutput(versionOut) writeMetadata(versionInfo, true) -// END // Always archive any artifacts including failed make logs.. try { context.timeout(time: buildTimeouts.BUILD_ARCHIVE_TIMEOUT, unit: 'HOURS') { @@ -1728,6 +1713,8 @@ def buildScriptsAssemble( } throw new Exception("[ERROR] Build archive timeout (${buildTimeouts.BUILD_ARCHIVE_TIMEOUT} HOURS) has been reached. Exiting...") } + postBuildWSclean(cleanWorkspaceAfter, cleanWorkspaceBuildOutputAfter) + } // context.stage('assemble') } // End of buildScriptsAssemble() 1643-1765 @@ -1894,7 +1881,7 @@ def buildScriptsAssemble( // JDK 16 + jpackage needs to be signed as well stash the resources folder containing the executables "${base_path}/jdk/modules/jdk.jpackage/jdk/jpackage/internal/resources/*" - // SXAEC: eclipse-codesign and assemble sections were inlined here + // SXAEC: eclipse-codesign and assemble sections were previously inlined here } else { // Not Windows/Mac JDK11+ (i.e. doesn't require internal signing) def buildArgs @@ -1986,43 +1973,10 @@ def buildScriptsAssemble( throw new Exception("[ERROR] Build archive timeout (${buildTimeouts.BUILD_ARCHIVE_TIMEOUT} HOURS) has been reached. Exiting...") } - // post-build workspace clean: - if (cleanWorkspaceAfter || cleanWorkspaceBuildOutputAfter) { - try { - context.timeout(time: buildTimeouts.NODE_CLEAN_TIMEOUT, unit: 'HOURS') { - // Note: Underlying org.apache DirectoryScanner used by cleanWs has a bug scanning where it misses files containing ".." so use rm -rf instead - // Issue: https://issues.jenkins.io/browse/JENKINS-64779 - if (context.WORKSPACE != null && !context.WORKSPACE.isEmpty()) { - if (cleanWorkspaceAfter) { - context.println 'Cleaning workspace non-hidden files: ' + context.WORKSPACE + '/*' - context.sh(script: 'rm -rf ' + context.WORKSPACE + '/*') - - // Clean remaining hidden files using cleanWs - try { - context.println 'Cleaning workspace hidden files using cleanWs: ' + context.WORKSPACE - context.cleanWs notFailBuild: true, disableDeferredWipeout: true, deleteDirs: true - } catch (e) { - context.println "Failed to clean ${e}" - } - } else if (cleanWorkspaceBuildOutputAfter) { - if ( enableSigning ) { - context.println 'ERROR? ENABLE_SIGNER and CLEAN_WORKSPACE_AFTER_BUILD both set' - } - context.println 'Cleaning workspace build output files: ' + openjdk_build_dir - batOrSh('rm -rf ' + openjdk_build_dir + ' ' + context.WORKSPACE + '/workspace/target ' + context.WORKSPACE + '/workspace/build/devkit ' + context.WORKSPACE + '/workspace/build/straceOutput') - } - } else { - context.println 'Warning: Unable to clean workspace as context.WORKSPACE is null/empty' - } - } - } catch (FlowInterruptedException e) { - // Set Github Commit Status - if (env.JOB_NAME.contains('pr-tester')) { - updateGithubCommitStatus('FAILED', 'Build FAILED') - } - throw new Exception("[ERROR] AIX clean workspace timeout (${buildTimeouts.AIX_CLEAN_TIMEOUT} HOURS) has been reached. Exiting...") - } + if ( !enableSigner ) { // Don't clean if we need the workspace for the later assemble phase + postBuildWSclean(cleanWorkspaceAfter, cleanWorkspaceBuildOutputAfter) } + // Set Github Commit Status if (env.JOB_NAME.contains('pr-tester')) { updateGithubCommitStatus('SUCCESS', 'Build PASSED') @@ -2111,6 +2065,47 @@ def buildScriptsAssemble( context.currentBuild.description = tmpDesc + "${context.NODE_NAME}" } + +def postBuildWSclean( + cleanWorkspaceAfter, + cleanWorkspaceBuildOutputAfter +) { + // post-build workspace clean: + if (cleanWorkspaceAfter || cleanWorkspaceBuildOutputAfter) { + try { + context.timeout(time: buildTimeouts.NODE_CLEAN_TIMEOUT, unit: 'HOURS') { + // Note: Underlying org.apache DirectoryScanner used by cleanWs has a bug scanning where it misses files containing ".." so use rm -rf instead + // Issue: https://issues.jenkins.io/browse/JENKINS-64779 + if (context.WORKSPACE != null && !context.WORKSPACE.isEmpty()) { + if (cleanWorkspaceAfter) { + context.println 'Cleaning workspace non-hidden files: ' + context.WORKSPACE + '/*' + context.sh(script: 'rm -rf ' + context.WORKSPACE + '/*') + + // Clean remaining hidden files using cleanWs + try { + context.println 'Cleaning workspace hidden files using cleanWs: ' + context.WORKSPACE + context.cleanWs notFailBuild: true, disableDeferredWipeout: true, deleteDirs: true + } catch (e) { + context.println "Failed to clean ${e}" + } + } else if (cleanWorkspaceBuildOutputAfter) { + context.println 'openjdk_build_pipelione: Cleaning workspace build output files' + batOrSh('rm -rf ' + context.WORKSPACE + '/workspace/build/src/build' + ' ' + context.WORKSPACE + '/workspace/target ' + context.WORKSPACE + '/workspace/build/devkit ' + context.WORKSPACE + '/workspace/build/straceOutput') + } + } else { + context.println 'Warning: Unable to clean workspace as context.WORKSPACE is null/empty' + } + } + } catch (FlowInterruptedException e) { + // Set Github Commit Status + if (env.JOB_NAME.contains('pr-tester')) { + updateGithubCommitStatus('FAILED', 'Build FAILED') + } + throw new Exception("[ERROR] AIX clean workspace timeout (${buildTimeouts.AIX_CLEAN_TIMEOUT} HOURS) has been reached. Exiting...") + } + } +} + /* Main function. This is what is executed remotely via the helper file kick_off_build.groovy, which is in turn executed by the downstream jobs. Running in downstream build job jdk-*-*-* called by kick_off_build.groovy @@ -2263,7 +2258,7 @@ def buildScriptsAssemble( } context.docker.build("build-image", "--build-arg image=${buildConfig.DOCKER_IMAGE} -f ${buildConfig.DOCKER_FILE} .").inside(buildConfig.DOCKER_ARGS) { - buildScripts( + buildScripts( cleanWorkspace, cleanWorkspaceAfter, cleanWorkspaceBuildOutputAfter, @@ -2307,6 +2302,7 @@ def buildScriptsAssemble( cleanWorkspaceAfter, cleanWorkspaceBuildOutputAfter, useAdoptShellScripts, + enableSigner, envVars ) } @@ -2320,9 +2316,9 @@ def buildScriptsAssemble( context.println "openjdk_build_pipeline: running assemble phase (invocation 1)" context.docker.image(buildConfig.DOCKER_IMAGE).inside(buildConfig.DOCKER_ARGS+" "+dockerRunArg) { buildScriptsAssemble( + envVars, cleanWorkspaceAfter, - useAdoptShellScripts, - envVars + cleanWorkspaceBuildOutputAfter ) } } @@ -2354,15 +2350,16 @@ def buildScriptsAssemble( cleanWorkspaceAfter, cleanWorkspaceBuildOutputAfter, useAdoptShellScripts, + enableSigner, envVars ) if ( enableSigner ) { buildScriptsEclipseSigner() context.println "openjdk_build_pipeline: running assemble phase (invocation 2)" buildScriptsAssemble( + envVars, cleanWorkspaceAfter, - useAdoptShellScripts, - envVars + cleanWorkspaceBuildOutputAfter ) } } @@ -2372,15 +2369,16 @@ def buildScriptsAssemble( cleanWorkspaceAfter, cleanWorkspaceBuildOutputAfter, useAdoptShellScripts, + enableSigner, envVars ) if ( enableSigner ) { buildScriptsEclipseSigner() context.println "openjdk_build_pipeline: running assemble phase (invocation 3)" buildScriptsAssemble( + envVars, cleanWorkspaceAfter, - useAdoptShellScripts, - envVars + cleanWorkspaceBuildOutputAfter ) } } @@ -2389,12 +2387,12 @@ def buildScriptsAssemble( } } + // Sign and archive jobs if needed if (enableSigner) { try { // Sign job timeout managed by Jenkins job config context.println "openjdk_build_pipeline: executing signing phase" - sign(versionInfo) } catch (FlowInterruptedException e) { throw new Exception("[ERROR] Sign job timeout (${buildTimeouts.SIGN_JOB_TIMEOUT} HOURS) has been reached OR the downstream sign job failed. Exiting...") From ee2633e7baae54f7045eff9802844fa66264b298 Mon Sep 17 00:00:00 2001 From: Stewart X Addison Date: Thu, 26 Sep 2024 16:02:30 +0100 Subject: [PATCH 05/19] Move post-build workspace clean code into build() Signed-off-by: Stewart X Addison --- .../common/openjdk_build_pipeline.groovy | 140 +++++++----------- 1 file changed, 56 insertions(+), 84 deletions(-) diff --git a/pipelines/build/common/openjdk_build_pipeline.groovy b/pipelines/build/common/openjdk_build_pipeline.groovy index fcbd2ae4c..d939c9610 100644 --- a/pipelines/build/common/openjdk_build_pipeline.groovy +++ b/pipelines/build/common/openjdk_build_pipeline.groovy @@ -1626,9 +1626,7 @@ class Build { } def buildScriptsAssemble( - buildConfigEnvVars, - cleanWorkspaceAfter, - cleanWorkspaceBuildOutputAfter + buildConfigEnvVars ) { def build_path @@ -1643,9 +1641,11 @@ def buildScriptsAssemble( batOrSh "rm -rf ${base_path}/jdk/modules/jdk.jpackage/jdk/jpackage/internal/resources/*" } context.stage('assemble') { - if ( buildConfig.TARGET_OS == 'windows' && buildConfig.DOCKER_IMAGE ) { + if ( buildConfig.TARGET_OS == 'windows' && buildConfig.DOCKER_IMAGE ) { // SXAEC: Still TBC on this to determine if something fails without it // Ref https://github.com/adoptium/infrastructure/issues/3723 + // Fails to unstash even in non-docker case without the chmod e.g. windbld#840 + context.bat('c:\\cygwin64\\bin\\find /cygdrive/c/workspace -name public_suffix_list.dat -ls') context.bat('chmod -R a+rwX /cygdrive/c/workspace/openjdk-build/workspace/build/src/build & echo Done & exit 0') } // Restore signed JMODs @@ -1670,9 +1670,9 @@ def buildScriptsAssemble( try { context.timeout(time: buildTimeouts.BUILD_JDK_TIMEOUT, unit: 'HOURS') { context.println "openjdk_build_pipeline: calling MABF to assemble on win/mac JDK11+" - // SXAEC: Running ls -l here generates the shortname links required - // by the build and create paths referenced in the config.status file - if ( !context.isUnix() ) { + if ( !context.isUnix() && buildConfig.DOCKER_IMAGE ) { + // SXAEC: Running ls -l here generates the shortname links required + // by the build and create paths referenced in the config.status file context.bat(script: 'ls -l /cygdrive/c "/cygdrive/c/Program Files (x86)" "/cygdrive/c/Program Files (x86)/Microsoft Visual Studio/2022" "/cygdrive/c/Program Files (x86)/Microsoft Visual Studio/2022/BuildTools/VC/Redist/MSVC" "/cygdrive/c/Program Files (x86)/Windows Kits/10/bin" "/cygdrive/c/Program Files (x86)/Microsoft Visual Studio/2022/BuildTools/VC/Tools/MSVC" "/cygdrive/c/Program Files (x86)/Windows Kits/10/include" "/cygdrive/c/Program Files (x86)/Windows Kits/10/lib"') } batOrSh("bash ${ADOPT_DEFAULTS_JSON['scriptDirectories']['buildfarm']} --assemble-exploded-image") @@ -1713,8 +1713,6 @@ def buildScriptsAssemble( } throw new Exception("[ERROR] Build archive timeout (${buildTimeouts.BUILD_ARCHIVE_TIMEOUT} HOURS) has been reached. Exiting...") } - postBuildWSclean(cleanWorkspaceAfter, cleanWorkspaceBuildOutputAfter) - } // context.stage('assemble') } // End of buildScriptsAssemble() 1643-1765 @@ -1726,19 +1724,16 @@ def buildScriptsAssemble( def buildScripts( cleanWorkspace, - cleanWorkspaceAfter, - cleanWorkspaceBuildOutputAfter, useAdoptShellScripts, enableSigner, buildConfigEnvVars ) { + // Create the repo handler with the user's defaults to ensure a temurin-build checkout is not null + // Pass actual ADOPT_DEFAULTS_JSON, and optional buildConfig CI and BUILD branch/tag overrides, + // so that RepoHandler checks out the desired repo correctly + def repoHandler = new RepoHandler(USER_REMOTE_CONFIGS, ADOPT_DEFAULTS_JSON, buildConfig.CI_REF, buildConfig.BUILD_REF) + repoHandler.setUserDefaultsJson(context, DEFAULTS_JSON['defaultsUrl']) return context.stage('build') { - // Create the repo handler with the user's defaults to ensure a temurin-build checkout is not null - // Pass actual ADOPT_DEFAULTS_JSON, and optional buildConfig CI and BUILD branch/tag overrides, - // so that RepoHandler checks out the desired repo correctly - def repoHandler = new RepoHandler(USER_REMOTE_CONFIGS, ADOPT_DEFAULTS_JSON, buildConfig.CI_REF, buildConfig.BUILD_REF) - repoHandler.setUserDefaultsJson(context, DEFAULTS_JSON['defaultsUrl']) - context.println 'USER_REMOTE_CONFIGS: ' context.println JsonOutput.toJson(USER_REMOTE_CONFIGS) context.println 'DEFAULTS_JSON: ' @@ -1858,9 +1853,9 @@ def buildScriptsAssemble( // Call make-adopt-build-farm.sh to do initial windows/mac build // windbld#254 context.println "openjdk_build_pipeline: Calling MABF on win/mac to build exploded image" -// batOrSh("bash ./${ADOPT_DEFAULTS_JSON['scriptDirectories']['buildfarm']}") + batOrSh("bash ./${ADOPT_DEFAULTS_JSON['scriptDirectories']['buildfarm']}") // Use cached version from an attempt at the first phase only - context.bat(script: "bash -c 'curl https://ci.adoptium.net/userContent/windows/openjdk-cached-workspace-phase1+8.tar.gz | tar -C /cygdrive/c/workspace/openjdk-build -xzf -'") +// context.bat(script: "bash -c 'curl https://ci.adoptium.net/userContent/windows/openjdk-cached-workspace-phase1+8.tar.gz | tar -C /cygdrive/c/workspace/openjdk-build -xzf -'") } def base_path = build_path if (openjdk_build_dir_arg == "") { @@ -1973,10 +1968,6 @@ def buildScriptsAssemble( throw new Exception("[ERROR] Build archive timeout (${buildTimeouts.BUILD_ARCHIVE_TIMEOUT} HOURS) has been reached. Exiting...") } - if ( !enableSigner ) { // Don't clean if we need the workspace for the later assemble phase - postBuildWSclean(cleanWorkspaceAfter, cleanWorkspaceBuildOutputAfter) - } - // Set Github Commit Status if (env.JOB_NAME.contains('pr-tester')) { updateGithubCommitStatus('SUCCESS', 'Build PASSED') @@ -2066,46 +2057,6 @@ def buildScriptsAssemble( } -def postBuildWSclean( - cleanWorkspaceAfter, - cleanWorkspaceBuildOutputAfter -) { - // post-build workspace clean: - if (cleanWorkspaceAfter || cleanWorkspaceBuildOutputAfter) { - try { - context.timeout(time: buildTimeouts.NODE_CLEAN_TIMEOUT, unit: 'HOURS') { - // Note: Underlying org.apache DirectoryScanner used by cleanWs has a bug scanning where it misses files containing ".." so use rm -rf instead - // Issue: https://issues.jenkins.io/browse/JENKINS-64779 - if (context.WORKSPACE != null && !context.WORKSPACE.isEmpty()) { - if (cleanWorkspaceAfter) { - context.println 'Cleaning workspace non-hidden files: ' + context.WORKSPACE + '/*' - context.sh(script: 'rm -rf ' + context.WORKSPACE + '/*') - - // Clean remaining hidden files using cleanWs - try { - context.println 'Cleaning workspace hidden files using cleanWs: ' + context.WORKSPACE - context.cleanWs notFailBuild: true, disableDeferredWipeout: true, deleteDirs: true - } catch (e) { - context.println "Failed to clean ${e}" - } - } else if (cleanWorkspaceBuildOutputAfter) { - context.println 'openjdk_build_pipelione: Cleaning workspace build output files' - batOrSh('rm -rf ' + context.WORKSPACE + '/workspace/build/src/build' + ' ' + context.WORKSPACE + '/workspace/target ' + context.WORKSPACE + '/workspace/build/devkit ' + context.WORKSPACE + '/workspace/build/straceOutput') - } - } else { - context.println 'Warning: Unable to clean workspace as context.WORKSPACE is null/empty' - } - } - } catch (FlowInterruptedException e) { - // Set Github Commit Status - if (env.JOB_NAME.contains('pr-tester')) { - updateGithubCommitStatus('FAILED', 'Build FAILED') - } - throw new Exception("[ERROR] AIX clean workspace timeout (${buildTimeouts.AIX_CLEAN_TIMEOUT} HOURS) has been reached. Exiting...") - } - } -} - /* Main function. This is what is executed remotely via the helper file kick_off_build.groovy, which is in turn executed by the downstream jobs. Running in downstream build job jdk-*-*-* called by kick_off_build.groovy @@ -2159,7 +2110,6 @@ def postBuildWSclean( updateGithubCommitStatus('PENDING', 'Pending') } } - if (buildConfig.DOCKER_IMAGE) { context.println "openjdk_build_pipeline: preparing to use docker image" // Docker build environment @@ -2181,7 +2131,6 @@ def postBuildWSclean( context.node(label) { addNodeToBuildDescription() // Cannot clean workspace from inside docker container - context.println 'SXAEC: batable and batted 2042 (rm cyclonedx-lib and security)' if ( buildConfig.TARGET_OS == 'windows' && buildConfig.DOCKER_IMAGE ) { context.bat('rm -rf c:/workspace/openjdk-build/cyclonedx-lib c:/workspace/openjdk-build/security') } @@ -2260,8 +2209,6 @@ def postBuildWSclean( context.docker.build("build-image", "--build-arg image=${buildConfig.DOCKER_IMAGE} -f ${buildConfig.DOCKER_FILE} .").inside(buildConfig.DOCKER_ARGS) { buildScripts( cleanWorkspace, - cleanWorkspaceAfter, - cleanWorkspaceBuildOutputAfter, useAdoptShellScripts, enableSigner, envVars @@ -2286,8 +2233,6 @@ def postBuildWSclean( context.docker.image(buildConfig.DOCKER_IMAGE).inside(buildConfig.DOCKER_ARGS+" "+dockerRunArg) { buildScripts( cleanWorkspace, - cleanWorkspaceAfter, - cleanWorkspaceBuildOutputAfter, useAdoptShellScripts, enableSigner, envVars @@ -2299,8 +2244,6 @@ def postBuildWSclean( context.docker.image(buildConfig.DOCKER_IMAGE).inside(buildConfig.DOCKER_ARGS+" "+dockerRunArg) { buildScripts( cleanWorkspace, - cleanWorkspaceAfter, - cleanWorkspaceBuildOutputAfter, useAdoptShellScripts, enableSigner, envVars @@ -2316,9 +2259,7 @@ def postBuildWSclean( context.println "openjdk_build_pipeline: running assemble phase (invocation 1)" context.docker.image(buildConfig.DOCKER_IMAGE).inside(buildConfig.DOCKER_ARGS+" "+dockerRunArg) { buildScriptsAssemble( - envVars, - cleanWorkspaceAfter, - cleanWorkspaceBuildOutputAfter + envVars ) } } @@ -2347,8 +2288,6 @@ def postBuildWSclean( context.ws(workspace) { buildScripts( cleanWorkspace, - cleanWorkspaceAfter, - cleanWorkspaceBuildOutputAfter, useAdoptShellScripts, enableSigner, envVars @@ -2357,17 +2296,13 @@ def postBuildWSclean( buildScriptsEclipseSigner() context.println "openjdk_build_pipeline: running assemble phase (invocation 2)" buildScriptsAssemble( - envVars, - cleanWorkspaceAfter, - cleanWorkspaceBuildOutputAfter + envVars ) } } } else { // Non-windows, non-docker buildScripts( cleanWorkspace, - cleanWorkspaceAfter, - cleanWorkspaceBuildOutputAfter, useAdoptShellScripts, enableSigner, envVars @@ -2376,9 +2311,7 @@ def postBuildWSclean( buildScriptsEclipseSigner() context.println "openjdk_build_pipeline: running assemble phase (invocation 3)" buildScriptsAssemble( - envVars, - cleanWorkspaceAfter, - cleanWorkspaceBuildOutputAfter + envVars ) } } @@ -2387,6 +2320,45 @@ def postBuildWSclean( } } + // post-build workspace clean: + context.node(label) { + if (cleanWorkspaceAfter || cleanWorkspaceBuildOutputAfter) { + try { + context.timeout(time: buildTimeouts.NODE_CLEAN_TIMEOUT, unit: 'HOURS') { + // Note: Underlying org.apache DirectoryScanner used by cleanWs has a bug scanning where it misses files containing ".." so use rm -rf instead + // Issue: https://issues.jenkins.io/browse/JENKINS-64779 + if (context.WORKSPACE != null && !context.WORKSPACE.isEmpty()) { + if (cleanWorkspaceAfter) { + context.println 'Cleaning workspace non-hidden files: ' + context.WORKSPACE + '/*' + context.sh(script: 'rm -rf ' + context.WORKSPACE + '/*') + + // Clean remaining hidden files using cleanWs + try { + context.println 'Cleaning workspace hidden files using cleanWs: ' + context.WORKSPACE + context.cleanWs notFailBuild: true, disableDeferredWipeout: true, deleteDirs: true + } catch (e) { + context.println "Failed to clean ${e}" + } + } else if (cleanWorkspaceBuildOutputAfter) { + if ( enableSigner ) { + context.println 'ERROR? ENABLE_SIGNER and CLEAN_WORKSPACE_AFTER_BUILD both set' + } + context.println 'Cleaning workspace build output files under ' + context.WORKSPACE + batOrSh('rm -rf ' + context.WORKSPACE + '/workspace/build/src/build ' + context.WORKSPACE + '/workspace/target ' + context.WORKSPACE + '/workspace/build/devkit ' + context.WORKSPACE + '/workspace/build/straceOutput') + } + } else { + context.println 'Warning: Unable to clean workspace as context.WORKSPACE is null/empty' + } + } + } catch (FlowInterruptedException e) { + // Set Github Commit Status + if (env.JOB_NAME.contains('pr-tester')) { + updateGithubCommitStatus('FAILED', 'Build FAILED') + } + throw new Exception("[ERROR] AIX clean workspace timeout (${buildTimeouts.AIX_CLEAN_TIMEOUT} HOURS) has been reached. Exiting...") + } + } + } // Sign and archive jobs if needed if (enableSigner) { From 9f2d0f3fcb238645f987289b3373b67720217488 Mon Sep 17 00:00:00 2001 From: Stewart X Addison Date: Tue, 8 Oct 2024 00:19:21 +0100 Subject: [PATCH 06/19] chmod -R at start of assemble phase even on non-docker Signed-off-by: Stewart X Addison --- pipelines/build/common/openjdk_build_pipeline.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pipelines/build/common/openjdk_build_pipeline.groovy b/pipelines/build/common/openjdk_build_pipeline.groovy index d939c9610..be40a8c1a 100644 --- a/pipelines/build/common/openjdk_build_pipeline.groovy +++ b/pipelines/build/common/openjdk_build_pipeline.groovy @@ -1641,12 +1641,13 @@ def buildScriptsAssemble( batOrSh "rm -rf ${base_path}/jdk/modules/jdk.jpackage/jdk/jpackage/internal/resources/*" } context.stage('assemble') { - if ( buildConfig.TARGET_OS == 'windows' && buildConfig.DOCKER_IMAGE ) { + if ( buildConfig.TARGET_OS == 'windows' ) { // SXAEC: Still TBC on this to determine if something fails without it // Ref https://github.com/adoptium/infrastructure/issues/3723 // Fails to unstash even in non-docker case without the chmod e.g. windbld#840 context.bat('c:\\cygwin64\\bin\\find /cygdrive/c/workspace -name public_suffix_list.dat -ls') context.bat('chmod -R a+rwX /cygdrive/c/workspace/openjdk-build/workspace/build/src/build & echo Done & exit 0') + context.bat('c:\\cygwin64\\bin\\find /cygdrive/c/workspace -name public_suffix_list.dat -ls') } // Restore signed JMODs context.unstash 'signed_jmods' From 3b8533b7969ede150f95e3ba3a81e103d44e79f6 Mon Sep 17 00:00:00 2001 From: Stewart X Addison Date: Fri, 11 Oct 2024 00:20:17 +0100 Subject: [PATCH 07/19] Separate clean workspace function Signed-off-by: Stewart X Addison --- .../common/openjdk_build_pipeline.groovy | 114 ++++++++++-------- 1 file changed, 67 insertions(+), 47 deletions(-) diff --git a/pipelines/build/common/openjdk_build_pipeline.groovy b/pipelines/build/common/openjdk_build_pipeline.groovy index be40a8c1a..811b261fd 100644 --- a/pipelines/build/common/openjdk_build_pipeline.groovy +++ b/pipelines/build/common/openjdk_build_pipeline.groovy @@ -1625,7 +1625,49 @@ class Build { } // context.stage } +def postBuildWSclean( + cleanWorkspaceAfter, + cleanWorkspaceBuildOutputAfter +) { + // post-build workspace clean: + if (cleanWorkspaceAfter || cleanWorkspaceBuildOutputAfter) { + try { + context.timeout(time: buildTimeouts.NODE_CLEAN_TIMEOUT, unit: 'HOURS') { + // Note: Underlying org.apache DirectoryScanner used by cleanWs has a bug scanning where it misses files containing ".." so use rm -rf instead + // Issue: https://issues.jenkins.io/browse/JENKINS-64779 + if (context.WORKSPACE != null && !context.WORKSPACE.isEmpty()) { + if (cleanWorkspaceAfter) { + context.println 'Cleaning workspace non-hidden files: ' + context.WORKSPACE + '/*' + context.sh(script: 'rm -rf ' + context.WORKSPACE + '/*') + + // Clean remaining hidden files using cleanWs + try { + context.println 'Cleaning workspace hidden files using cleanWs: ' + context.WORKSPACE + context.cleanWs notFailBuild: true, disableDeferredWipeout: true, deleteDirs: true + } catch (e) { + context.println "Failed to clean ${e}" + } + } else if (cleanWorkspaceBuildOutputAfter) { + context.println 'Cleaning workspace build output files under ' + context.WORKSPACE + batOrSh('rm -rf ' + context.WORKSPACE + '/workspace/build/src/build ' + context.WORKSPACE + '/workspace/target ' + context.WORKSPACE + '/workspace/build/devkit ' + context.WORKSPACE + '/workspace/build/straceOutput') + } + } else { + context.println 'Warning: Unable to clean workspace as context.WORKSPACE is null/empty' + } + } + } catch (FlowInterruptedException e) { + // Set Github Commit Status + if (env.JOB_NAME.contains('pr-tester')) { + updateGithubCommitStatus('FAILED', 'Build FAILED') + } + throw new Exception("[ERROR] AIX clean workspace timeout (${buildTimeouts.AIX_CLEAN_TIMEOUT} HOURS) has been reached. Exiting...") + } + } +} + def buildScriptsAssemble( + cleanWorkspaceAfter, + cleanWorkspaceBuildOutputAfter, buildConfigEnvVars ) { def build_path @@ -1645,9 +1687,7 @@ def buildScriptsAssemble( // SXAEC: Still TBC on this to determine if something fails without it // Ref https://github.com/adoptium/infrastructure/issues/3723 // Fails to unstash even in non-docker case without the chmod e.g. windbld#840 - context.bat('c:\\cygwin64\\bin\\find /cygdrive/c/workspace -name public_suffix_list.dat -ls') - context.bat('chmod -R a+rwX /cygdrive/c/workspace/openjdk-build/workspace/build/src/build & echo Done & exit 0') - context.bat('c:\\cygwin64\\bin\\find /cygdrive/c/workspace -name public_suffix_list.dat -ls') + context.bat('chmod -R a+rwX ' + '/cygdrive/c/workspace/openjdk-build/workspace/build/src/build/windows-x86_64-server-release') } // Restore signed JMODs context.unstash 'signed_jmods' @@ -1714,6 +1754,7 @@ def buildScriptsAssemble( } throw new Exception("[ERROR] Build archive timeout (${buildTimeouts.BUILD_ARCHIVE_TIMEOUT} HOURS) has been reached. Exiting...") } + postBuildWSclean(cleanWorkspaceAfter, cleanWorkspaceBuildOutputAfter) } // context.stage('assemble') } // End of buildScriptsAssemble() 1643-1765 @@ -1725,6 +1766,8 @@ def buildScriptsAssemble( def buildScripts( cleanWorkspace, + cleanWorkspaceAfter, + cleanWorkspaceOutputAfter, useAdoptShellScripts, enableSigner, buildConfigEnvVars @@ -1968,7 +2011,9 @@ def buildScriptsAssemble( } throw new Exception("[ERROR] Build archive timeout (${buildTimeouts.BUILD_ARCHIVE_TIMEOUT} HOURS) has been reached. Exiting...") } - + if ( !enableSigner ) { // Don't clean if we need the workspace for the later assemble phase + postBuildWSclean(cleanWorkspaceAfter, cleanWorkspaceBuildOutputAfter) + } // Set Github Commit Status if (env.JOB_NAME.contains('pr-tester')) { updateGithubCommitStatus('SUCCESS', 'Build PASSED') @@ -2057,7 +2102,6 @@ def buildScriptsAssemble( context.currentBuild.description = tmpDesc + "${context.NODE_NAME}" } - /* Main function. This is what is executed remotely via the helper file kick_off_build.groovy, which is in turn executed by the downstream jobs. Running in downstream build job jdk-*-*-* called by kick_off_build.groovy @@ -2208,8 +2252,10 @@ def buildScriptsAssemble( } context.docker.build("build-image", "--build-arg image=${buildConfig.DOCKER_IMAGE} -f ${buildConfig.DOCKER_FILE} .").inside(buildConfig.DOCKER_ARGS) { - buildScripts( + buildScripts( cleanWorkspace, + cleanWorkspaceAfter, + cleanWorkspaceBuildOutputAfter, useAdoptShellScripts, enableSigner, envVars @@ -2234,6 +2280,8 @@ def buildScriptsAssemble( context.docker.image(buildConfig.DOCKER_IMAGE).inside(buildConfig.DOCKER_ARGS+" "+dockerRunArg) { buildScripts( cleanWorkspace, + cleanWorkspaceAfter, + cleanWorkspaceBuildOutputAfter, useAdoptShellScripts, enableSigner, envVars @@ -2245,6 +2293,8 @@ def buildScriptsAssemble( context.docker.image(buildConfig.DOCKER_IMAGE).inside(buildConfig.DOCKER_ARGS+" "+dockerRunArg) { buildScripts( cleanWorkspace, + cleanWorkspaceAfter, + cleanWorkspaceBuildOutputAfter, useAdoptShellScripts, enableSigner, envVars @@ -2258,8 +2308,10 @@ def buildScriptsAssemble( context.ws(workspace) { context.println "Signing with non-default workspace location ${workspace}" context.println "openjdk_build_pipeline: running assemble phase (invocation 1)" - context.docker.image(buildConfig.DOCKER_IMAGE).inside(buildConfig.DOCKER_ARGS+" "+dockerRunArg) { + context.docker.image(buildConfig.DOCKER_IMAGE).inside(buildConfig.DOCKER_ARGS+" "+dockerRunArg) { buildScriptsAssemble( + cleanWorkspaceAfter, + cleanWorkspaceBuildOutputAfter, envVars ) } @@ -2289,6 +2341,8 @@ def buildScriptsAssemble( context.ws(workspace) { buildScripts( cleanWorkspace, + cleanWorkspaceAfter, + cleanWorkspaceBuildOutputAfter, useAdoptShellScripts, enableSigner, envVars @@ -2297,6 +2351,8 @@ def buildScriptsAssemble( buildScriptsEclipseSigner() context.println "openjdk_build_pipeline: running assemble phase (invocation 2)" buildScriptsAssemble( + cleanWorkspaceAfter, + cleanWorkspaceBuildOutputAfter, envVars ) } @@ -2304,6 +2360,8 @@ def buildScriptsAssemble( } else { // Non-windows, non-docker buildScripts( cleanWorkspace, + cleanWorkspaceAfter, + cleanWorkspaceBuildOutputAfter, useAdoptShellScripts, enableSigner, envVars @@ -2312,6 +2370,8 @@ def buildScriptsAssemble( buildScriptsEclipseSigner() context.println "openjdk_build_pipeline: running assemble phase (invocation 3)" buildScriptsAssemble( + cleanWorkspaceAfter, + cleanWorkspaceBuildOutputAfter, envVars ) } @@ -2321,46 +2381,6 @@ def buildScriptsAssemble( } } - // post-build workspace clean: - context.node(label) { - if (cleanWorkspaceAfter || cleanWorkspaceBuildOutputAfter) { - try { - context.timeout(time: buildTimeouts.NODE_CLEAN_TIMEOUT, unit: 'HOURS') { - // Note: Underlying org.apache DirectoryScanner used by cleanWs has a bug scanning where it misses files containing ".." so use rm -rf instead - // Issue: https://issues.jenkins.io/browse/JENKINS-64779 - if (context.WORKSPACE != null && !context.WORKSPACE.isEmpty()) { - if (cleanWorkspaceAfter) { - context.println 'Cleaning workspace non-hidden files: ' + context.WORKSPACE + '/*' - context.sh(script: 'rm -rf ' + context.WORKSPACE + '/*') - - // Clean remaining hidden files using cleanWs - try { - context.println 'Cleaning workspace hidden files using cleanWs: ' + context.WORKSPACE - context.cleanWs notFailBuild: true, disableDeferredWipeout: true, deleteDirs: true - } catch (e) { - context.println "Failed to clean ${e}" - } - } else if (cleanWorkspaceBuildOutputAfter) { - if ( enableSigner ) { - context.println 'ERROR? ENABLE_SIGNER and CLEAN_WORKSPACE_AFTER_BUILD both set' - } - context.println 'Cleaning workspace build output files under ' + context.WORKSPACE - batOrSh('rm -rf ' + context.WORKSPACE + '/workspace/build/src/build ' + context.WORKSPACE + '/workspace/target ' + context.WORKSPACE + '/workspace/build/devkit ' + context.WORKSPACE + '/workspace/build/straceOutput') - } - } else { - context.println 'Warning: Unable to clean workspace as context.WORKSPACE is null/empty' - } - } - } catch (FlowInterruptedException e) { - // Set Github Commit Status - if (env.JOB_NAME.contains('pr-tester')) { - updateGithubCommitStatus('FAILED', 'Build FAILED') - } - throw new Exception("[ERROR] AIX clean workspace timeout (${buildTimeouts.AIX_CLEAN_TIMEOUT} HOURS) has been reached. Exiting...") - } - } - } - // Sign and archive jobs if needed if (enableSigner) { try { From f648e2c276d4624e477822131048e9a52d763b55 Mon Sep 17 00:00:00 2001 From: Stewart X Addison Date: Fri, 11 Oct 2024 18:38:52 +0100 Subject: [PATCH 08/19] Reinstate windows release definition for jdk11u Signed-off-by: Stewart X Addison --- pipelines/build/common/openjdk_build_pipeline.groovy | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/pipelines/build/common/openjdk_build_pipeline.groovy b/pipelines/build/common/openjdk_build_pipeline.groovy index 811b261fd..27ed5f957 100644 --- a/pipelines/build/common/openjdk_build_pipeline.groovy +++ b/pipelines/build/common/openjdk_build_pipeline.groovy @@ -1687,7 +1687,7 @@ def buildScriptsAssemble( // SXAEC: Still TBC on this to determine if something fails without it // Ref https://github.com/adoptium/infrastructure/issues/3723 // Fails to unstash even in non-docker case without the chmod e.g. windbld#840 - context.bat('chmod -R a+rwX ' + '/cygdrive/c/workspace/openjdk-build/workspace/build/src/build/windows-x86_64-server-release') + context.bat('chmod -R a+rwX ' + '/cygdrive/c/workspace/openjdk-build/workspace/build/src/build/*') } // Restore signed JMODs context.unstash 'signed_jmods' @@ -1905,11 +1905,9 @@ def buildScriptsAssemble( if (openjdk_build_dir_arg == "") { // If not using a custom openjdk build dir, then query what autoconf created as the build sub-folder if ( context.isUnix() ) { - context.println "Setting base path via sh" - base_path = context.sh(script: "ls -d ${build_path}/* | tr -d '\\n'", returnStdout:true) + base_path = context.sh(script: "ls -d ${build_path}/*", returnStdout:true) } else { - context.println "Setting fixed base_path for now on Windows" - base_path = "workspace/build/src/build/windows-x86_64-server-release" + base_path = context.bat(script: "@ls -d ${build_path}/*", returnStdout:true).trim() } } context.println "base build path for jmod signing = ${base_path}" From 7d369aa1afdf3d4a548f5b76919a0b50f368e890 Mon Sep 17 00:00:00 2001 From: Stewart X Addison Date: Sat, 12 Oct 2024 11:24:32 +0100 Subject: [PATCH 09/19] Reduce stashing and avoid pre-unstash chmod in non-docker case Signed-off-by: Stewart X Addison --- .../build/common/openjdk_build_pipeline.groovy | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/pipelines/build/common/openjdk_build_pipeline.groovy b/pipelines/build/common/openjdk_build_pipeline.groovy index 27ed5f957..0ca7591fb 100644 --- a/pipelines/build/common/openjdk_build_pipeline.groovy +++ b/pipelines/build/common/openjdk_build_pipeline.groovy @@ -1683,7 +1683,7 @@ def buildScriptsAssemble( batOrSh "rm -rf ${base_path}/jdk/modules/jdk.jpackage/jdk/jpackage/internal/resources/*" } context.stage('assemble') { - if ( buildConfig.TARGET_OS == 'windows' ) { + if ( buildConfig.TARGET_OS == 'windows' && buildConfig.DOCKER_IMAGE) { // SXAEC: Still TBC on this to determine if something fails without it // Ref https://github.com/adoptium/infrastructure/issues/3723 // Fails to unstash even in non-docker case without the chmod e.g. windbld#840 @@ -1767,7 +1767,7 @@ def buildScriptsAssemble( def buildScripts( cleanWorkspace, cleanWorkspaceAfter, - cleanWorkspaceOutputAfter, + cleanWorkspaceBuildOutputAfter, useAdoptShellScripts, enableSigner, buildConfigEnvVars @@ -1912,9 +1912,15 @@ def buildScriptsAssemble( } context.println "base build path for jmod signing = ${base_path}" context.stash name: 'jmods', - includes: "${base_path}/hotspot/variant-server/**/*," + - "${base_path}/support/modules_cmds/**/*," + - "${base_path}/support/modules_libs/**/*," + + includes: "${base_path}/hotspot/variant-server/**/*.exe," + + "${base_path}/hotspot/variant-server/**/*.dll," + + "${base_path}/hotspot/variant-server/**/*.dylib," + + "${base_path}/support/modules_cmds/**/*.exe," + + "${base_path}/support/modules_cmds/**/*.dll," + + "${base_path}/support/modules_cmds/**/*.dylib," + + "${base_path}/support/modules_libs/**/*.exe," + + "${base_path}/support/modules_libs/**/*.dll," + + "${base_path}/support/modules_libs/**/*.dylib," + // JDK 16 + jpackage needs to be signed as well stash the resources folder containing the executables "${base_path}/jdk/modules/jdk.jpackage/jdk/jpackage/internal/resources/*" From 18be2a507c50c7169166281e5a4fa157886cc55a Mon Sep 17 00:00:00 2001 From: Stewart X Addison Date: Sat, 12 Oct 2024 12:16:16 +0100 Subject: [PATCH 10/19] Disable internal sign phase on jdk8u Signed-off-by: Stewart X Addison --- pipelines/build/common/openjdk_build_pipeline.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pipelines/build/common/openjdk_build_pipeline.groovy b/pipelines/build/common/openjdk_build_pipeline.groovy index 0ca7591fb..95448eafb 100644 --- a/pipelines/build/common/openjdk_build_pipeline.groovy +++ b/pipelines/build/common/openjdk_build_pipeline.groovy @@ -2305,7 +2305,8 @@ def buildScriptsAssemble( ) } } - if ( enableSigner ) { + // Is thre potential for not enabling the signer on jdk8u instead of having this clause? + if ( enableSigner && buildConfig.JAVA_TO_BUILD != 'jdk8u' ) { context.println "openjdk_build_pipeline: running eclipse signing phase" buildScriptsEclipseSigner() def workspace = 'C:/workspace/openjdk-build/' From b3d9a675eb0a43959b44bb6e23afd5fd8c83d79f Mon Sep 17 00:00:00 2001 From: Stewart X Addison Date: Mon, 14 Oct 2024 19:01:55 +0100 Subject: [PATCH 11/19] Final cleanups of comments and debug Signed-off-by: Stewart X Addison --- .../common/openjdk_build_pipeline.groovy | 58 +++++++------------ 1 file changed, 20 insertions(+), 38 deletions(-) diff --git a/pipelines/build/common/openjdk_build_pipeline.groovy b/pipelines/build/common/openjdk_build_pipeline.groovy index 95448eafb..14ed825f8 100644 --- a/pipelines/build/common/openjdk_build_pipeline.groovy +++ b/pipelines/build/common/openjdk_build_pipeline.groovy @@ -1323,11 +1323,6 @@ class Build { MetaData data = formMetadata(version, initialWrite) Boolean metaWrittenOut = false - /* - * This is causing a problem when set to false. Sometimes it seems ok - * with the windows one, and sometimes not e.g. - * windbld#473/475/476/477 - */ listArchives().each({ file -> def type = 'jdk' if (file.contains('-jre')) { @@ -1343,11 +1338,11 @@ class Build { } else if (file.contains('-sbom')) { type = 'sbom' } - context.println "(writeMetaData for " + file + ") Batable and batted assuming sha256sum on windows 1340 windbld#388 - No - fails #479" + context.println "writeMetaData for " + file String hash if ( context.isUnix() ) { - context.println "Non-windows non-docker detected - running sh" + context.println "Non-windows non-docker detected - running sh to generate SHA256 sums in writeMetadata" hash = context.sh(script: """\ if [ -x "\$(command -v shasum)" ]; then (shasum -a 256 | cut -f1 -d' ') <$file @@ -1356,7 +1351,7 @@ class Build { fi """.stripIndent(), returnStdout: true, returnStatus: false).replaceAll('\n', '') } else { - context.println "Windows detected - running bat" + context.println "Windows detected - running bat to generate SHA256 sums in writeMetadata" hash = context.bat(script: "sha256sum ${file} | cut -f1 -d' '") // .replaceAll('\n', '') } @@ -1515,26 +1510,14 @@ class Build { context.println 'Checked out repo:' batOrSh('git status') context.println 'Checked out HEAD commit SHA:' - // windbld#245 batOrSh('git rev-parse HEAD') } def buildScriptsEclipseSigner() { def build_path -// openjdk_build_dir_arg unused after https://github.com/adoptium/ci-jenkins-pipelines/pull/1084 -// Ref https://adoptium.slack.com/archives/C09NW3L2J/p1725957286184479?thread_ts=1725900849.766449&cid=C09NW3L2J -// def openjdk_build_dir_arg - build_path = 'workspace/build/src/build' -// openjdk_build_dir_arg = "" - // SXA: This did not seem to set correctly with 'def base_path = build_path" def base_path base_path = build_path -// if (openjdk_build_dir_arg == "") { -// // If not using a custom openjdk build dir, then query what autoconf created as the build sub-folder -// context.println 'SXA: not batable 1648 - windbld#263' -// base_path = context.sh(script: "ls -d ${build_path}/* | tr -d '\\n'", returnStdout:true) -// } def repoHandler = new RepoHandler(USER_REMOTE_CONFIGS, ADOPT_DEFAULTS_JSON, buildConfig.CI_REF, buildConfig.BUILD_REF) context.stage('internal sign') { context.node('eclipse-codesign') { @@ -1549,6 +1532,7 @@ class Build { // Copy pre assembled binary ready for JMODs to be codesigned context.unstash 'jmods' def target_os = "${buildConfig.TARGET_OS}" + // TODO: Split this out into a separate script at some point context.withEnv(['base_os='+target_os, 'base_path='+base_path]) { // groovylint-disable context.sh ''' @@ -1683,10 +1667,8 @@ def buildScriptsAssemble( batOrSh "rm -rf ${base_path}/jdk/modules/jdk.jpackage/jdk/jpackage/internal/resources/*" } context.stage('assemble') { + // This would ideally not be required but it's due to lack of UID mapping in windows containers if ( buildConfig.TARGET_OS == 'windows' && buildConfig.DOCKER_IMAGE) { - // SXAEC: Still TBC on this to determine if something fails without it - // Ref https://github.com/adoptium/infrastructure/issues/3723 - // Fails to unstash even in non-docker case without the chmod e.g. windbld#840 context.bat('chmod -R a+rwX ' + '/cygdrive/c/workspace/openjdk-build/workspace/build/src/build/*') } // Restore signed JMODs @@ -1712,8 +1694,8 @@ def buildScriptsAssemble( context.timeout(time: buildTimeouts.BUILD_JDK_TIMEOUT, unit: 'HOURS') { context.println "openjdk_build_pipeline: calling MABF to assemble on win/mac JDK11+" if ( !context.isUnix() && buildConfig.DOCKER_IMAGE ) { - // SXAEC: Running ls -l here generates the shortname links required - // by the build and create paths referenced in the config.status file + // Running ls -l here generates the shortname links required by the + // build and create paths referenced in the config.status file context.bat(script: 'ls -l /cygdrive/c "/cygdrive/c/Program Files (x86)" "/cygdrive/c/Program Files (x86)/Microsoft Visual Studio/2022" "/cygdrive/c/Program Files (x86)/Microsoft Visual Studio/2022/BuildTools/VC/Redist/MSVC" "/cygdrive/c/Program Files (x86)/Windows Kits/10/bin" "/cygdrive/c/Program Files (x86)/Microsoft Visual Studio/2022/BuildTools/VC/Tools/MSVC" "/cygdrive/c/Program Files (x86)/Windows Kits/10/include" "/cygdrive/c/Program Files (x86)/Windows Kits/10/lib"') } batOrSh("bash ${ADOPT_DEFAULTS_JSON['scriptDirectories']['buildfarm']} --assemble-exploded-image") @@ -1835,8 +1817,6 @@ def buildScriptsAssemble( if (context.WORKSPACE != null && !context.WORKSPACE.isEmpty()) { context.println 'Removing workspace openjdk build directory: ' + openjdk_build_dir batOrSh('rm -rf ' + openjdk_build_dir) - // SXAEC: This handles when none of the clean options are used when extracting cached tarballs ... - batOrSh('rm -rf ' + context.WORKSPACE + '/workspace/target/*') } else { context.println 'Warning: Unable to remove workspace openjdk build directory as context.WORKSPACE is null/empty' } @@ -1895,11 +1875,8 @@ def buildScriptsAssemble( context.withEnv(['BUILD_ARGS=' + signBuildArgs]) { context.println 'Building an exploded image for signing' // Call make-adopt-build-farm.sh to do initial windows/mac build - // windbld#254 context.println "openjdk_build_pipeline: Calling MABF on win/mac to build exploded image" batOrSh("bash ./${ADOPT_DEFAULTS_JSON['scriptDirectories']['buildfarm']}") - // Use cached version from an attempt at the first phase only -// context.bat(script: "bash -c 'curl https://ci.adoptium.net/userContent/windows/openjdk-cached-workspace-phase1+8.tar.gz | tar -C /cygdrive/c/workspace/openjdk-build -xzf -'") } def base_path = build_path if (openjdk_build_dir_arg == "") { @@ -1924,7 +1901,8 @@ def buildScriptsAssemble( // JDK 16 + jpackage needs to be signed as well stash the resources folder containing the executables "${base_path}/jdk/modules/jdk.jpackage/jdk/jpackage/internal/resources/*" - // SXAEC: eclipse-codesign and assemble sections were previously inlined here + // eclipse-codesign and assemble sections were inlined here before + // https://github.com/adoptium/ci-jenkins-pipelines/pull/1117 } else { // Not Windows/Mac JDK11+ (i.e. doesn't require internal signing) def buildArgs @@ -1961,7 +1939,6 @@ def buildScriptsAssemble( context.withEnv(['BUILD_ARGS=' + buildArgs]) { context.println "openjdk_build_pipeline: calling MABF to do single pass build when USE_ADOPT_SHELL_SCRIPTS is false" batOrSh("bash ./${DEFAULTS_JSON['scriptDirectories']['buildfarm']}") -// context.bat(script: "bash -c 'curl https://ci.adoptium.net/userContent/windows/openjdk-cached-workspace.tar.gz | tar -C /cygdrive/c/workspace/openjdk-build -xpzf -'") } context.println '[CHECKOUT] Reverting pre-build user temurin-build checkout...' repoHandler.checkoutUserPipelines(context) @@ -1975,12 +1952,14 @@ def buildScriptsAssemble( } throw new Exception("[ERROR] Build JDK timeout (${buildTimeouts.BUILD_JDK_TIMEOUT} HOURS) has been reached. Exiting...") } + // TODO: Make the "internal signing/assembly" part independent of + // ENABLE_SIGNER so that this platform-specific logic is not required if ((buildConfig.TARGET_OS == 'mac' || buildConfig.TARGET_OS == 'windows') && buildConfig.JAVA_TO_BUILD != 'jdk8u' && enableSigner) { - context.println "Signing phase required - skipping metadata reading" + context.println "openjdk_build_pipeline: Internal signing phase required - skipping metadata reading" } else { - // Run a downstream job on riscv machine that returns the java version. Otherwise, just read the version.txt + // Run a downstream job on riscv machine that returns the java version. Otherwise, just read the version.txt String versionOut - if (buildConfig.BUILD_ARGS.contains('--cross-compile')) { + if (buildConfig.BUILD_ARGS.contains('--cross-compile')) { context.println "[WARNING] Don't read faked version.txt on cross compiled build! Archiving early and running downstream job to retrieve java version..." versionOut = readCrossCompiledVersionString() } else { @@ -1992,7 +1971,7 @@ def buildScriptsAssemble( if (!((buildConfig.TARGET_OS == 'mac' || buildConfig.TARGET_OS == 'windows') && buildConfig.JAVA_TO_BUILD != 'jdk8u' && enableSigner)) { writeMetadata(versionInfo, true) } else { - context.println "SXAEC: Skipping writing incomplete metadata - needs to be added to second phase" + context.println "Skipping writing incomplete metadata for now - will be done in the assemble phase instead" } } finally { @@ -2110,7 +2089,6 @@ def buildScriptsAssemble( Main function. This is what is executed remotely via the helper file kick_off_build.groovy, which is in turn executed by the downstream jobs. Running in downstream build job jdk-*-*-* called by kick_off_build.groovy */ -//SXAEC// @SuppressWarnings('unused') def build() { context.timestamps { try { @@ -2255,6 +2233,7 @@ def buildScriptsAssemble( throw new Exception("[ERROR] Controller docker file scm checkout timeout (${buildTimeouts.DOCKER_CHECKOUT_TIMEOUT} HOURS) has been reached. Exiting...") } + context.println "openjdk_build_pipeline: building in docker image from docker file " + buildConfig.DOCKER_FILE context.docker.build("build-image", "--build-arg image=${buildConfig.DOCKER_IMAGE} -f ${buildConfig.DOCKER_FILE} .").inside(buildConfig.DOCKER_ARGS) { buildScripts( cleanWorkspace, @@ -2280,6 +2259,7 @@ def buildScriptsAssemble( context.println "openjdk_build_pipeline: running exploded build in docker on Windows" def workspace = 'C:/workspace/openjdk-build/' context.echo("Switched to using non-default workspace path ${workspace}") + context.println "openjdk_build_pipeline: building in windows docker image " + buildConfig.DOCKER_IMAGE context.ws(workspace) { context.docker.image(buildConfig.DOCKER_IMAGE).inside(buildConfig.DOCKER_ARGS+" "+dockerRunArg) { buildScripts( @@ -2293,7 +2273,7 @@ def buildScriptsAssemble( } } } else { - context.println "openjdk_build_pipeline: running initial build in docker on non-windows" + context.println "openjdk_build_pipeline: running initial build in docker on non-windows with image " + buildConfig.DOCKER_IMAGE context.docker.image(buildConfig.DOCKER_IMAGE).inside(buildConfig.DOCKER_ARGS+" "+dockerRunArg) { buildScripts( cleanWorkspace, @@ -2343,6 +2323,7 @@ def buildScriptsAssemble( workspace = env.CYGWIN_WORKSPACE } context.echo("Switched to using non-default workspace path ${workspace}") + context.println "openjdk_build_pipeline: running build without docker on windows" context.ws(workspace) { buildScripts( cleanWorkspace, @@ -2363,6 +2344,7 @@ def buildScriptsAssemble( } } } else { // Non-windows, non-docker + context.println "openjdk_build_pipeline: running build without docker on non-windows platform" buildScripts( cleanWorkspace, cleanWorkspaceAfter, From 8955002d24d09e9844a98797390aa1a3f3736ca1 Mon Sep 17 00:00:00 2001 From: Stewart X Addison Date: Fri, 25 Oct 2024 13:52:26 +0100 Subject: [PATCH 12/19] Cherry-pick (ish) of 000de40 Signed-off-by: Stewart X Addison --- pipelines/build/common/openjdk_build_pipeline.groovy | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/pipelines/build/common/openjdk_build_pipeline.groovy b/pipelines/build/common/openjdk_build_pipeline.groovy index 14ed825f8..a0a645eca 100644 --- a/pipelines/build/common/openjdk_build_pipeline.groovy +++ b/pipelines/build/common/openjdk_build_pipeline.groovy @@ -2434,12 +2434,10 @@ def buildScriptsAssemble( } else { platform = buildConfig.ARCHITECTURE + '_' + buildConfig.TARGET_OS } - if ( !(platform =='aarch64_windows') ) { - if ( !(buildConfig.JAVA_TO_BUILD == 'jdk8u' && platform == 's390x_linux') ) { - context.echo "openjdk_build_pipeline: Remote trigger Eclipse Temurin AQA_Test_Pipeline job with ${platform} ${buildConfig.JAVA_TO_BUILD}" - def remoteTargets = remoteTriggerJckTests(platform, filename) - context.parallel remoteTargets - } + if ( !(buildConfig.JAVA_TO_BUILD == 'jdk8u' && platform == 's390x_linux') ) { + context.echo "openjdk_build_pipeline: Remote trigger Eclipse Temurin AQA_Test_Pipeline job with ${platform} ${buildConfig.JAVA_TO_BUILD}" + def remoteTargets = remoteTriggerJckTests(platform, filename) + context.parallel remoteTargets } } From 44babcc965f36e84886986dea09062667e61d391 Mon Sep 17 00:00:00 2001 From: Andrew Leonard <31470007+andrew-m-leonard@users.noreply.github.com> Date: Tue, 8 Oct 2024 16:19:31 +0100 Subject: [PATCH 13/19] Add aarch64Windows arch to filename mapping to daily slack status tool (#1122) Signed-off-by: Andrew Leonard --- tools/nightly_build_and_test_stats.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/nightly_build_and_test_stats.groovy b/tools/nightly_build_and_test_stats.groovy index d1ac51b9a..7210547f6 100644 --- a/tools/nightly_build_and_test_stats.groovy +++ b/tools/nightly_build_and_test_stats.groovy @@ -214,6 +214,7 @@ def verifyReleaseContent(String version, String release, String variant, Map sta // Map of config architecture to artifact name def archToAsset = [x64Linux: "x64_linux", x64Windows: "x64_windows", + aarch64Windows: "aarch64_windows", x64Mac: "x64_mac", x64AlpineLinux: "x64_alpine-linux", ppc64Aix: "ppc64_aix", From 78ce38754892a26aa47d062a5435f950a37ecef0 Mon Sep 17 00:00:00 2001 From: Andrew Leonard <31470007+andrew-m-leonard@users.noreply.github.com> Date: Fri, 11 Oct 2024 09:04:52 +0100 Subject: [PATCH 14/19] Enable temurin vs2022_redist devkit (#1123) Signed-off-by: Andrew Leonard --- .../jobs/configurations/jdk11u_pipeline_config.groovy | 6 +++--- .../jobs/configurations/jdk17u_pipeline_config.groovy | 8 +++----- .../jobs/configurations/jdk21u_pipeline_config.groovy | 5 ++--- .../jobs/configurations/jdk23u_pipeline_config.groovy | 5 ++--- .../jobs/configurations/jdk24_pipeline_config.groovy | 5 ++--- .../jobs/configurations/jdk8u_pipeline_config.groovy | 4 ++-- 6 files changed, 14 insertions(+), 19 deletions(-) diff --git a/pipelines/jobs/configurations/jdk11u_pipeline_config.groovy b/pipelines/jobs/configurations/jdk11u_pipeline_config.groovy index 4bf1d2cac..8e16dc941 100644 --- a/pipelines/jobs/configurations/jdk11u_pipeline_config.groovy +++ b/pipelines/jobs/configurations/jdk11u_pipeline_config.groovy @@ -79,7 +79,7 @@ class Config11 { 'temurin' : '--disable-ccache' ], buildArgs : [ - 'temurin' : '--jvm-variant client,server --create-sbom' + 'temurin' : '--jvm-variant client,server --create-sbom --use-adoptium-devkit vs2022_redist_14.40.33807_10.0.26100.0' ], test : 'default' ], @@ -92,7 +92,7 @@ class Config11 { 'temurin' : '--disable-ccache' ], buildArgs : [ - 'temurin' : '--jvm-variant client,server --create-sbom' + 'temurin' : '--jvm-variant client,server --create-sbom --use-adoptium-devkit vs2022_redist_14.40.33807_10.0.26100.0' ], test : 'default' ], @@ -253,7 +253,7 @@ class Config11 { 'temurin' : '--disable-ccache' ], buildArgs : [ - 'temurin' : '--jvm-variant client,server --create-sbom --cross-compile' + 'temurin' : '--jvm-variant client,server --create-sbom --cross-compile --use-adoptium-devkit vs2022_redist_14.40.33807_10.0.26100.0' ] ] ] diff --git a/pipelines/jobs/configurations/jdk17u_pipeline_config.groovy b/pipelines/jobs/configurations/jdk17u_pipeline_config.groovy index 6c350c22d..3aa7991af 100644 --- a/pipelines/jobs/configurations/jdk17u_pipeline_config.groovy +++ b/pipelines/jobs/configurations/jdk17u_pipeline_config.groovy @@ -70,9 +70,8 @@ class Config17 { arch : 'x64', additionalNodeLabels: 'win2022&&vs2019', test : 'default', - configureArgs : "--with-ucrt-dll-dir='C:/progra~2/wi3cf2~1/10/Redist/10.0.22000.0/ucrt/DLLs/x64'", buildArgs : [ - 'temurin' : '--create-jre-image --create-sbom' + 'temurin' : '--create-jre-image --create-sbom --use-adoptium-devkit vs2022_redist_14.40.33807_10.0.26100.0' ] ], @@ -81,9 +80,8 @@ class Config17 { arch : 'x86-32', additionalNodeLabels: 'win2022&&vs2019', test : 'default', - configureArgs : "--with-ucrt-dll-dir='C:/progra~2/wi3cf2~1/10/Redist/10.0.22000.0/ucrt/DLLs/x86'", buildArgs : [ - 'temurin' : '--jvm-variant client,server --create-jre-image --create-sbom' + 'temurin' : '--jvm-variant client,server --create-jre-image --create-sbom --use-adoptium-devkit vs2022_redist_14.40.33807_10.0.26100.0' ] ], @@ -182,7 +180,7 @@ class Config17 { additionalNodeLabels: 'win2022&&vs2019', test : 'default', buildArgs : [ - 'temurin' : '--create-jre-image --create-sbom --cross-compile' + 'temurin' : '--create-jre-image --create-sbom --cross-compile --use-adoptium-devkit vs2022_redist_14.40.33807_10.0.26100.0' ] ] ] diff --git a/pipelines/jobs/configurations/jdk21u_pipeline_config.groovy b/pipelines/jobs/configurations/jdk21u_pipeline_config.groovy index 2209d8ce6..b1b61c1e0 100644 --- a/pipelines/jobs/configurations/jdk21u_pipeline_config.groovy +++ b/pipelines/jobs/configurations/jdk21u_pipeline_config.groovy @@ -70,9 +70,8 @@ class Config21 { test: [ weekly : ['sanity.openjdk', 'sanity.system', 'extended.system', 'sanity.perf', 'sanity.functional', 'extended.functional', 'extended.openjdk', 'extended.perf', 'special.functional', 'special.openjdk', 'dev.functional', 'dev.system'] ], - configureArgs : "--with-ucrt-dll-dir='C:/progra~2/wi3cf2~1/10/Redist/10.0.22621.0/ucrt/DLLs/x64'", buildArgs : [ - 'temurin' : '--create-jre-image --create-sbom' + 'temurin' : '--create-jre-image --create-sbom --use-adoptium-devkit vs2022_redist_14.40.33807_10.0.26100.0' ] ], @@ -164,7 +163,7 @@ class Config21 { additionalNodeLabels: 'win2022&&vs2022', test : 'default', buildArgs : [ - 'temurin' : '--create-jre-image --create-sbom --cross-compile' + 'temurin' : '--create-jre-image --create-sbom --cross-compile --use-adoptium-devkit vs2022_redist_14.40.33807_10.0.26100.0' ] ] ] diff --git a/pipelines/jobs/configurations/jdk23u_pipeline_config.groovy b/pipelines/jobs/configurations/jdk23u_pipeline_config.groovy index d56a07076..decb963f3 100644 --- a/pipelines/jobs/configurations/jdk23u_pipeline_config.groovy +++ b/pipelines/jobs/configurations/jdk23u_pipeline_config.groovy @@ -71,9 +71,8 @@ class Config23 { test: [ weekly : ['sanity.openjdk', 'sanity.system', 'extended.system', 'sanity.perf', 'sanity.functional', 'extended.functional', 'extended.openjdk', 'extended.perf', 'special.functional', 'special.openjdk', 'dev.functional', 'dev.system'] ], - configureArgs : "--with-ucrt-dll-dir='C:/progra~2/wi3cf2~1/10/Redist/10.0.22621.0/ucrt/DLLs/x64'", buildArgs : [ - 'temurin' : '--create-jre-image --create-sbom' + 'temurin' : '--create-jre-image --create-sbom --use-adoptium-devkit vs2022_redist_14.40.33807_10.0.26100.0' ] ], @@ -165,7 +164,7 @@ class Config23 { additionalNodeLabels: 'win2022&&vs2022', test : 'default', buildArgs : [ - 'temurin' : '--create-jre-image --create-sbom --cross-compile' + 'temurin' : '--create-jre-image --create-sbom --cross-compile --use-adoptium-devkit vs2022_redist_14.40.33807_10.0.26100.0' ] ] ] diff --git a/pipelines/jobs/configurations/jdk24_pipeline_config.groovy b/pipelines/jobs/configurations/jdk24_pipeline_config.groovy index 6968bdc4a..52b73d420 100644 --- a/pipelines/jobs/configurations/jdk24_pipeline_config.groovy +++ b/pipelines/jobs/configurations/jdk24_pipeline_config.groovy @@ -71,9 +71,8 @@ class Config24 { test: [ weekly : ['sanity.openjdk', 'sanity.system', 'extended.system', 'sanity.perf', 'sanity.functional', 'extended.functional', 'extended.openjdk', 'extended.perf', 'special.functional', 'special.openjdk', 'dev.functional', 'dev.system'] ], - configureArgs : "--with-ucrt-dll-dir='C:/progra~2/wi3cf2~1/10/Redist/10.0.22621.0/ucrt/DLLs/x64'", buildArgs : [ - 'temurin' : '--create-jre-image --create-sbom' + 'temurin' : '--create-jre-image --create-sbom --use-adoptium-devkit vs2022_redist_14.40.33807_10.0.26100.0' ] ], @@ -165,7 +164,7 @@ class Config24 { additionalNodeLabels: 'win2022&&vs2022', test : 'default', buildArgs : [ - 'temurin' : '--create-jre-image --create-sbom --cross-compile' + 'temurin' : '--create-jre-image --create-sbom --cross-compile --use-adoptium-devkit vs2022_redist_14.40.33807_10.0.26100.0' ] ] ] diff --git a/pipelines/jobs/configurations/jdk8u_pipeline_config.groovy b/pipelines/jobs/configurations/jdk8u_pipeline_config.groovy index 8ecbe233b..00be49c5c 100644 --- a/pipelines/jobs/configurations/jdk8u_pipeline_config.groovy +++ b/pipelines/jobs/configurations/jdk8u_pipeline_config.groovy @@ -74,7 +74,7 @@ class Config8 { 'temurin' : '--disable-ccache' ], buildArgs : [ - 'temurin' : '--create-sbom' + 'temurin' : '--create-sbom --use-adoptium-devkit vs2022_redist_14.40.33807_10.0.26100.0' ] ], @@ -86,7 +86,7 @@ class Config8 { 'temurin' : '--disable-ccache' ], buildArgs : [ - temurin : '--jvm-variant client,server --create-sbom' + temurin : '--jvm-variant client,server --create-sbom --use-adoptium-devkit vs2022_redist_14.40.33807_10.0.26100.0' ], test : 'default' ], From a6efd490b03a86f05c7c5a5679973db51b555a68 Mon Sep 17 00:00:00 2001 From: sophia-guo Date: Fri, 11 Oct 2024 08:44:29 -0400 Subject: [PATCH 15/19] Add dev.system for aarch64_mac for jdk21+ (#1124) Co-authored-by: Andrew Leonard <31470007+andrew-m-leonard@users.noreply.github.com> --- pipelines/jobs/configurations/jdk21u_pipeline_config.groovy | 4 +++- pipelines/jobs/configurations/jdk22u_pipeline_config.groovy | 4 +++- pipelines/jobs/configurations/jdk24_pipeline_config.groovy | 4 +++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/pipelines/jobs/configurations/jdk21u_pipeline_config.groovy b/pipelines/jobs/configurations/jdk21u_pipeline_config.groovy index b1b61c1e0..ae549114f 100644 --- a/pipelines/jobs/configurations/jdk21u_pipeline_config.groovy +++ b/pipelines/jobs/configurations/jdk21u_pipeline_config.groovy @@ -137,7 +137,9 @@ class Config21 { os : 'mac', arch : 'aarch64', additionalNodeLabels: 'xcode15.0.1', - test : 'default', + test: [ + weekly : ['sanity.openjdk', 'sanity.system', 'extended.system', 'sanity.perf', 'sanity.functional', 'extended.functional', 'extended.openjdk', 'extended.perf', 'special.functional', 'special.openjdk', 'dev.functional', 'dev.system'] + ], buildArgs : [ 'temurin' : '--create-jre-image --create-sbom' ] diff --git a/pipelines/jobs/configurations/jdk22u_pipeline_config.groovy b/pipelines/jobs/configurations/jdk22u_pipeline_config.groovy index d579175bb..7ea2e4807 100644 --- a/pipelines/jobs/configurations/jdk22u_pipeline_config.groovy +++ b/pipelines/jobs/configurations/jdk22u_pipeline_config.groovy @@ -135,7 +135,9 @@ class Config22 { os : 'mac', arch : 'aarch64', additionalNodeLabels: 'xcode15.0.1', - test : 'default', + test: [ + weekly : ['sanity.openjdk', 'sanity.system', 'extended.system', 'sanity.perf', 'sanity.functional', 'extended.functional', 'extended.openjdk', 'extended.perf', 'special.functional', 'special.openjdk', 'dev.functional', 'dev.system'] + ], buildArgs : [ 'temurin' : '--create-jre-image --create-sbom' ] diff --git a/pipelines/jobs/configurations/jdk24_pipeline_config.groovy b/pipelines/jobs/configurations/jdk24_pipeline_config.groovy index 52b73d420..0a2b50321 100644 --- a/pipelines/jobs/configurations/jdk24_pipeline_config.groovy +++ b/pipelines/jobs/configurations/jdk24_pipeline_config.groovy @@ -138,7 +138,9 @@ class Config24 { os : 'mac', arch : 'aarch64', additionalNodeLabels: 'xcode15.0.1', - test : 'default', + test: [ + weekly : ['sanity.openjdk', 'sanity.system', 'extended.system', 'sanity.perf', 'sanity.functional', 'extended.functional', 'extended.openjdk', 'extended.perf', 'special.functional', 'special.openjdk', 'dev.functional', 'dev.system'] + ], buildArgs : [ 'temurin' : '--create-jre-image --create-sbom' ] From 83e382b55d743b7cc3b7240bef14d7699eeb3833 Mon Sep 17 00:00:00 2001 From: George Adams Date: Sun, 13 Oct 2024 15:23:45 +0100 Subject: [PATCH 16/19] remove JDK22 from release pipeline generator (#1125) --- pipelines/build/regeneration/release_pipeline_generator.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipelines/build/regeneration/release_pipeline_generator.groovy b/pipelines/build/regeneration/release_pipeline_generator.groovy index ede1cf301..5d132dc97 100644 --- a/pipelines/build/regeneration/release_pipeline_generator.groovy +++ b/pipelines/build/regeneration/release_pipeline_generator.groovy @@ -7,7 +7,7 @@ file used as jenkinsfile to generator official release pipeline */ // ensure releaseVersions is updated before create releaseTag -def releaseVersions = [8,11,17,21,22,23] +def releaseVersions = [8,11,17,21,23] // Regenerate release-openjdkX-pipeline per each jdk version listed in releaseVersions From 26ce97f39ab482c2930b2b5e07875423dabd35b9 Mon Sep 17 00:00:00 2001 From: Andrew Leonard <31470007+andrew-m-leonard@users.noreply.github.com> Date: Wed, 16 Oct 2024 16:39:51 +0100 Subject: [PATCH 17/19] jdk23+ update version GA tag check should be using updates repo (#1127) (#1128) * jdk-23.0.1 release jdkBranch GA tag check using wrong repo * jdk-23.0.1 release jdkBranch GA tag check using wrong repo --------- Signed-off-by: Andrew Leonard --- pipelines/build/openjdk_pipeline.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pipelines/build/openjdk_pipeline.groovy b/pipelines/build/openjdk_pipeline.groovy index c5cdc5702..d4e85f5c9 100644 --- a/pipelines/build/openjdk_pipeline.groovy +++ b/pipelines/build/openjdk_pipeline.groovy @@ -26,7 +26,8 @@ Map DEFAULTS_JSON = null def findGaCommitSHA(String jdkVersion, String jdkBranch, Boolean annotatedTag) { // Determine OpenJDK and Adoptium mirror repository def repo - if (jdkVersion.toInteger() >= 23) { + // Is it a jdk-23+ stablizationjdk branch version? ie.jdk-23, jdk-24, ... + if (jdkVersion.toInteger() >= 23 && !jdkBranch.contains(".0")) { // jdk-23+ first release is a branch within the jdk(head) repository repo = "jdk" } else { From 951fd044b1e4eb7deadb24958a20e46d393fa651 Mon Sep 17 00:00:00 2001 From: Stewart X Addison Date: Tue, 29 Oct 2024 15:46:40 +0000 Subject: [PATCH 18/19] Fix up empty bits Signed-off-by: Stewart X Addison --- pipelines/build/common/openjdk_build_pipeline.groovy | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pipelines/build/common/openjdk_build_pipeline.groovy b/pipelines/build/common/openjdk_build_pipeline.groovy index a0a645eca..455113460 100644 --- a/pipelines/build/common/openjdk_build_pipeline.groovy +++ b/pipelines/build/common/openjdk_build_pipeline.groovy @@ -2433,7 +2433,7 @@ def buildScriptsAssemble( platform = 'x86-64_' + buildConfig.TARGET_OS } else { platform = buildConfig.ARCHITECTURE + '_' + buildConfig.TARGET_OS - } + } if ( !(buildConfig.JAVA_TO_BUILD == 'jdk8u' && platform == 's390x_linux') ) { context.echo "openjdk_build_pipeline: Remote trigger Eclipse Temurin AQA_Test_Pipeline job with ${platform} ${buildConfig.JAVA_TO_BUILD}" def remoteTargets = remoteTriggerJckTests(platform, filename) @@ -2471,7 +2471,6 @@ def buildScriptsAssemble( } } } -// } // assemble stage } return { From d201bff229bcc555918b8d9fe406c05fa2612f2d Mon Sep 17 00:00:00 2001 From: Stewart X Addison Date: Wed, 6 Nov 2024 11:44:26 +0000 Subject: [PATCH 19/19] Move workspace scope and use it for cleaning cyclone-dx and security directories Signed-off-by: Stewart X Addison --- .../build/common/openjdk_build_pipeline.groovy | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/pipelines/build/common/openjdk_build_pipeline.groovy b/pipelines/build/common/openjdk_build_pipeline.groovy index 976855976..60b7752aa 100644 --- a/pipelines/build/common/openjdk_build_pipeline.groovy +++ b/pipelines/build/common/openjdk_build_pipeline.groovy @@ -2137,6 +2137,10 @@ def buildScriptsAssemble( updateGithubCommitStatus('PENDING', 'Pending') } } + def workspace + if (buildConfig.TARGET_OS == 'windows') { + workspace = 'C:/workspace/openjdk-build/' + } if (buildConfig.DOCKER_IMAGE) { context.println "openjdk_build_pipeline: preparing to use docker image" // Docker build environment @@ -2159,7 +2163,10 @@ def buildScriptsAssemble( addNodeToBuildDescription() // Cannot clean workspace from inside docker container if ( buildConfig.TARGET_OS == 'windows' && buildConfig.DOCKER_IMAGE ) { - context.bat('rm -rf c:/workspace/openjdk-build/cyclonedx-lib c:/workspace/openjdk-build/security') + context.ws(workspace) { + context.bat("rm -rf " + context.WORKSPACE + "/cyclonedx-lib " + + context.WORKSPACE + "/security") + } } if (cleanWorkspace) { try { @@ -2257,7 +2264,6 @@ def buildScriptsAssemble( } if (buildConfig.TARGET_OS == 'windows') { context.println "openjdk_build_pipeline: running exploded build in docker on Windows" - def workspace = 'C:/workspace/openjdk-build/' context.echo("Switched to using non-default workspace path ${workspace}") context.println "openjdk_build_pipeline: building in windows docker image " + buildConfig.DOCKER_IMAGE context.ws(workspace) { @@ -2289,7 +2295,6 @@ def buildScriptsAssemble( if ( enableSigner && buildConfig.JAVA_TO_BUILD != 'jdk8u' ) { context.println "openjdk_build_pipeline: running eclipse signing phase" buildScriptsEclipseSigner() - def workspace = 'C:/workspace/openjdk-build/' context.ws(workspace) { context.println "Signing with non-default workspace location ${workspace}" context.println "openjdk_build_pipeline: running assemble phase (invocation 1)" @@ -2318,7 +2323,6 @@ def buildScriptsAssemble( context.echo("checking ${buildConfig.TARGET_OS}") if (buildConfig.TARGET_OS == 'windows') { // See https://github.com/adoptium/infrastucture/issues/1284#issuecomment-621909378 for justification of the below path - def workspace = 'C:/workspace/openjdk-build/' if (env.CYGWIN_WORKSPACE) { workspace = env.CYGWIN_WORKSPACE } @@ -2436,7 +2440,7 @@ def buildScriptsAssemble( } if ( !(buildConfig.JAVA_TO_BUILD == 'jdk8u' && platform == 's390x_linux') ) { context.echo "openjdk_build_pipeline: Remote trigger Eclipse Temurin AQA_Test_Pipeline job with ${platform} ${buildConfig.JAVA_TO_BUILD}" - def remoteTargets = remoteTriggerJckTests(platform, filename) + def remoteTargets = remoteTriggerJckTests(platform, filename) context.parallel remoteTargets } }