From 2c76461b462fa6e23d0d48208a684c5387e1816d Mon Sep 17 00:00:00 2001 From: judovana Date: Thu, 4 Jul 2024 04:20:37 +0200 Subject: [PATCH] Added optional arg to -D/--podman/--docker base image to use (#3869) * Added optional arg to -D/--podman/--docker base image to use it is also correctly passed to image generator, however taht is not tuned to work * seaparate local declaration and no backticks and no sed * Using current arch as arch instead of ahrdcoded * Added fedora depndencies With this, the fedora containers finshed correctly with images * Now working also on centos 9 * Now centos:stream8 * Fixed indentation * Update sbin/common/common.sh Co-authored-by: Martijn Verburg * missing article Co-authored-by: Martijn Verburg * Added more coments to optional image argumetn check * Removed hardcoded x64 arch * Added few more commens if comments are on * Reorgnaized order of operations in docekrtfile Generally sort them by disk footprint size. Motivation is, that allt he snapshjots were created after the install of all deps hapened. So instead of snapshoting 55mb base image with new symlink, it was snapshoting 1g or more image with all deps. Now all small-sized operations happens firsr then basic dependences happens, so most of the other operations can run last are huge ones: installing of all dependencies and cloning the jdk All is closed byu change of user to openjdk That reuired splitting of dn/apt to two parts, and thus extracting a bit of suddenly duplicated code to functions * $() and " for bash lint * Added support for centos:7 * Update sbin/common/config_init.sh Co-authored-by: Martijn Verburg * Update sbin/common/config_init.sh Co-authored-by: Martijn Verburg * Update sbin/common/config_init.sh Co-authored-by: Martijn Verburg * removed missleading comment --------- Co-authored-by: Martijn Verburg --- docker-build.sh | 5 +- docker/dockerfile-generator.sh | 235 ++++++++++++++++++++++++++++----- makejdk-any-platform.1 | 5 +- sbin/common/common.sh | 32 +++-- sbin/common/config_init.sh | 38 +++++- 5 files changed, 264 insertions(+), 51 deletions(-) diff --git a/docker-build.sh b/docker-build.sh index b7211f464..efa1a62ce 100755 --- a/docker-build.sh +++ b/docker-build.sh @@ -95,14 +95,15 @@ buildOpenJDKViaDocker() # TODO This could be extracted overridden by the user if we support more # architectures going forwards - local container_architecture="x86_64/ubuntu" + local container_architecture + container_architecture="$(uname -m)/${BUILD_CONFIG[CONTAINER_IMAGE]//:*/}" local build_variant_flag="" BUILD_CONFIG[DOCKER_FILE_PATH]="docker/${BUILD_CONFIG[OPENJDK_CORE_VERSION]}/$container_architecture" if [ "${BUILD_CONFIG[BUILD_VARIANT]}" == "openj9" ]; then build_variant_flag="--openj9" fi - docker/dockerfile-generator.sh --version "${BUILD_CONFIG[OPENJDK_FEATURE_NUMBER]}" --path "${BUILD_CONFIG[DOCKER_FILE_PATH]}" "$build_variant_flag" \ + docker/dockerfile-generator.sh --version "${BUILD_CONFIG[OPENJDK_FEATURE_NUMBER]}" --path "${BUILD_CONFIG[DOCKER_FILE_PATH]}" "$build_variant_flag" --base-image "${BUILD_CONFIG[CONTAINER_IMAGE]}" \ --dirs "${workspacedir} ${targetdir} ${targetbuilddir} ${configdir} ${localsourcesdir}" --command "${BUILD_CONFIG[CONTAINER_AS_ROOT]} ${BUILD_CONFIG[CONTAINER_COMMAND]}" # shellcheck disable=SC1090,SC1091 diff --git a/docker/dockerfile-generator.sh b/docker/dockerfile-generator.sh index 932224e05..d1d000039 100755 --- a/docker/dockerfile-generator.sh +++ b/docker/dockerfile-generator.sh @@ -24,8 +24,21 @@ DOCKERFILE_DIR= DOCKERFILE_PATH= # Default to JDK8 JDK_VERSION=8 +IMAGE="ubuntu:18.04" JDK_MAX= JDK_GA= +DNF_INSTALL=dnf + +UBUNTU_PREAMBLE="apt-get update \\ + && apt-get install -qq -u --no-install-recommends \\ + software-properties-common \\ + dirmngr \\ + gpg-agent \\ + coreutils \\ + && apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 0x219BD9C9 \\ + && add-apt-repository 'deb http://repos.azulsystems.com/ubuntu stable main' \\ + && apt-get update \\ + && apt-get -y upgrade \\" getFile() { if [ $# -ne 2 ]; then @@ -93,6 +106,11 @@ processArgs() { shift shift ;; + --base-image) + IMAGE="${2}" + shift + shift + ;; --path) DOCKERFILE_DIR=$2 shift @@ -152,6 +170,7 @@ usage() { Options: --help | -h Print this message and exit --build Build the docker image after generation and create interactive container + --base-image set the base image if used container. Default: $IMAGE --clean Remove all dockerfiles (Dockerfile*) from '--path' --comments Prints comments into the dockerfile --dirs space separated list of dirs to be created, with proper permissions @@ -186,31 +205,71 @@ printPreamble() { # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************** -FROM ubuntu:18.04 +FROM $IMAGE LABEL maintainer=\"AdoptOpenJDK \" " >> "$DOCKERFILE_PATH" } -# Put in apt packages required for building a JDK -printAptPackages() { + +printAptPackagesBase() { if [ ${COMMENTS} == true ]; then echo " -# Install required OS tools +# Install required OS tools to setup environment as .deb via apt-get # dirmngr, gpg-agent & coreutils are all required for the apt-add repository command" >> "$DOCKERFILE_PATH" fi + echo " +RUN $UBUNTU_PREAMBLE + && apt-get install -qq -y --no-install-recommends \\ + curl \\ + git \\ + unzip \\ + wget \\ + zip " >> "$DOCKERFILE_PATH" + echo " +RUN rm -rf /var/lib/apt/lists/*" >> "$DOCKERFILE_PATH" +} +printDnfPackagesBase() { + if [ ${COMMENTS} == true ]; then + echo " +# Install required OS tools to setup environment as rpms via dnf" >> "$DOCKERFILE_PATH" + fi + local skipGpg="" # it may bite from time to time + #local skipGpg="--nogpgcheck" + local erasing="--allowerasing" + if [ ${DNF_INSTALL} = yum ] ; then + erasing="" + fi + if echo "${IMAGE}" | grep -e "stream8" -e "centos:7" ; then + echo " +RUN cd /etc/yum.repos.d/ ; sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* +RUN cd /etc/yum.repos.d/ ; sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* " >> "$DOCKERFILE_PATH" + fi echo " -RUN apt-get update \\ - && apt-get install -qq -u --no-install-recommends \\ - software-properties-common \\ - dirmngr \\ - gpg-agent \\ - coreutils \\ - && apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 0x219BD9C9 \\ - && add-apt-repository 'deb http://repos.azulsystems.com/ubuntu stable main' \\ - && apt-get update \\ - && apt-get -y upgrade \\ +RUN ${DNF_INSTALL} $skipGpg -y update $erasing +RUN ${DNF_INSTALL} $skipGpg -y install $erasing \\ + bzip2-libs \\ + bzip2 \\ + curl \\ + git \\ + unzip \\ + /usr/bin/which \\ + wget \\ + zip " >> "$DOCKERFILE_PATH" + echo " +RUN ${DNF_INSTALL} clean all" >> "$DOCKERFILE_PATH" +} + +printAptPackagesJdk() { + if [ ${COMMENTS} == true ]; then + echo " +# Install required OS tools to build JDK as .deb via apt-get +# dirmngr, gpg-agent & coreutils are all required for the apt-add repository command" >> "$DOCKERFILE_PATH" + fi + + echo " +RUN $UBUNTU_PREAMBLE && apt-get install -qq -y --no-install-recommends \\ ant \\ ant-contrib \\ @@ -218,9 +277,7 @@ RUN apt-get update \\ ca-certificates \\ cmake \\ cpio \\ - curl \\ file \\ - git \\ libasound2-dev \\ libcups2-dev \\ libelf-dev \\ @@ -236,10 +293,7 @@ RUN apt-get update \\ make \\ perl \\ ssh \\ - systemtap-sdt-dev \\ - unzip \\ - wget \\ - zip \\" >> "$DOCKERFILE_PATH" + systemtap-sdt-dev \\" >> "$DOCKERFILE_PATH" if [ ${OPENJ9} = true ]; then echo " gcc-7 \\ @@ -268,6 +322,62 @@ RUN apt-get update \\ echo " && rm -rf /var/lib/apt/lists/*" >> "$DOCKERFILE_PATH" } +printDnfPackagesJdk() { + if [ ${COMMENTS} == true ]; then + echo " +# Install required OS tools to build JDK as rpms via dnf" >> "$DOCKERFILE_PATH" + fi + local skipGpg="" # it may bite from time to time + #local skipGpg="--nogpgcheck" + local erasing="--allowerasing" + if [ ${DNF_INSTALL} = yum ] ; then + erasing="" + fi + echo " +RUN ${DNF_INSTALL} $skipGpg -y install $erasing \\ + ant \\ + autoconf \\ + automake \\ + ca-certificates \\ + cmake \\ + cpio \\ + diffutils \\ + file \\ + alsa-lib-devel \\ + cups-devel \\ + gcc \\ + gcc-c++ \\ + gdb \\ + fontconfig-devel \\ + freetype-devel \\ + libtool \\ + libX11-devel \\ + libXi-devel \\ + libXinerama-devel \\ + libXrandr-devel \\ + libXrender-devel \\ + libXt-devel \\ + libXtst-devel \\ + lksctp-tools-devel \\ + lksctp-tools pcsc-lite-libs \\ + make \\ + perl \\ + procps-ng \\ + openssh-clients \\ + openssl \\ + systemtap-sdt-devel \\ + kernel-headers \\ + \"lcms*\" \\ + nss-devel \\ " >> "$DOCKERFILE_PATH" + if echo "${IMAGE}" | grep fedora ; then + echo " libstdc++-static \\ + pcsc-lite-devel \\ " >> "$DOCKERFILE_PATH" + fi + echo " tzdata-java " >> "$DOCKERFILE_PATH" + echo " +RUN ${DNF_INSTALL} clean all" >> "$DOCKERFILE_PATH" +} + printCreateFolder() { echo " RUN mkdir -p /openjdk/target @@ -285,6 +395,10 @@ ENV CC=gcc-7 CXX=g++-7" >> "$DOCKERFILE_PATH" } printCustomDirs() { + if [ ${COMMENTS} == true ]; then + echo "# In podman (in docker do not harm) shared folder is owned by root, and is read for others, unless it already exists" >> "$DOCKERFILE_PATH" + echo "# So we have to create all future-mounted dirs, with proper owner and permissions" >> "$DOCKERFILE_PATH" + fi for dir in ${DIRS} ; do echo "RUN mkdir -p $dir" >> "$DOCKERFILE_PATH" echo "RUN chmod 755 $dir" >> "$DOCKERFILE_PATH" @@ -293,6 +407,10 @@ printCustomDirs() { } printDockerJDKs() { + if [ ${COMMENTS} == true ]; then + echo " + # Linking of boot jdk must happen after the system jdk is isntalled, as it is iverwriting whatever java/javac from system" >> "$DOCKERFILE_PATH" + fi # JDK8 uses zulu-7 to as it's bootjdk if [ "${JDK_VERSION}" != 8 ] && [ "${JDK_VERSION}" != "${JDK_MAX}" ]; then if [ "${JDK_VERSION}" == 11 ]; then @@ -341,7 +459,7 @@ printDockerJDKs() { printJDK() { local JDKVersion=$1 echo " -RUN sh -c \"mkdir -p /usr/lib/jvm/jdk$JDKVersion && wget 'https://api.adoptium.net/v3/binary/latest/$JDKVersion/ga/linux/x64/jdk/hotspot/normal/adoptium?project=jdk' -O - | tar xzf - -C /usr/lib/jvm/jdk$JDKVersion --strip-components=1\"" >> "$DOCKERFILE_PATH" +RUN sh -c \"mkdir -p /usr/lib/jvm/jdk$JDKVersion && wget 'https://api.adoptium.net/v3/binary/latest/$JDKVersion/ga/linux/$(adoptiumArch)/jdk/hotspot/normal/adoptium?project=jdk' -O - | tar xzf - -C /usr/lib/jvm/jdk$JDKVersion --strip-components=1\"" >> "$DOCKERFILE_PATH" } printGitCloneJenkinsPipelines(){ @@ -370,19 +488,75 @@ RUN useradd -u \$HostUID -ms /bin/bash build WORKDIR /openjdk/build RUN chown -R build /openjdk/" >> "$DOCKERFILE_PATH" printCustomDirs +} + +printUserSet(){ echo " USER build" >> "$DOCKERFILE_PATH" } -printContainerVars(){ +adoptiumArch() { + local arch + arch=$(uname -m) + if [ "$arch" = "x86_64" ] ; then arch="x64" ; fi + echo "$arch" +} + +printContainerVars() { echo " ARG OPENJDK_CORE_VERSION ENV OPENJDK_CORE_VERSION=\$OPENJDK_CORE_VERSION -ENV ARCHITECTURE=x64 +ENV ARCHITECTURE=$(adoptiumArch) ENV JDK_PATH=jdk ENV JDK8_BOOT_DIR=/usr/lib/jvm/jdk8" >> "$DOCKERFILE_PATH" } +isRpm() { + echo "${IMAGE}" | grep -i -e "fedora" -e "centos" -e "rocky" -e "stream" -e "rhel" +} + +isDeb() { + echo "${IMAGE}" | grep -i -e "ubuntu" -e "debian" +} + +isYum() { + if echo "${IMAGE}" | grep -e "stream7" -e "centos:7" ; then + DNF_INSTALL=yum + else + DNF_INSTALL=dnf + fi +} + +printDepsBase() { + if isRpm ; then + isYum + printDnfPackagesBase + elif isDeb ; then + printAptPackagesBase + # OpenJ9 MUST use gcc7, HS doesn't have to + if [ ${OPENJ9} == true ]; then + printgcc + fi + else + echo "Unknown system, can not install build deps: $IMAGE" + fi +} + +printDepsJdk() { + if isRpm ; then + isYum + printDnfPackagesJdk + elif isDeb ; then + printAptPackagesJdk + # OpenJ9 MUST use gcc7, HS doesn't have to + if [ ${OPENJ9} == true ]; then + printgcc + fi + else + echo "Unknown system, can not install build deps: $IMAGE" + fi +} + generateFile() { mkdir -p "$DOCKERFILE_DIR" if [ -f "$DOCKERFILE_PATH" ]; then @@ -412,15 +586,9 @@ processArgs "$@" generateFile generateConfig printPreamble -printAptPackages -# OpenJ9 MUST use gcc7, HS doesn't have to -if [ ${OPENJ9} == true ]; then - printgcc -fi - -printDockerJDKs +printUserCreate +printDepsBase printGitCloneJenkinsPipelines - # If building the image straight away, it can't be assumed the folders to be copied are in place # Therefore create an image that instead git clones openjdk-build and a build can be started there if [ ${BUILD} == false ]; then @@ -428,8 +596,9 @@ if [ ${BUILD} == false ]; then else printGitClone fi - -printUserCreate +printDepsJdk +printDockerJDKs +printUserSet printContainerVars echo "Dockerfile created at $DOCKERFILE_PATH" diff --git a/makejdk-any-platform.1 b/makejdk-any-platform.1 index 80cb4158c..ddb1658dd 100755 --- a/makejdk-any-platform.1 +++ b/makejdk-any-platform.1 @@ -22,7 +22,7 @@ that you are building for further details. \fBExample:\fR The simplest use case to run is: -"./makejdk-any-platform.sh -D jdk8u" +"./makejdk-any-platform.sh -D fedora:latest jdk8u" This will start a Docker container and build you the latest Java 8 Temurin binary from the source at https://github.com/adoptium/openjdk-jdk8u @@ -97,8 +97,9 @@ specify the location for the built binary, e.g. /path/. This is typically used in conjunction with \fB<-T>\fR to create a custom path / file name for the resulting binary. .TP -.BR \-D ", " \-\-docker ", " \-\-podman +.BR \-D ", " \-\-docker ", " \-\-podman " " \fI[base_to_use]\fR build OpenJDK in a docker/podman container. -D will autodetect, using podman if found, docker otherwise. +You can optionally set base image to use. Default is ubuntu:18.04. Dnf/apt is determined by hardcoded list. When podman is used, there is bug requiring \fI--custom-cacerts\fR to be used under some circumstances. It is known issue: https://github.com/adoptium/temurin-build/issues/3862 .TP diff --git a/sbin/common/common.sh b/sbin/common/common.sh index 6cb632802..b4d4be965 100755 --- a/sbin/common/common.sh +++ b/sbin/common/common.sh @@ -22,19 +22,12 @@ function setOpenJdkVersion() { # # Format: jdkNN[u] # - local forest_name=$1 + local forest_name="${1}" echo "Setting version based on forest_name=${forest_name}" - - # The argument passed here have actually very strict format of jdk8, jdk8u..., jdk - # the build may fail later if this is not honoured. - # If your repository has a different name, you can use --version or build from dir/snapshot - local forest_name_check1=0 - local forest_name_check2=0 - # This two returns condition is there to make grep on solaris happy. -e, -q and \( and \| do not work on that platform - echo "$forest_name" | grep "^jdk[0-9]\\{1,3\\}[u]\\{0,1\\}$" >/dev/null || forest_name_check1=$? - echo "$forest_name" | grep "^jdk$" >/dev/null || forest_name_check2=$? - if [ ${forest_name_check1} -ne 0 ] && [ ${forest_name_check2} -ne 0 ]; then + forest_name_check=0 + checkOpenJdkVersion "${forest_name}" || forest_name_check=$? + if [ ${forest_name_check} -ne 0 ] ; then echo "The mandatory repo argument has a very strict format 'jdk[0-9]{1,3}[u]{0,1}' or just plain 'jdk' for tip. '$forest_name' does not match." echo "This can be worked around by using '--version jdkXYu'. If set (and matching) then the main argument can have any value." exit 1 @@ -94,6 +87,23 @@ function setOpenJdkVersion() { setBranch } +function checkOpenJdkVersion() { + local forest_name="${1}" + # The argument passed here has a very strict format of jdk8, jdk8u..., jdk + # the build may fail later if this is not honoured. + # If your repository has a different name, you can use --version or build from a dir/snapshot + local forest_name_check1=0 + local forest_name_check2=0 + # This two returns condition is there to make grep on solaris happy. -e, -q and \( and \| do not work on that platform + echo "$forest_name" | grep "^jdk[0-9]\\{1,3\\}[u]\\{0,1\\}$" >/dev/null || forest_name_check1=$? + echo "$forest_name" | grep "^jdk$" >/dev/null || forest_name_check2=$? + if [ ${forest_name_check1} -ne 0 ] && [ ${forest_name_check2} -ne 0 ]; then + return 1 + else + return 0 + fi +} + # Set the default BUILD_CONFIG[BRANCH] for the jdk version being built # For "hotspot" and "Temurin" builds of non-"u" jdk-23+ the branch is dev_ function setBranch() { diff --git a/sbin/common/config_init.sh b/sbin/common/config_init.sh index 97ea90d12..d2bccb4b8 100755 --- a/sbin/common/config_init.sh +++ b/sbin/common/config_init.sh @@ -47,6 +47,7 @@ CLEAN_GIT_REPO CLEAN_LIBS CONTAINER_COMMAND CONTAINER_NAME +CONTAINER_IMAGE COPY_MACOSX_FREE_FONT_LIB_FOR_JDK_FLAG COPY_MACOSX_FREE_FONT_LIB_FOR_JRE_FLAG COPY_TO_HOST @@ -274,13 +275,19 @@ function parseConfigurationArguments() { BUILD_CONFIG[TARGET_DIR]="$1"; shift;; "-D" ) - if which podman > /dev/null ; then BUILD_CONFIG[CONTAINER_COMMAND]="podman" ; else BUILD_CONFIG[CONTAINER_COMMAND]="docker" ; fi;; + if which podman > /dev/null ; then BUILD_CONFIG[CONTAINER_COMMAND]="podman" ; else BUILD_CONFIG[CONTAINER_COMMAND]="docker" ; fi; + if setCustomImage "${1-}"; then shift ; fi + ;; "--docker" ) - BUILD_CONFIG[CONTAINER_COMMAND]="docker";; + BUILD_CONFIG[CONTAINER_COMMAND]="docker"; + if setCustomImage "${1-}"; then shift ; fi + ;; "--podman" ) - BUILD_CONFIG[CONTAINER_COMMAND]="podman";; + BUILD_CONFIG[CONTAINER_COMMAND]="podman"; + if setCustomImage "${1-}"; then shift ; fi + ;; "--debug-docker" ) BUILD_CONFIG[DEBUG_DOCKER]="true";; @@ -418,6 +425,30 @@ function setOpenjdkSourceDir() { fi } +# Set custom image base if set +function setCustomImage() { + local imageCandidate="${1}" + # is next param empty? + if [[ -z ${imageCandidate} ]] ; then + echo "default image will be used: ${BUILD_CONFIG[CONTAINER_IMAGE]}" + return 1 + fi + # is the next parameter a switch? + if [[ ${imageCandidate} == -* ]] ; then + echo "default image will be used: ${BUILD_CONFIG[CONTAINER_IMAGE]}" + return 1 + fi + # is the next param a main arg? + if checkOpenJdkVersion "${imageCandidate}" ; then + echo "default image will be used: ${BUILD_CONFIG[CONTAINER_IMAGE]}" + return 1 + fi + # not empty, not switch, not main arg - therefore it must be an image, use it + BUILD_CONFIG[CONTAINER_IMAGE]="${imageCandidate}" + echo "base image will be set to: ${BUILD_CONFIG[CONTAINER_IMAGE]}" + return 0 +} + # Set the config defaults function configDefaults() { @@ -544,6 +575,7 @@ function configDefaults() { BUILD_CONFIG[DOCKER_SOURCE_VOLUME_NAME]=${BUILD_CONFIG[DOCKER_SOURCE_VOLUME_NAME]:-"openjdk-source-volume"} BUILD_CONFIG[CONTAINER_NAME]=${BUILD_CONFIG[CONTAINER_NAME]:-openjdk_container} + BUILD_CONFIG[CONTAINER_IMAGE]=${BUILD_CONFIG[CONTAINER_IMAGE]:-"ubuntu:18.04"} BUILD_CONFIG[TMP_CONTAINER_NAME]=${BUILD_CONFIG[TMP_CONTAINER_NAME]:-openjdk-copy-src} BUILD_CONFIG[CLEAN_DOCKER_BUILD]=${BUILD_CONFIG[CLEAN_DOCKER_BUILD]:-false}