From cdd1c398982860122260fc6c51a4a21aa1f8200b Mon Sep 17 00:00:00 2001 From: Gibson Chikafa Date: Mon, 24 Jun 2024 15:29:59 +0200 Subject: [PATCH] Build docker image with hops hadoop distribution --- Jenkinsfile | 41 +++++++++++++++++++ build_branch.txt | 1 + deploy/kubernetes/docker/Dockerfile | 27 ++++-------- .../kubernetes/docker/base/debian/Dockerfile | 2 +- deploy/kubernetes/docker/build.sh | 41 +++++++++++++------ version.txt | 1 + 6 files changed, 79 insertions(+), 34 deletions(-) create mode 100644 Jenkinsfile create mode 100644 build_branch.txt create mode 100644 version.txt diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 0000000000..b915abc3df --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,41 @@ +pipeline { + agent { + label "local" + } + environment { + VERSION = readFile "${env.WORKSPACE}/version.txt" + BUILD_BRANCH = readFile "${env.WORKSPACE}/build_branch.txt" + } + stages { + stage("build and publish") { + agent { + label "local" + } + steps { + withCredentials([usernamePassword(credentialsId: 'cred', passwordVariable: 'NEXUS_CREDS_PSW', usernameVariable: 'NEXUS_CREDS_USR')]) { + sh """ + set -ex + docker login -u ${NEXUS_CREDS_USR} -p ${NEXUS_CREDS_PSW} + git checkout ${BUILD_BRANCH} + ./build_distribution.sh --spark-profile spark3 --hadoop-profile hadoop3.2 --without-dashboard + cd deploy/kubernetes/docker || exit + ./build.sh --hadoop-version 3.2.0.13-EE-SNAPSHOT --registry docker.hops.works --nexus-user $NEXUS_CREDS_USR --nexus-password $NEXUS_CREDS_PSW + cd ../../.. + + mkdir -p /opt/repository/master/rss/${VERSION}/ + cp client/target/rss-client-${VERSION}.jar /opt/repository/master/rss/${VERSION}/ + """ + } + } + } + } + + post { + success { + build job:'Remote shuffle services', parameters: [ + string(name: 'image', value: "rss"), + string(name: 'branch', value: "master") + ] + } + } +} \ No newline at end of file diff --git a/build_branch.txt b/build_branch.txt new file mode 100644 index 0000000000..4b52459bfe --- /dev/null +++ b/build_branch.txt @@ -0,0 +1 @@ +branch-0.9 \ No newline at end of file diff --git a/deploy/kubernetes/docker/Dockerfile b/deploy/kubernetes/docker/Dockerfile index 6f665ac09d..acda83924e 100644 --- a/deploy/kubernetes/docker/Dockerfile +++ b/deploy/kubernetes/docker/Dockerfile @@ -1,19 +1,4 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# +# syntax=docker/dockerfile:experimental ARG BASE_IMAGE=uniffle-base:latest FROM ${BASE_IMAGE} @@ -24,6 +9,7 @@ ARG RSS_VERSION ARG AUTHOR=unknown ARG GIT_BRANCH=unknown ARG GIT_COMMIT=unknown +ARG HADOOP_FILE=hops-$HADOOP_VERSION.tgz LABEL author=${AUTHOR} LABEL git-branch=${GIT_BRANCH} @@ -44,10 +30,11 @@ COPY rss-env.sh /data/rssadmin/rss/bin COPY start.sh /data/rssadmin/rss/bin -COPY hadoop-${HADOOP_VERSION}.tar.gz /data/rssadmin -RUN tar -zxvf /data/rssadmin/hadoop-${HADOOP_VERSION}.tar.gz -C /data/rssadmin -RUN mv /data/rssadmin/hadoop-${HADOOP_VERSION} /data/rssadmin/hadoop -RUN rm -rf /data/rssadmin/hadoop-${HADOOP_VERSION}.tar.gz +COPY $HADOOP_FILE /data/rssadmin +RUN mkdir -p /data/rssadmin/hops-${HADOOP_VERSION} +RUN tar -zxvf /data/rssadmin/$HADOOP_FILE --strip-components=1 -C /data/rssadmin/hops-${HADOOP_VERSION} +RUN mv /data/rssadmin/hops-${HADOOP_VERSION} /data/rssadmin/hadoop +RUN rm -rf /data/rssadmin/$HADOOP_FILE COPY hadoopconfig/ /data/rssadmin/hadoop/etc/hadoop ENV RSS_VERSION ${RSS_VERSION} diff --git a/deploy/kubernetes/docker/base/debian/Dockerfile b/deploy/kubernetes/docker/base/debian/Dockerfile index 901d0fd16e..a84f7674dd 100644 --- a/deploy/kubernetes/docker/base/debian/Dockerfile +++ b/deploy/kubernetes/docker/base/debian/Dockerfile @@ -20,5 +20,5 @@ ARG BASE_IMAGE=eclipse-temurin:11-jdk FROM ${BASE_IMAGE} RUN apt-get update && \ - apt-get install -y zlib1g zlib1g-dev lzop lsof netcat dnsutils less procps iputils-ping && \ + apt-get install -y zlib1g zlib1g-dev lzop lsof netcat dnsutils less procps iputils-ping curl && \ apt-get clean && rm -rf /var/lib/apt/lists/* diff --git a/deploy/kubernetes/docker/build.sh b/deploy/kubernetes/docker/build.sh index 58478c3520..09ef3ee00b 100755 --- a/deploy/kubernetes/docker/build.sh +++ b/deploy/kubernetes/docker/build.sh @@ -29,14 +29,14 @@ function exit_with_usage() { echo "+------------------------------------------------------------------------------------------------------+" echo "| ./build.sh [--hadoop-version ] [--registry ] [--author ] |" echo "| [--base-os-distribution ] [--base-image ] |" - echo "| [--push-image ] [--apache-mirror ] |" + echo "| [--push-image ] [--apache-mirror ] [--nexus-user ] + [--nexus-password ] |" echo "+------------------------------------------------------------------------------------------------------+" exit 1 } REGISTRY="docker.io/library" -HADOOP_VERSION=2.8.5 -HADOOP_SHORT_VERSION=$(echo $HADOOP_VERSION | awk -F "." '{print $1"."$2}') +HADOOP_VERSION=3.2.0.13-SNAPSHOT AUTHOR=$(whoami) # If you are based in China, you could pass --apache-mirror when building this. APACHE_MIRROR="https://dlcdn.apache.org" @@ -70,6 +70,14 @@ while (( "$#" )); do PUSH_IMAGE="$2" shift ;; + --nexus-user) + NEXUS_USER="$2" + shift + ;; + --nexus-password) + NEXUS_PASSWORD="$2" + shift + ;; --help) exit_with_usage ;; @@ -92,6 +100,8 @@ while (( "$#" )); do shift done +HADOOP_SHORT_VERSION=$(echo $HADOOP_VERSION | awk -F "." '{print $1"."$2}') + if [ -z "$BASE_IMAGE" ]; then echo "start building base image: uniffle-base" docker build -t "uniffle-base:latest" \ @@ -101,14 +111,15 @@ else echo "using base image(${BASE_IMAGE}) to build rss server" fi - -HADOOP_FILE=hadoop-${HADOOP_VERSION}.tar.gz -ARCHIVE_HADOOP_URL=https://archive.apache.org/dist/hadoop/core/hadoop-${HADOOP_VERSION}/${HADOOP_FILE} -HADOOP_URL=${APACHE_MIRROR}/hadoop/core/hadoop-${HADOOP_VERSION}/${HADOOP_FILE} -echo "HADOOP_URL is either ${HADOOP_URL} or ${ARCHIVE_HADOOP_URL}" -if [ ! -e "$HADOOP_FILE" ]; \ - then wget "${HADOOP_URL}" || wget "$ARCHIVE_HADOOP_URL"; \ - else echo "${HADOOP_FILE} has been downloaded"; \ +HADOOP_FILE=hops-$HADOOP_VERSION.tgz +if [ ! -e "$HADOOP_FILE" ]; then + if [[ -n "$NEXUS_USER" ]] && [[ -n "$NEXUS_PASSWORD" ]] ; then + wget --user $NEXUS_USER --password $NEXUS_PASSWORD https://nexus.hops.works/repository/hopshadoop/$HADOOP_FILE + else + wget https://repo.hops.works/master/$HADOOP_FILE + fi +else + echo "${HADOOP_FILE} has been downloaded"; fi RSS_DIR=../../.. @@ -126,10 +137,13 @@ cp "$RSS_DIR/$RSS_FILE" . GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD) GIT_COMMIT=$(git describe --dirty --always --tags | sed 's/-/./g') -echo "image version: ${IMAGE_VERSION:=$RSS_VERSION-$GIT_COMMIT}" +IMAGE_VERSION=$(cat $RSS_DIR/version.txt | awk '{$1=$1; print}' | sed '/^$/d') IMAGE=$REGISTRY/rss-server:$IMAGE_VERSION +echo "image version: ${IMAGE_VERSION}" +IMAGE=$REGISTRY/rss:$IMAGE_VERSION echo "building image: $IMAGE" -docker build --network=host -t "$IMAGE" \ + +DOCKER_BUILDKIT=1 docker build --network=host -t "$IMAGE" \ --build-arg RSS_VERSION="$RSS_VERSION" \ --build-arg HADOOP_VERSION="$HADOOP_VERSION" \ --build-arg HADOOP_SHORT_VERSION="$HADOOP_SHORT_VERSION" \ @@ -137,6 +151,7 @@ docker build --network=host -t "$IMAGE" \ --build-arg GIT_COMMIT="$GIT_COMMIT" \ --build-arg GIT_BRANCH="$GIT_BRANCH" \ --build-arg BASE_IMAGE="$BASE_IMAGE" \ + --build-arg HADOOP_FILE="$HADOOP_FILE" \ -f Dockerfile --no-cache . if [ x"${PUSH_IMAGE}" == x"true" ]; then diff --git a/version.txt b/version.txt new file mode 100644 index 0000000000..899f24fc75 --- /dev/null +++ b/version.txt @@ -0,0 +1 @@ +0.9.0 \ No newline at end of file