From 61f33cbedcea66b4244ddbb7137ce8668a463159 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Tue, 25 Apr 2023 16:21:53 +0200 Subject: [PATCH 1/2] Simplify building of the meta-packages Signed-off-by: Evan Lezar --- debian/control | 6 +- docker/Dockerfile.amazonlinux | 40 ----- docker/{Dockerfile.ubuntu => Dockerfile.deb} | 14 +- docker/Dockerfile.debian | 53 ------- docker/Dockerfile.opensuse-leap | 40 ----- docker/{Dockerfile.centos => Dockerfile.rpm} | 12 +- docker/docker.mk | 156 +++---------------- rpm/SPECS/nvidia-container-runtime.spec | 4 +- 8 files changed, 37 insertions(+), 288 deletions(-) delete mode 100644 docker/Dockerfile.amazonlinux rename docker/{Dockerfile.ubuntu => Dockerfile.deb} (81%) delete mode 100644 docker/Dockerfile.debian delete mode 100644 docker/Dockerfile.opensuse-leap rename docker/{Dockerfile.centos => Dockerfile.rpm} (80%) diff --git a/debian/control b/debian/control index 08c9cd78..9a51306b 100644 --- a/debian/control +++ b/debian/control @@ -11,6 +11,6 @@ Build-Depends: debhelper (>= 9) Package: nvidia-container-runtime Architecture: all Depends: ${misc:Depends}, nvidia-container-toolkit (>= @TOOLKIT_VERSION@), nvidia-container-toolkit (<< 2.0.0) -Description: NVIDIA container runtime - Provides a modified version of runc allowing users to run GPU enabled - containers. +Description: NVIDIA Container Toolkit meta-package + A meta-package that allows installation flows expecting the nvidia-container-runtime + to be migrated to installing the NVIDIA Container Toolkit packages directly. diff --git a/docker/Dockerfile.amazonlinux b/docker/Dockerfile.amazonlinux deleted file mode 100644 index 36bb554f..00000000 --- a/docker/Dockerfile.amazonlinux +++ /dev/null @@ -1,40 +0,0 @@ -ARG BASEIMAGE -FROM ${BASEIMAGE} - -RUN yum install -y \ - ca-certificates \ - wget \ - git \ - rpm-build \ - make && \ - rm -rf /var/cache/yum/* - -RUN yum install -y curl - -# packaging -ARG PKG_NAME -ARG PKG_VERS -ARG PKG_REV -ARG TOOLKIT_VERSION - -ENV PKG_NAME "${PKG_NAME}" -ENV VERSION $PKG_VERS -ENV RELEASE $PKG_REV -ENV TOOLKIT_VERSION $TOOLKIT_VERSION - -# output directory -ENV DIST_DIR=/tmp/${PKG_NAME}-$PKG_VERS/SOURCES -RUN mkdir -p $DIST_DIR /dist - -WORKDIR $DIST_DIR/.. -COPY rpm . - -CMD arch=$(uname -m) && \ - rpmbuild --clean -bb \ - -D "_topdir $PWD" \ - -D "release_date $(date +'%a %b %d %Y')" \ - -D "version $VERSION" \ - -D "release $RELEASE" \ - -D "toolkit_version $TOOLKIT_VERSION" \ - SPECS/${PKG_NAME}.spec && \ - mv RPMS/noarch/*.rpm /dist diff --git a/docker/Dockerfile.ubuntu b/docker/Dockerfile.deb similarity index 81% rename from docker/Dockerfile.ubuntu rename to docker/Dockerfile.deb index 4daef760..0359e908 100644 --- a/docker/Dockerfile.ubuntu +++ b/docker/Dockerfile.deb @@ -1,32 +1,28 @@ ARG BASEIMAGE FROM ${BASEIMAGE} +# packaging dependencies ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update && apt-get install -y --no-install-recommends \ - wget \ - ca-certificates \ - git \ - build-essential \ dh-make \ fakeroot \ + build-essential \ devscripts \ lsb-release && \ rm -rf /var/lib/apt/lists/* -RUN apt-get update && \ - apt-get install -y curl && \ - rm -rf /var/lib/apt/lists/* - # packaging ARG PKG_NAME ARG PKG_VERS ARG PKG_REV ARG TOOLKIT_VERSION +ARG DOCKER_VERSION ENV DEBFULLNAME "NVIDIA CORPORATION" ENV DEBEMAIL "cudatools@nvidia.com" ENV PKG_NAME "${PKG_NAME}" ENV REVISION "$PKG_VERS-$PKG_REV" +ENV DOCKER_VERSION $DOCKER_VERSION ENV TOOLKIT_VERSION $TOOLKIT_VERSION ENV SECTION "" @@ -45,5 +41,5 @@ RUN sed -i "s;@TOOLKIT_VERSION@;${TOOLKIT_VERSION};" debian/control && \ if [ "$REVISION" != "$(dpkg-parsechangelog --show-field=Version)" ]; then exit 1; fi CMD export DISTRIB="$(lsb_release -cs)" && \ - debuild -eREVISION -eDISTRIB -eSECTION --dpkg-buildpackage-hook='sh debian/prepare' -i -us -uc -b && \ + debuild --preserve-env --dpkg-buildpackage-hook='sh debian/prepare' -i -us -uc -b && \ mv /tmp/*.deb /dist diff --git a/docker/Dockerfile.debian b/docker/Dockerfile.debian deleted file mode 100644 index 15cb79fa..00000000 --- a/docker/Dockerfile.debian +++ /dev/null @@ -1,53 +0,0 @@ -ARG BASEIMAGE -FROM ${BASEIMAGE} - -ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get update && apt-get install -y --no-install-recommends \ - wget \ - ca-certificates \ - git \ - build-essential \ - dh-make \ - fakeroot \ - devscripts \ - lsb-release && \ - rm -rf /var/lib/apt/lists/* - -RUN echo "deb http://ftp.debian.org/debian $(lsb_release -cs)-backports main" > /etc/apt/sources.list.d/backports.list - -# runc dependencies -RUN apt-get update && \ - apt-get install -y curl && \ - rm -rf /var/lib/apt/lists/* - -# packaging -ARG PKG_NAME -ARG PKG_VERS -ARG PKG_REV -ARG TOOLKIT_VERSION - -ENV DEBFULLNAME "NVIDIA CORPORATION" -ENV DEBEMAIL "cudatools@nvidia.com" -ENV PKG_NAME "${PKG_NAME}" -ENV REVISION "$PKG_VERS-$PKG_REV" -ENV TOOLKIT_VERSION $TOOLKIT_VERSION -ENV SECTION "" - -# output directory -ENV DIST_DIR=/tmp/${PKG_NAME}-$PKG_VERS -RUN mkdir -p $DIST_DIR /dist - -WORKDIR $DIST_DIR -COPY debian ./debian - -RUN rm -f $DIST_DIR/debian/changelog -RUN sed -i "s;@TOOLKIT_VERSION@;${TOOLKIT_VERSION};" debian/control && \ - dch --create --package="${PKG_NAME}" \ - --newversion "${REVISION}" \ - "Bump nvidia-container-toolkit dependency to ${TOOLKIT_VERSION}" && \ - dch -r "" && \ - if [ "$REVISION" != "$(dpkg-parsechangelog --show-field=Version)" ]; then exit 1; fi - -CMD export DISTRIB="$(lsb_release -cs)" && \ - debuild -eREVISION -eDISTRIB -eSECTION --dpkg-buildpackage-hook='sh debian/prepare' -i -us -uc -b && \ - mv /tmp/*.deb /dist \ No newline at end of file diff --git a/docker/Dockerfile.opensuse-leap b/docker/Dockerfile.opensuse-leap deleted file mode 100644 index 5b6a73c6..00000000 --- a/docker/Dockerfile.opensuse-leap +++ /dev/null @@ -1,40 +0,0 @@ -ARG BASEIMAGE -FROM ${BASEIMAGE} - -RUN zypper install -y \ - ca-certificates \ - wget \ - git \ - rpm-build && \ - rm -rf /var/cache/zypp/* - -RUN zypper install -y curl make && \ - rm -rf /var/cache/zypp/* - -# packaging -ARG PKG_NAME -ARG PKG_VERS -ARG PKG_REV -ARG TOOLKIT_VERSION - -ENV PKG_NAME "${PKG_NAME}" -ENV VERSION $PKG_VERS -ENV RELEASE $PKG_REV -ENV TOOLKIT_VERSION $TOOLKIT_VERSION - -# output directory -ENV DIST_DIR=/tmp/${PKG_NAME}-$PKG_VERS/SOURCES -RUN mkdir -p $DIST_DIR /dist - -WORKDIR $DIST_DIR/.. -COPY rpm . - -CMD arch=$(uname -m) && \ - rpmbuild --clean -bb \ - -D "_topdir $PWD" \ - -D "release_date $(date +'%a %b %d %Y')" \ - -D "version $VERSION" \ - -D "release $RELEASE" \ - -D "toolkit_version $TOOLKIT_VERSION" \ - SPECS/${PKG_NAME}.spec && \ - mv RPMS/noarch/*.rpm /dist diff --git a/docker/Dockerfile.centos b/docker/Dockerfile.rpm similarity index 80% rename from docker/Dockerfile.centos rename to docker/Dockerfile.rpm index c899c622..382253bd 100644 --- a/docker/Dockerfile.centos +++ b/docker/Dockerfile.rpm @@ -1,11 +1,8 @@ ARG BASEIMAGE FROM ${BASEIMAGE} +# packaging dependencies RUN yum install -y \ - ca-certificates \ - wget \ - git \ - make \ rpm-build && \ rm -rf /var/cache/yum/* @@ -14,10 +11,12 @@ ARG PKG_NAME ARG PKG_VERS ARG PKG_REV ARG TOOLKIT_VERSION +ARG DOCKER_VERSION ENV PKG_NAME "${PKG_NAME}" ENV VERSION $PKG_VERS ENV RELEASE $PKG_REV +ENV DOCKER_VERSION $DOCKER_VERSION ENV TOOLKIT_VERSION $TOOLKIT_VERSION # output directory @@ -27,12 +26,11 @@ RUN mkdir -p $DIST_DIR /dist WORKDIR $DIST_DIR/.. COPY rpm . -CMD arch=$(uname -m) && \ - rpmbuild --clean -bb \ +CMD rpmbuild --clean -bb \ -D "_topdir $PWD" \ -D "release_date $(date +'%a %b %d %Y')" \ -D "version $VERSION" \ -D "release $RELEASE" \ -D "toolkit_version $TOOLKIT_VERSION" \ - SPECS/${PKG_NAME}.spec && \ + SPECS/nvidia-container-runtime.spec && \ mv RPMS/noarch/*.rpm /dist diff --git a/docker/docker.mk b/docker/docker.mk index d933015f..cf482039 100644 --- a/docker/docker.mk +++ b/docker/docker.mk @@ -16,164 +16,52 @@ DOCKER ?= docker MKDIR ?= mkdir DIST_DIR ?= $(CURDIR)/dist -# Supported OSs by architecture -AMD64_TARGETS := ubuntu20.04 ubuntu18.04 ubuntu16.04 debian10 debian9 -X86_64_TARGETS := fedora35 centos7 centos8 rhel7 rhel8 amazonlinux2 opensuse-leap15.1 -PPC64LE_TARGETS := ubuntu18.04 ubuntu16.04 centos7 centos8 rhel7 rhel8 -ARM64_TARGETS := ubuntu20.04 ubuntu18.04 -AARCH64_TARGETS := fedora35 centos8 rhel8 amazonlinux2 +# Supported packaging formats +FORMAT_TARGETS := deb rpm -# By default run all native docker-based targets -docker-native: +# We add utility targets to support common os-arch combinations by mapping to the required format targets. +DEB_TARGETS := debian10-amd64 ubuntu18.04-amd64 ubuntu18.04-arm64 ubuntu18.04-ppc64le +RPM_TARGETS := amazonlinux2-aarch64 amazonlinux2-x86_64 centos7-x86_64 centos8-aarch64 centos8-ppc64le centos8-x86_64 opensuse-leap15.1-x86_64 + +$(DEB_TARGETS): %: deb +$(RPM_TARGETS): %: rpm # Define top-level build targets docker%: SHELL:=/bin/bash -# Native targets -PLATFORM ?= $(shell uname -m) -ifeq ($(PLATFORM),x86_64) -NATIVE_TARGETS := $(AMD64_TARGETS) $(X86_64_TARGETS) -$(AMD64_TARGETS): %: %-amd64 -$(X86_64_TARGETS): %: %-x86_64 -else ifeq ($(PLATFORM),ppc64le) -NATIVE_TARGETS := $(PPC64LE_TARGETS) -$(PPC64LE_TARGETS): %: %-ppc64le -else ifeq ($(PLATFORM),aarch64) -NATIVE_TARGETS := $(ARM64_TARGETS) $(AARCH64_TARGETS) -$(ARM64_TARGETS): %: %-arm64 -$(AARCH64_TARGETS): %: %-aarch64 -endif -docker-native: $(NATIVE_TARGETS) - -# amd64 targets -AMD64_TARGETS := $(patsubst %, %-amd64, $(AMD64_TARGETS)) -$(AMD64_TARGETS): ARCH := amd64 -$(AMD64_TARGETS): %: --% -docker-amd64: $(AMD64_TARGETS) - -# x86_64 targets -X86_64_TARGETS := $(patsubst %, %-x86_64, $(X86_64_TARGETS)) -$(X86_64_TARGETS): ARCH := x86_64 -$(X86_64_TARGETS): %: --% -docker-x86_64: $(X86_64_TARGETS) - -# arm64 targets -ARM64_TARGETS := $(patsubst %, %-arm64, $(ARM64_TARGETS)) -$(ARM64_TARGETS): ARCH := arm64 -$(ARM64_TARGETS): %: --% -docker-arm64: $(ARM64_TARGETS) - -# aarch64 targets -AARCH64_TARGETS := $(patsubst %, %-aarch64, $(AARCH64_TARGETS)) -$(AARCH64_TARGETS): ARCH := aarch64 -$(AARCH64_TARGETS): %: --% -docker-aarch64: $(AARCH64_TARGETS) - -# ppc64le targets -PPC64LE_TARGETS := $(patsubst %, %-ppc64le, $(PPC64LE_TARGETS)) -$(PPC64LE_TARGETS): ARCH := ppc64le -$(PPC64LE_TARGETS): WITH_LIBELF := yes -$(PPC64LE_TARGETS): %: --% -docker-ppc64le: $(PPC64LE_TARGETS) - -# docker target to build for all os/arch combinations -docker-all: $(AMD64_TARGETS) $(X86_64_TARGETS) \ - $(ARM64_TARGETS) $(AARCH64_TARGETS) \ - $(PPC64LE_TARGETS) +$(FORMAT_TARGETS): %: --% # Default variables for all private '--' targets below. # One private target is defined for each OS we support. ---%: TARGET_PLATFORM = $(*) ---%: VERSION = $(patsubst $(OS)%-$(ARCH),%,$(TARGET_PLATFORM)) ---%: BASEIMAGE = $(OS):$(VERSION) ---%: BUILDIMAGE = nvidia/$(LIB_NAME)/$(OS)$(VERSION)-$(ARCH) ---%: DOCKERFILE = $(CURDIR)/docker/Dockerfile.$(OS) ---%: ARTIFACTS_DIR = $(DIST_DIR)/$(OS)$(VERSION)/$(ARCH) +--%: FORMAT = $(*) +--%: BUILDIMAGE = nvidia/$(LIB_NAME)/$(FORMAT)-all +--%: DOCKERFILE = $(CURDIR)/docker/Dockerfile.$(FORMAT) +--%: ARTIFACTS_DIR = $(DIST_DIR)/$(FORMAT)/all --%: docker-build-% @ -DEB_LIB_VERSION = $(LIB_VERSION)$(if $(LIB_TAG),~$(LIB_TAG)) -DEB_PKG_REV = 1 -DEB_TOOLKIT_VERSION = $(TOOLKIT_VERSION)$(if $(TOOLKIT_TAG),~$(TOOLKIT_TAG)) -DEB_TOOLKIT_REV = 1 - -RPM_LIB_VERSION = $(LIB_VERSION) -RPM_PKG_REV = $(if $(LIB_TAG),0.1.$(LIB_TAG),1) -RPM_TOOLKIT_VERSION = $(TOOLKIT_VERSION) -RPM_TOOLKIT_REV = $(if $(TOOLKIT_TAG),0.1.$(TOOLKIT_TAG),1) - -# private OS targets with defaults -# private ubuntu target ---ubuntu%: OS := ubuntu ---ubuntu%: PKG_VERS = $(DEB_LIB_VERSION) ---ubuntu%: PKG_REV = $(DEB_PKG_REV) ---ubuntu%: MIN_TOOLKIT_PKG_VERSION = $(DEB_TOOLKIT_VERSION)-$(DEB_TOOLKIT_REV) - -# private debian target ---debian%: OS := debian ---debian%: PKG_VERS = $(DEB_LIB_VERSION) ---debian%: PKG_REV = $(DEB_PKG_REV) ---debian%: MIN_TOOLKIT_PKG_VERSION = $(DEB_TOOLKIT_VERSION)-$(DEB_TOOLKIT_REV) - - -# private centos target ---centos%: OS := centos ---centos%: PKG_VERS = $(LIB_VERSION) ---centos%: PKG_REV = $(RPM_PKG_REV) ---centos%: MIN_TOOLKIT_PKG_VERSION = $(RPM_TOOLKIT_VERSION)-$(RPM_TOOLKIT_REV) ---centos8%: BASEIMAGE = quay.io/centos/centos:stream8 - -# private amazonlinux target ---amazonlinux%: OS := amazonlinux ---amazonlinux%: PKG_VERS = $(LIB_VERSION) ---amazonlinux%: PKG_REV = $(RPM_PKG_REV) ---amazonlinux%: MIN_TOOLKIT_PKG_VERSION = $(RPM_TOOLKIT_VERSION)-$(RPM_TOOLKIT_REV) - -# private opensuse-leap target with overrides ---opensuse-leap%: OS := opensuse-leap ---opensuse-leap%: PKG_VERS = $(LIB_VERSION) ---opensuse-leap%: PKG_REV = $(RPM_PKG_REV) ---opensuse-leap%: MIN_TOOLKIT_PKG_VERSION = $(RPM_TOOLKIT_VERSION)-$(RPM_TOOLKIT_REV) ---opensuse-leap%: BASEIMAGE = opensuse/leap:$(VERSION) - -# private rhel target (actually built on centos) ---rhel%: OS := centos ---rhel%: PKG_VERS = $(LIB_VERSION) ---rhel%: PKG_REV = $(RPM_PKG_REV) ---rhel%: MIN_TOOLKIT_PKG_VERSION = $(RPM_TOOLKIT_VERSION)-$(RPM_TOOLKIT_REV) ---rhel%: VERSION = $(patsubst rhel%-$(ARCH),%,$(TARGET_PLATFORM)) ---rhel%: ARTIFACTS_DIR = $(DIST_DIR)/rhel$(VERSION)/$(ARCH) ---rhel8%: BASEIMAGE = quay.io/centos/centos:stream8 +PKG_VERS = $(LIB_VERSION)$(if $(LIB_TAG),~$(LIB_TAG)) +PKG_REV = 1 +MIN_TOOLKIT_PKG_VERSION = $(TOOLKIT_VERSION)$(if $(TOOLKIT_TAG),~$(TOOLKIT_TAG))-1 -# private fedora target (actually built on centos) ---fedora%: OS := centos ---fedora%: PKG_VERS = $(LIB_VERSION) ---fedora%: PKG_REV = $(RPM_PKG_REV) ---fedora%: MIN_TOOLKIT_PKG_VERSION = $(RPM_TOOLKIT_VERSION)-$(RPM_TOOLKIT_REV) ---fedora%: VERSION = $(patsubst fedora%-$(ARCH),%,$(TARGET_PLATFORM)) ---fedora%: ARTIFACTS_DIR = $(DIST_DIR)/fedora$(VERSION)/$(ARCH) ---fedora%: BASEIMAGE = quay.io/centos/centos:stream8 +--deb: BASEIMAGE := ubuntu:18.04 -PLATFORM_ARGS ?= --platform=linux/$(ARCH) -ifneq ($(strip $(ADD_DOCKER_PLATFORM_ARGS)),) -DOCKER_PLATFORM_ARGS = $(PLATFORM_ARGS) -endif +--rpm: BASEIMAGE := quay.io/centos/centos:stream8 docker-build-%: - @echo "Building for $(TARGET_PLATFORM)" - docker pull $(PLATFORM_ARGS) $(BASEIMAGE) + @echo "Building $(FORMAT) packages to $(ARTIFACTS_DIR)" + docker pull $(BASEIMAGE) DOCKER_BUILDKIT=1 \ - $(DOCKER) build $(DOCKER_PLATFORM_ARGS) \ + $(DOCKER) build \ --progress=plain \ --build-arg BASEIMAGE="$(BASEIMAGE)" \ - --build-arg GOLANG_VERSION="$(GOLANG_VERSION)" \ --build-arg TOOLKIT_VERSION="$(MIN_TOOLKIT_PKG_VERSION)" \ --build-arg PKG_NAME="$(LIB_NAME)" \ --build-arg PKG_VERS="$(PKG_VERS)" \ --build-arg PKG_REV="$(PKG_REV)" \ --tag $(BUILDIMAGE) \ --file $(DOCKERFILE) . - $(DOCKER) run $(DOCKER_PLATFORM_ARGS) \ + $(DOCKER) run \ -e DISTRIB \ -e SECTION \ -v $(ARTIFACTS_DIR):/dist \ diff --git a/rpm/SPECS/nvidia-container-runtime.spec b/rpm/SPECS/nvidia-container-runtime.spec index 15974e03..4ec11f07 100644 --- a/rpm/SPECS/nvidia-container-runtime.spec +++ b/rpm/SPECS/nvidia-container-runtime.spec @@ -18,8 +18,8 @@ Obsoletes: nvidia-container-runtime < 2.0.0 Requires: nvidia-container-toolkit >= %{toolkit_version}, nvidia-container-toolkit < 2.0.0 %description -Provides a modified version of runc allowing users to run GPU enabled -containers. +A meta-package that allows installation flows expecting the nvidia-container-runtime +to be migrated to installing the NVIDIA Container Toolkit packages directly. %prep cp %{SOURCE0} . From ea1dd5cab637ec5938c72635369b8791d132db59 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Tue, 25 Apr 2023 16:26:03 +0200 Subject: [PATCH 2/2] Update CI Signed-off-by: Evan Lezar --- .gitlab-ci.yml | 131 +++++++++++++++++++------------------------------ 1 file changed, 51 insertions(+), 80 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e377a849..34ffc491 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -18,11 +18,6 @@ default: - name: docker:dind command: ["--experimental"] -# Build packages for all supported OS / ARCH combinations -stages: - - build-one - - build-all - variables: # We specify the LIB_VERSION, TOOLKIT_VERSION, and TOOLKIT_TAG variable to allow packages # to be built. @@ -31,45 +26,51 @@ variables: TOOLKIT_VERSION: 999.999.999 TOOLKIT_TAG: dummy+toolkit -.build-setup: &build-setup - before_script: - - apk update - - apk upgrade - - apk add coreutils build-base sed git bash make - - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes -c yes +# Build packages for all supported OS / ARCH combinations +stages: + - trigger + - build -# build-one jobs build packages for a single OS / ARCH combination. -# -# They are run during the first stage of the pipeline as a smoke test to ensure -# that we can successfully build packages on all of our architectures for a -# single OS. They are triggered on any change to an MR. No artifacts are -# produced as part of build-one jobs. -.build-one-setup: &build-one-setup - <<: *build-setup - stage: build-one - only: - - merge_requests +.pipeline-trigger-rules: + rules: + # We trigger the pipeline if started manually + - if: $CI_PIPELINE_SOURCE == "web" + # We trigger the pipeline on the main branch + - if: $CI_COMMIT_BRANCH == "main" + # We trigger the pipeline on the release- branches + - if: $CI_COMMIT_BRANCH =~ /^release-.*$/ + # We trigger the pipeline on tags + - if: $CI_COMMIT_TAG && $CI_COMMIT_TAG != "" + +workflow: + rules: + # We trigger the pipeline on a merge request + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + # We then add all the regular triggers + - !reference [.pipeline-trigger-rules, rules] + +# The main or manual job is used to filter out distributions or architectures that are not required on +# every build. +.main-or-manual: + rules: + - !reference [.pipeline-trigger-rules, rules] + - if: $CI_PIPELINE_SOURCE == "schedule" + when: manual -# build-all jobs build packages for every OS / ARCH combination we support. -# -# They are run under two conditions: -# 1) Automatically whenever a new tag is pushed to the repo (e.g. v1.1.0) -# 2) Manually by a reviewer just before merging a MR. -# -# Unlike build-one jobs, it takes a long time to build the full suite -# OS / ARCH combinations, so this is optimized to only run once per MR -# (assuming it all passes). A full set of artifacts including the packages -# built for each OS / ARCH are produced as a result of these jobs. -.build-all-setup: &build-all-setup - <<: *build-setup - stage: build-all - timeout: 2h 30m +# The trigger-pipeline job adds a manualy triggered job to the pipeline on merge requests. +trigger-pipeline: + stage: trigger + script: + - echo "starting pipeline" rules: - - if: $CI_COMMIT_TAG - when: always - - if: $CI_MERGE_REQUEST_ID + - !reference [.main-or-manual, rules] + - if: $CI_PIPELINE_SOURCE == "merge_request_event" when: manual + allow_failure: false + - when: always +.build-setup: + stage: build variables: ARTIFACTS_NAME: "${CI_PROJECT_NAME}-${CI_COMMIT_REF_SLUG}-${CI_JOB_NAME}-artifacts-${CI_PIPELINE_ID}" ARTIFACTS_DIR: "${CI_PROJECT_NAME}-${CI_COMMIT_REF_SLUG}-artifacts-${CI_PIPELINE_ID}" @@ -80,46 +81,16 @@ variables: paths: - ${ARTIFACTS_DIR} -# The full set of build-one jobs organizes to build -# ubuntu18.04 in parallel on each of our supported ARCHs. -build-one-amd64: - <<: *build-one-setup - script: - - make ubuntu18.04-amd64 - -build-one-ppc64le: - <<: *build-one-setup - script: - - make ubuntu18.04-ppc64le - -build-one-arm64: - <<: *build-one-setup - script: - - make ubuntu18.04-arm64 - -# The full set of build-all jobs organized to -# have builds for each ARCH run in parallel. -build-all-amd64: - <<: *build-all-setup - script: - - make docker-amd64 - -build-all-x86_64: - <<: *build-all-setup - script: - - make docker-x86_64 - -build-all-ppc64le: - <<: *build-all-setup - script: - - make docker-ppc64le - -build-all-arm64: - <<: *build-all-setup - script: - - make docker-arm64 + before_script: + - apk update + - apk upgrade + - apk add coreutils build-base sed git bash make -build-all-aarch64: - <<: *build-all-setup +build: + extends: + - .build-setup + parallel: + matrix: + - PACKAGING: [deb, rpm] script: - - make docker-aarch64 + - make ${PACKAGING}