From bf6c92e65f90feb25f9329939faff929558ad82d Mon Sep 17 00:00:00 2001 From: Dave Dykstra <2129743+DrDaveD@users.noreply.github.com> Date: Mon, 9 Sep 2024 15:03:51 -0500 Subject: [PATCH 1/2] add ci checks --- .github/workflows/build.yml | 43 ++++++++++++++++++++++++++ ChangeLog | 4 +++ ci/docker-run | 24 +++++++++++++++ ci/make-and-test | 61 +++++++++++++++++++++++++++++++++++++ ci/privileged-run | 25 +++++++++++++++ makedist | 8 +++-- 6 files changed, 163 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/build.yml create mode 100755 ci/docker-run create mode 100755 ci/make-and-test create mode 100755 ci/privileged-run diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..1fe2ee9 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,43 @@ +name: Make distributions and test + +on: + # run workflows on main master and release/** branches + push: + branches: + - main + - master + - release/** + # run workflows on pull requests against the same branches + pull_request: + branches: + - main + - master + - release/** + +# automatically cancel redundant builds +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + makeandtest: + name: ${{ matrix.distro }}:${{ matrix.version }} + strategy: + fail-fast: false + matrix: + include: + - distro: eurolinux/centos-7 + version: latest + - distro: rockylinux + version: 8 + - distro: rockylinux + version: 9 + - distro: opensuse/leap + version: 15 + runs-on: ubuntu-24.04 + steps: + - name: Get source code + uses: actions/checkout@v3 + + - name: Run make-and-test under docker + run: ./ci/docker-run ${{ matrix.distro }}:${{ matrix.version }} diff --git a/ChangeLog b/ChangeLog index 68fa63b..7b0dcf7 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,7 @@ +- Switch to the archive for el7 epel packages. +- Add CI checks on every pull request for a fairly through test of + all the combinations of modes, distributions, and machine types. + cvmfsexec-4.40 - 5 July 2024 - Switch to vault.centos.org for el7 fuse-libs package. diff --git a/ci/docker-run b/ci/docker-run new file mode 100755 index 0000000..97c063d --- /dev/null +++ b/ci/docker-run @@ -0,0 +1,24 @@ +#!/bin/bash -e +# Run docker as shown at +# https://djw8605.github.io/2016/05/03/building-centos-packages-on-travisci/ +# +# This more complicated setup is needed for github actions too because +# they do not provide a mechanism for reliably enabling user namespaces. +# Github actions does at least start a VM with docker already running. + +# Assumes running on Ubuntu 24+ + +DOCKER_HUB_URI="$1" +docker pull "$DOCKER_HUB_URI" + +DOCKER_CONTAINER_NAME="test_${OS_TYPE##*/}_${OS_VERSION//./_}" + +set -x +docker run --privileged --network=host -v "$(pwd):/source:rw" \ + -e DOCKER_HUB_URI="$DOCKER_HUB_URI" \ + --name "$DOCKER_CONTAINER_NAME" "$DOCKER_HUB_URI" /bin/bash -exc \ + "cd /source && ./ci/privileged-run" + +docker ps -a +docker stop "$DOCKER_CONTAINER_NAME" +docker rm -v "$DOCKER_CONTAINER_NAME" diff --git a/ci/make-and-test b/ci/make-and-test new file mode 100755 index 0000000..97db76d --- /dev/null +++ b/ci/make-and-test @@ -0,0 +1,61 @@ +#!/bin/bash +# Now running as an unprivileged user with user namespaces enabled in a +# container and required packages installed. Make distributions and test. + +. /etc/os-release + +set -ex +for DIST in default osg egi; do + if [ "$DIST" = egi ] && [[ "$VERSION_ID" != 7* ]]; then + # egi not yet supported for el8 or el9 + continue + fi + rm -rf dist /tmp/cvmfsexec + : test makedist + ./makedist $DIST + : test mode 3 + ./cvmfsexec atlas.cern.ch -- ls /cvmfs/atlas.cern.ch/repo + : test self-extracting distribution + ./makedist -o /tmp/cvmfsexec + /tmp/cvmfsexec atlas.cern.ch -- ls /cvmfs/atlas.cern.ch/repo + ./makedist -o /tmp/cvmfsexec + : test mode 1 + rm -rf dist/var/lib/cvmfs/shared + ./mountrepo `cd dist/cvmfs; echo *config*` + ./mountrepo atlas.cern.ch + ls dist/cvmfs/atlas.cern.ch/repo + if [[ "$ID" == *suse* ]]; then + # the sle15 cvmfs suse build as of 2.11.5 does not support fuse3 + ./umountrepo -a + continue + fi + : test mode 4 + if [ ! -d apptainer ]; then + # get singularity or apptainer from oasis cvmfs + ./mountrepo oasis.opensciencegrid.org + if [[ "$VERSION_ID" == 7* ]]; then + APPT=dist/cvmfs/oasis.opensciencegrid.org/mis/singularity/current + else + APPT=dist/cvmfs/oasis.opensciencegrid.org/mis/apptainer/current + fi + mkdir apptainer + cp -r $APPT/`arch` $APPT/bin apptainer + fi + ./umountrepo -a + rm -rf dist /tmp/cvmfsexec + ./makedist -s $DIST + ./makedist -s -o /tmp/cvmfsexec + SINGCVMFS_REPOSITORIES=atlas.cern.ch PATH=$PATH:$PWD/apptainer/bin \ + ./singcvmfs exec -cip docker://$DOCKER_HUB_URI ls /cvmfs/atlas.cern.ch/repo + if [[ "$VERSION_ID" == 8* ]]; then + : try to at least make other supported architecture distributions + for arch in aarch64 ppc64le; do + rm -rf dist /tmp/cvmfsexec + ./makedist -m rhel8-$arch $DIST + ./makedist -o /tmp/cvmfsexec + rm -rf dist /tmp/cvmfsexec + ./makedist -s -m rhel8-$arch $DIST + ./makedist -s -o /tmp/cvmfsexec + done + fi +done diff --git a/ci/privileged-run b/ci/privileged-run new file mode 100755 index 0000000..4b405a6 --- /dev/null +++ b/ci/privileged-run @@ -0,0 +1,25 @@ +#!/bin/bash +# Running in a privileged container. Install required packages and +# switch to an unprivileged user to run the tests. + +set -ex +if [ -f /usr/bin/zypper ]; then + # suse + zypper install -y tar gzip openssl-1_1 fuse fuse3 +else + # rhel + yum install -y procps-ng cpio findutils fuse fuse3 + if [[ $DOCKER_HUB_URI == *:8 ]]; then + yum install -y yum-utils + fi +fi + +# because host kernel is Ubuntu 24+, this enables user namespaces +sysctl kernel.apparmor_restrict_unprivileged_userns=0 + +# switch to an unprivileged user +useradd -u 1000 --create-home -s /bin/bash testuser +# leave .git as original owner for post job cleanup +chown testuser . +chown -R testuser * +su testuser -c ci/make-and-test diff --git a/makedist b/makedist index 199829d..3db1ec5 100755 --- a/makedist +++ b/makedist @@ -185,6 +185,10 @@ fi echo "Making $SINGMSG$DISTTYPE distribution for $MACHTYPE" getcoprurl() { + if [ ! -f /usr/bin/yumdownloader ]; then + echo "yumdownloader not found, skipping trying to get $1 from copr" >&2 + return + fi typeset TMPF=$(mktemp) typeset REPONAME=makedist-$1 cat >$TMPF < Date: Mon, 9 Sep 2024 15:05:31 -0500 Subject: [PATCH 2/2] update to 4.41 --- ChangeLog | 1 + cvmfsexec | 2 +- singcvmfs | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index 7b0dcf7..888bced 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,4 @@ +cvmfsexec-4.41 - 9 September 2024 - Switch to the archive for el7 epel packages. - Add CI checks on every pull request for a fairly through test of all the combinations of modes, distributions, and machine types. diff --git a/cvmfsexec b/cvmfsexec index 4f24330..758a95b 100755 --- a/cvmfsexec +++ b/cvmfsexec @@ -9,7 +9,7 @@ #set -x #PS4='c$$+ ' -VERSION=4.40 +VERSION=4.41 usage() { diff --git a/singcvmfs b/singcvmfs index 824cccf..7ede655 100755 --- a/singcvmfs +++ b/singcvmfs @@ -3,7 +3,7 @@ # with the singularity --fusemount option. # Written by Dave Dykstra March 2020 -VERSION=4.40 +VERSION=4.41 ME="`basename $0`"