Nightly Builds #414
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| name: Nightly Builds | |
| on: | |
| # This workflow needs to pass if there is a PR changing it. | |
| pull_request: | |
| paths: | |
| - ".github/workflows/nightly.yaml" | |
| workflow_dispatch: # Or manually - for testing | |
| jobs: | |
| build-and-test: | |
| runs-on: self-hosted | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| with: | |
| path: s3gw | |
| submodules: true | |
| fetch-depth: 0 | |
| - name: Checkout Ceph HEAD | |
| working-directory: s3gw/ceph | |
| run: | | |
| git fetch | |
| git checkout s3gw | |
| git submodule update --init --recursive | |
| - name: Checkout s3tests | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: ceph/s3-tests | |
| path: s3tests | |
| - name: Setup Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: 3.11 | |
| - name: Install Dependencies | |
| run: | | |
| YQ_GH_URL=https://github.com/mikefarah/yq/releases/download | |
| YQ_VERSION=v4.31.1 | |
| YQ_CHECKSUM=1aef844cbecbbf81036449ea5e7dfcdf19d2d374fab6303fdb8f849d04275d76 | |
| sudo apt-get update | |
| sudo apt-get install -y \ | |
| bc \ | |
| wget \ | |
| s3cmd | |
| # Unfortunately, since yq is only available through snap on Ubuntu and | |
| # that doesn't work in docker containers (at least not out of the | |
| # box), this abomination is the way to go to install yq. | |
| echo "${YQ_CHECKSUM} yq" >> checksum | |
| wget -O yq "${YQ_GH_URL}/${YQ_VERSION}/yq_linux_amd64" | |
| sha256sum -c checksum \ | |
| && sudo mv yq /usr/bin/yq \ | |
| && sudo chmod +x /usr/bin/yq | |
| # known to work version of boto3 | |
| python3 -m pip install boto3==1.24.96 | |
| python3 -m pip install -r s3tests/requirements.txt | |
| - name: Quay Login | |
| uses: docker/login-action@v2 | |
| with: | |
| registry: quay.io | |
| username: ${{ secrets.QUAY_USERNAME }} | |
| password: ${{ secrets.QUAY_PASSWORD }} | |
| - name: Generate Tag based on Date | |
| id: date | |
| run: | | |
| DATE="$(date +%Y-%m-%d)" | |
| echo "tag=nightly-${DATE}" >> $GITHUB_OUTPUT | |
| - name: Build Unittests | |
| run: | | |
| docker build \ | |
| --build-arg CMAKE_BUILD_TYPE=Release \ | |
| --build-arg NPROC=16 \ | |
| --file s3gw/Dockerfile \ | |
| --tag s3gw-unittests \ | |
| --target s3gw-unittests \ | |
| s3gw | |
| - name: Build s3gw Container Image | |
| run: | | |
| docker build \ | |
| --build-arg CMAKE_BUILD_TYPE=Release \ | |
| --build-arg NPROC=16 \ | |
| --build-arg S3GW_VERSION=${{ steps.date.outputs.tag }} \ | |
| --build-arg QUAY_EXPIRATION=1w \ | |
| --file s3gw/Dockerfile \ | |
| --tag quay.io/s3gw/s3gw:nightly-latest \ | |
| --tag quay.io/s3gw/s3gw:${{ steps.date.outputs.tag }} \ | |
| s3gw | |
| docker push quay.io/s3gw/s3gw:nightly-latest | |
| docker push quay.io/s3gw/s3gw:${{ steps.date.outputs.tag }} | |
| - name: Run Unittests | |
| run: | | |
| docker run --rm s3gw-unittests | |
| - name: Prepare environment | |
| env: | |
| GITHUB_KEY: ${{ secrets.STATUS_DEPLOY_SECRET }} | |
| run: | | |
| mkdir -p ${HOME}/.ssh || true | |
| rm -f ${HOME}/.ssh/github-s3gw-status || true | |
| chmod 700 ${HOME}/.ssh || exit 1 | |
| cat <<EOF>${HOME}/.ssh/github-s3gw-status | |
| ${GITHUB_KEY} | |
| EOF | |
| chmod 400 ${HOME}/.ssh/github-s3gw-status | |
| cat <<EOF>${HOME}/.ssh/config | |
| Host github-status | |
| Hostname github.com | |
| IdentityFile ${HOME}/.ssh/github-s3gw-status | |
| User git | |
| EOF | |
| chmod 600 ${HOME}/.ssh/config | |
| cat <<EOF>${HOME}/.ssh/known_hosts | |
| github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk= | |
| EOF | |
| chmod 600 ${HOME}/.ssh/known_hosts | |
| - name: Run S3tests | |
| run: | | |
| set +e # don't exit on error | |
| set -x | |
| # needed for GNU parallel because the version in GH runners is bugged | |
| mkdir -p ${HOME}/.parallel | |
| export DEBUG=1 | |
| export S3TEST_PARALLEL=ON | |
| export CEPH_DIR="${GITHUB_WORKSPACE}/s3gw/ceph" | |
| export OUTPUT_DIR="${GITHUB_WORKSPACE}/s3test-results" | |
| export S3GW_CONTAINER="quay.io/s3gw/s3gw:nightly-latest" | |
| export FORCE_CONTAINER=ON | |
| export FORCE_DOCKER=ON | |
| export \ | |
| FIXTURES="${CEPH_DIR}/qa/rgw/store/sfs/tests/fixtures" | |
| export S3TEST_REPO="${GITHUB_WORKSPACE}/s3tests" | |
| export S3TEST_CONF="${FIXTURES}/s3tests.conf" | |
| export S3TEST_LIST="${FIXTURES}/s3-tests.txt" | |
| # There are some s3tests that don't finish at all. Only run | |
| # known-to-pass tests for now. TODO: fix infinite looping tests | |
| # sed -r -i 's/^# //' "${S3TEST_LIST}" | |
| pushd s3tests | |
| ${GITHUB_WORKSPACE}/s3gw/tools/tests/s3tests-runner.sh | |
| popd | |
| git clone github-status:s3gw-tech/s3gw-status.git s3gw-status.git || exit 1 | |
| pushd s3gw-status.git || exit 1 | |
| git config user.name "github nightly bot" | |
| git config user.email "[email protected]" | |
| cp \ | |
| "${OUTPUT_DIR}/report.json" \ | |
| results/s3tests/${{ steps.date.outputs.tag }}.json | |
| git add results/s3tests/${{ steps.date.outputs.tag }}.json || exit 1 | |
| git commit -m "Nightly Results $(date +%Y-%m-%d)" || exit 1 | |
| git push || exit 1 | |
| # We expect some tests to fail, we just want to see _which_ tests fail | |
| # exit 0 | |
| - name: Cleanup SSH environment | |
| run: | | |
| rm -f ${HOME}/.ssh/github-s3gw-status |