Skip to content

Fix ncp-check-version for docker #815

Fix ncp-check-version for docker

Fix ncp-check-version for docker #815

Workflow file for this run

# https://www.docker.com/blog/docker-v2-github-action-is-now-ga/
# https://www.docker.com/blog/multi-arch-build-and-images-the-simple-way/
# https://docs.github.com/en/actions/guides/publishing-docker-images
name: 'Docker Integration Tests and Release'
on:
workflow_call:
inputs:
git_ref:
required: true
type: string
arch:
required: false
type: string
default: 'x86'
push:
branches:
- docker-stable
- docker-devel
pull_request:
branches:
- docker-stable
- docker-devel
- release/docker/*
jobs:
build:
if: ${{ github.repository == 'nextcloud/nextcloudpi' }}
runs-on: ubuntu-20.04
env:
VERSION: "${{ inputs.git_ref || github.ref }}"
ARCH: "${{ inputs.arch || 'x86' }}"
LOG_DCKR: "\\033[1;34mDCKR::\\033[0m"
LOG_NCP: "\\033[1;36m~NCP::\\033[0m"
LOG_CICD: "\\033[1;35mCICD::\\033[0m"
LOG_TEST: "\\033[1;33mTEST::\\033[0m"
LOG_DIAG: "\\033[1;31mDIAG::\\033[0m"
steps:
- name: Setup Docker Buildx
uses: docker/setup-buildx-action@v2
# - name: setup binfmt
# run: |
# sudo apt-get install binfmt-support qemu-user-static
#
- name: Set up QEMU
uses: docker/setup-qemu-action@v2
- name: Checkout code
uses: actions/checkout@v3
with:
ref: "${{ env.VERSION }}"
- name: Login to docker
run: |
echo "${{ secrets.DOCKER_PASSWORD_INTERNAL }}" | docker login -u "${{ secrets.DOCKER_LOGIN_INTERNAL }}" --password-stdin
- name: Build images
id: build-container
run: |
./build/build-docker.sh "${ARCH?}"
docker tag "ownyourbits/nextcloudpi-${ARCH?}:latest" "thecalcaholic/ncp-internal-${ARCH?}:${{ github.run_id }}"
testing_image="ownyourbits/nextcloudpi-${ARCH?}:latest"
[[ "${ARCH?}" == "x86" ]] || testing_image="ownyourbits/ncp-qemu-fix-${ARCH?}:latest"
docker tag "${testing_image}" "thecalcaholic/ncp-internal-${ARCH?}:${{ github.run_id }}-testing"
docker push "thecalcaholic/ncp-internal-${ARCH?}:${{ github.run_id }}"
docker push "thecalcaholic/ncp-internal-${ARCH?}:${{ github.run_id }}-testing"
update-test:
# TODO: Fix update-test for armhf
# The issues are originating in a seemingly bug in apache2 that only occurs in the GH action and only
# if a docker volume or directory is mounted to /data.
# In this configuration, apache will serve the default placeholder page instead of the sites configured in ncp.
if: ${{ inputs.arch != 'armhf' }}
needs:
- build
runs-on: ubuntu-20.04
env:
VERSION: "${{ inputs.git_ref || github.ref }}"
ARCH: "${{ inputs.arch || 'x86' }}"
LOG_DCKR: "\\033[1;34mDCKR::\\033[0m"
LOG_NCP: "\\033[1;36m~NCP::\\033[0m"
LOG_CICD: "\\033[1;35mCICD::\\033[0m"
LOG_TEST: "\\033[1;33mTEST::\\033[0m"
LOG_DIAG: "\\033[1;31mDIAG::\\033[0m"
steps:
- name: Setup Docker Buildx
uses: docker/setup-buildx-action@v2
- name: Checkout code
uses: actions/checkout@v3
with:
ref: "${{ env.VERSION }}"
fetch-depth: 0
- name: Get previous release
id: get_previous_version
shell: bash
run: |
set -ex
if [[ "${{ env.VERSION }}" == "refs/heads/devel" ]]
then
version="latest"
else
ref="${{ github.base_ref }}"
[[ -n "$ref" ]] || ref="${{ github.ref }}"
git checkout "$ref"
if [[ "${{ github.ref_type }}" != "tag" ]] || ! git describe --tags > /dev/null
then
git fetch -fu --tags origin "${ref}:${ref}"
fi
version="$(git describe --tags || true)"
[[ "$version" =~ ^(docker-)?v.*-.*-.* ]] || {
git checkout HEAD~1
version="$(git describe --tags || echo latest)"
}
version="${version#docker-}"
version="${version%-*-*}"
fi
curl "https://hub.docker.com/v2/repositories/ownyourbits/nextcloudpi-${ARCH}/tags/${version}" | grep 'errinfo' > /dev/null && {
echo "Image ownyourbits/nextcloudpi-${ARCH}:${version} does not exist - falling back to :latest"
version="latest"
}
version="${version#docker-}"
echo "Previous version is '$version'"
echo "PREVIOUS_VERSION=$version" >> "$GITHUB_ENV"
echo "previous_version=${version}" >> $GITHUB_OUTPUT
- name: Set up QEMU
uses: docker/setup-qemu-action@v2
- name: Enable qemu SUID
run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes --credential yes
- name: Start ncp container
run: |
docker volume create ncdata
platform=linux/${ARCH?}
[[ "$ARCH" == 'x86' ]] && platform=linux/amd64
tmpdir=$(mktemp -d)
pushd "$tmpdir"
cat <<EOF > ./Dockerfile
FROM --platform=$platform ownyourbits/nextcloudpi:${{ steps.get_previous_version.outputs.previous_version }}
RUN echo 'Mutex posixsem' >> /etc/apache2/mods-available/ssl.conf
EOF
docker buildx build --platform "$platform" -t ncp-testing -f Dockerfile --load .
popd
rm -r "$tmpdir"
docker run --platform "$platform" -v ncdata:/data -d --rm -p 8443:443 -p 4443:4443 --name nextcloudpi ncp-testing localhost
- name: Checkout code
uses: actions/checkout@v3
with:
ref: "${{ env.VERSION }}"
- name: Setup Firefox
uses: browser-actions/setup-firefox@latest
- name: Setup GeckoDriver
uses: ChlodAlejandro/setup-geckodriver@latest
with:
token: ${{ secrets.GITHUB_TOKEN }}
- name: Setup Selenium
run: pip install selenium
- name: Wait for container startup
run: |
set -e
docker logs nextcloudpi |& awk "{ print \"${LOG_DCKR} \" \$0 }"
echo -e "${LOG_DCKR} =========="
docker logs -f nextcloudpi |& awk "{ print \"${LOG_DCKR} \" \$0 }" &
attempt=0
stage=0
success=false
for attempt in {1..30}
do
echo -e "${LOG_CICD} Waiting for container startup (attempt $attempt/30)..."
redis_pw="$(docker exec nextcloudpi bash -c ". /usr/local/etc/library.sh; get_nc_config_value 'redis\"][\"password'")"
redis_socket="$(docker exec nextcloudpi bash -c ". /usr/local/etc/library.sh; get_nc_config_value 'redis\"][\"host'")"
if [[ "$stage" == "0" ]] && docker logs nextcloudpi 2> /dev/null | grep '^Init done'
then
stage=1
elif [[ "$(docker exec nextcloudpi ncc maintenance:mode)" =~ .*disabled.* ]] \
&& docker exec nextcloudpi ncc status \
&& docker exec nextcloudpi redis-cli -s "$redis_socket" -a "$redis_pw" set redisready yes \
&& docker exec nextcloudpi redis-cli -s "$redis_socket" -a "$redis_pw" get redisready \
&& curl -k https://localhost:8443/activate/ > /dev/null
then
success=true
echo -e "${LOG_CICD} Startup successful"
break
fi
attempt=$((attempt + 1))
sleep 5
done
[[ "$success" == "true" ]] || {
echo -e "${LOG_CICD} Timeout reached."
exit 1
}
- name: Activate container
working-directory: ./tests
run: |
docker logs nextcloudpi |& awk "{ print \"${LOG_DCKR} \" \$0 }"
echo -e "${LOG_DCKR} =========="
docker logs -f nextcloudpi |& awk "{ print \"${LOG_DCKR} \" \$0 }" &
docker exec nextcloudpi bash -c 'tail -f /var/log/ncp.log' |& awk "{ print \"${LOG_NCP} \" \$0 }" &
cmd=(python activation_tests.py --no-gui localhost 8443 4443)
[[ "${ARCH?}" == "x86" ]] || { sleep 60; cmd+=(--timeout 300); }
success=false
for attempt in {1..10}
do
echo -e "${LOG_CICD} == Activation Tests (attempt $attempt/10) =="
"${cmd[@]}" |& awk "{ print \"${LOG_TEST} \" \$0 }"
[[ ${PIPESTATUS[0]} -eq 0 ]] || {
tail -n 20 geckodriver.log |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true
echo -e "${LOG_CICD} Activation test (attempt $attempt/10) failed!"
docker exec nextcloudpi bash /usr/local/bin/ncp-diag |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true
cat /proc/sys/fs/binfmt_misc/qemu-aarch64 |& awk "{ print \"${LOG_DIAG} (qemu-aarch64) \" \$0 }" || true
cat /proc/sys/fs/binfmt_misc/qemu-arm |& awk "{ print \"${LOG_DIAG} (qemu-arm) \" \$0 }" || true
sleep 12
continue
}
success=true
break
done
[[ "$success" == "true" ]] || {
echo -e "${LOG_CICD} Activation test failed in all attempts!"
echo "Creating error report..."
docker exec nextcloudpi bash /usr/local/bin/ncp-report > error-report.txt
echo "done."
exit 1
}
echo -e "${LOG_CICD} Activation test successful"
- name: Start new container
run: |
docker stop nextcloudpi
docker run -d -v ncdata:/data --rm -p 8443:443 -p 4443:4443 -e NOBACKUP=true --name nextcloudpi thecalcaholic/ncp-internal-${ARCH?}:${{ github.run_id }}-testing localhost
- name: Wait for container startup
run: |
set -e
docker logs nextcloudpi |& awk "{ print \"${LOG_DCKR} \" \$0 }"
echo -e "${LOG_DCKR} =========="
docker logs -f nextcloudpi |& awk "{ print \"${LOG_DCKR} \" \$0 }" &
attempt=0
stage=0
for attempt in {1..200}
do
echo -e "${LOG_CICD} Waiting for container startup (attempt $attempt/30)..."
redis_pw="$(docker exec nextcloudpi bash -c ". /usr/local/etc/library.sh; get_nc_config_value 'redis\"][\"password'")"
redis_socket="$(docker exec nextcloudpi bash -c ". /usr/local/etc/library.sh; get_nc_config_value 'redis\"][\"host'")"
if [[ "$stage" == "0" ]] && docker logs nextcloudpi 2> /dev/null | grep '^Init done' > /dev/null
then
stage=1
elif [[ "$(docker exec nextcloudpi ncc maintenance:mode)" =~ .*disabled.* ]] \
&& docker exec nextcloudpi ncc status \
&& docker exec nextcloudpi redis-cli -s "$redis_socket" -a "$redis_pw" set redisready yes \
&& docker exec nextcloudpi redis-cli -s "$redis_socket" -a "$redis_pw" get redisready \
&& curl -k https://localhost:8443/activate/
then
echo -e "${LOG_CICD} Startup successful"
break
elif [[ "$attempt" -ge 30 ]]
then
echo -e "${LOG_CICD} Timeout reached."
exit 1
fi
attempt=$((attempt + 1))
sleep 5
done
echo "Wait one more minute..."
sleep 60
- name: Update Nextcloud
run: |
set -e
docker logs nextcloudpi |& awk "{ print \"${LOG_DCKR} \" \$0 }"
echo -e "${LOG_DCKR} =========="
docker logs -f nextcloudpi |& awk "{ print \"${LOG_DCKR} \" \$0 }" &
current_nc_version="$(docker exec nextcloudpi ncc status | grep "version:" | awk '{ print $3 }')"
latest_nc_version="$(docker exec nextcloudpi cat /usr/local/etc/ncp.cfg | jq -r '.nextcloud_version')"
if [[ "$current_nc_version" == "$latest_nc_version".* ]]
then
echo -e "${LOG_CICD} Nextcloud is up to date - skipping NC update test."
else
docker exec nextcloudpi bash -c "DBG=x ncp-update-nc ${latest_nc_version?}" |& awk "{ print \"${LOG_DCKR} \" \$0 }"
[[ ${PIPESTATUS[0]} -eq 0 ]] || {
echo -e "${LOG_CICD} Nextcloud Update failed"
echo -e "${LOG_CICD} Creating error report..."
docker exec nextcloudpi bash /usr/local/bin/ncp-report > error-report.txt
docker exec nextcloudpi bash /usr/local/bin/ncp-diag |& awk "{ print \"${LOG_DIAG} \" \$0 }"
exit 1
}
fi
- name: Wait for update completion
run: |
set -e
docker logs nextcloudpi |& awk "{ print \"${LOG_DCKR} \" \$0 }"
echo -e "${LOG_DCKR} =========="
docker logs -f nextcloudpi |& awk "{ print \"${LOG_DCKR} \" \$0 }" &
attempt=0
for attempt in {1..30}
do
echo -e "${LOG_CICD} Waiting for update completion (attempt $attempt/30)..."
redis_pw="$(docker exec nextcloudpi bash -c ". /usr/local/etc/library.sh; get_nc_config_value 'redis\"][\"password'")"
redis_socket="$(docker exec nextcloudpi bash -c ". /usr/local/etc/library.sh; get_nc_config_value 'redis\"][\"host'")"
if [[ "$(docker exec nextcloudpi ncc maintenance:mode)" =~ .*disabled.* ]] \
&& docker exec nextcloudpi ncc status \
&& docker exec nextcloudpi redis-cli -s "$redis_socket" -a "$redis_pw" set redisready yes \
&& docker exec nextcloudpi redis-cli -s "$redis_socket" -a "$redis_pw" get redisready
then
echo -e "${LOG_CICD} Startup successful"
break
elif [[ "$attempt" -ge 30 ]]
then
echo -e "${LOG_CICD} Timeout reached."
exit 1
fi
attempt=$((attempt + 1))
sleep 5
done
- name: Integration Tests
working-directory: ./tests
run: |
docker logs nextcloudpi |& awk "{ print \"${LOG_DCKR} \" \$0 }"
echo -e "${LOG_DCKR} =========="
docker logs -f nextcloudpi |& awk "{ print \"${LOG_DCKR} \" \$0 }" &
docker exec nextcloudpi bash -c 'tail -f /var/log/ncp.log' |& awk "{ print \"${LOG_NCP} \" \$0 }" &
[[ "${ARCH?}" == "x86" ]] || sleep 240
sleep 20
success=false
for attempt in {1..5}
do
echo -e "${LOG_CICD} == System Tests (attempt $attempt/5) =="
python system_tests.py --no-ping --non-interactive |& awk "{ print \"${LOG_TEST} \" \$0 }"
[[ ${PIPESTATUS[0]} -eq 0 ]] || {
echo -e "${LOG_CICD} System test (attempt $attempt) failed!"
sleep 12
continue
}
success=true
break
done
[[ "$success" == "true" ]] || {
echo -e "${LOG_CICD} System test failed in all attempts!"
echo -e "${LOG_CICD} Creating error report..."
docker exec nextcloudpi bash /usr/local/bin/ncp-report > error-report.txt
exit 1
}
echo -e "${LOG_CICD} System test successful"
success=false
for attempt in {1..3}
do
echo -e "${LOG_CICD} == Nextcloud Tests (attempt $attempt/5) =="
python nextcloud_tests.py --no-gui localhost 8443 4443 |& awk "{ print \"${LOG_TEST} \" \$0 }"
[[ ${PIPESTATUS[0]} -eq 0 ]] || {
tail -n 20 geckodriver.log |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true
echo -e "${LOG_CICD} Nextcloud test (attempt $attempt/3) failed!"
sleep 12
continue
}
success=true
break
done
[[ "$success" == "true" ]] || {
echo -e "${LOG_CICD} Nextcloud test failed in all attempts!"
echo -e "${LOG_CICD} Creating error report..."
docker exec nextcloudpi bash /usr/local/bin/ncp-report > error-report.txt
echo -e "${LOG_CICD} done."
exit 1
}
echo -e "${LOG_CICD} Nextcloud test successful"
docker exec nextcloudpi bash /usr/local/bin/ncp-diag |& awk "{ print \"${LOG_DIAG} \" \$0 }"
- name: "Upload error report"
if: ${{ failure() }}
uses: actions/upload-artifact@v3
with:
name: ${{ github.run_id }}-docker-${{ env.ARCH }}-update-test-error-report
path: tests/error-report.txt
test:
needs:
- build
runs-on: ubuntu-20.04
env:
VERSION: "${{ inputs.git_ref || github.ref }}"
ARCH: "${{ inputs.arch || 'x86' }}"
LOG_DCKR: "\\033[1;34mDCKR::\\033[0m"
LOG_NCP: "\\033[1;36m~NCP::\\033[0m"
LOG_CICD: "\\033[1;35mCICD::\\033[0m"
LOG_TEST: "\\033[1;33mTEST::\\033[0m"
LOG_DIAG: "\\033[1;31mDIAG::\\033[0m"
steps:
- name: Set up QEMU
uses: docker/setup-qemu-action@v2
- name: Login to docker
run: |
echo "${{ secrets.DOCKER_PASSWORD_INTERNAL }}" | docker login -u "${{ secrets.DOCKER_LOGIN_INTERNAL }}" --password-stdin
- name: Start ncp container
run: |
docker run -d --rm -p 8443:443 -p 4443:4443 --name nextcloudpi thecalcaholic/ncp-internal-${ARCH?}:${{ github.run_id }}-testing localhost
- name: Checkout code
uses: actions/checkout@v3
with:
ref: "${{ env.VERSION }}"
- name: Setup Firefox
uses: browser-actions/setup-firefox@latest
- name: Setup GeckoDriver
uses: ChlodAlejandro/setup-geckodriver@latest
with:
token: ${{ secrets.GITHUB_TOKEN }}
- name: Setup Selenium
run: pip install selenium
- name: Wait for container startup
run: |
set -e
docker logs nextcloudpi |& awk "{ print \"${LOG_DCKR} \" \$0 }"
echo -e "${LOG_DCKR} =========="
docker logs -f nextcloudpi |& awk "{ print \"${LOG_DCKR} \" \$0 }" &
attempt=0
stage=0
for attempt in {1..200}
do
echo -e "${LOG_CICD} Waiting for container startup (attempt $attempt/30)..."
redis_pw="$(docker exec nextcloudpi bash -c ". /usr/local/etc/library.sh; get_nc_config_value 'redis\"][\"password'")"
redis_socket="$(docker exec nextcloudpi bash -c ". /usr/local/etc/library.sh; get_nc_config_value 'redis\"][\"host'")"
if [[ "$stage" == "0" ]] && docker logs nextcloudpi 2> /dev/null | grep '^Init done' > /dev/null
then
stage=1
elif [[ "$(docker exec nextcloudpi ncc maintenance:mode)" =~ .*disabled.* ]] \
&& docker exec nextcloudpi ncc status \
&& docker exec nextcloudpi redis-cli -s "$redis_socket" -a "$redis_pw" set redisready yes \
&& docker exec nextcloudpi redis-cli -s "$redis_socket" -a "$redis_pw" get redisready \
&& curl -k https://localhost:8443/activate/
then
echo -e "${LOG_CICD} Startup successful"
break
elif [[ "$attempt" -ge 30 ]]
then
echo -e "${LOG_CICD} Timeout reached."
exit 1
fi
attempt=$((attempt + 1))
sleep 5
done
- name: Integration Tests
working-directory: ./tests
run: |
docker logs nextcloudpi |& awk "{ print \"${LOG_DCKR} \" \$0 }"
echo -e "${LOG_DCKR} =========="
docker logs -f nextcloudpi |& awk "{ print \"${LOG_DCKR} \" \$0 }" &
docker exec nextcloudpi bash -c 'tail -f /var/log/ncp.log' |& awk "{ print \"${LOG_NCP} \" \$0 }" &
cmd=(python activation_tests.py --no-gui localhost 8443 4443)
[[ "${ARCH?}" == "x86" ]] || { sleep 60; cmd+=(--timeout 300); }
success=false
for attempt in {1..10}
do
echo -e "${LOG_CICD} == Activation Tests (attempt $attempt/10) =="
"${cmd[@]}" |& awk "{ print \"${LOG_TEST} \" \$0 }"
[[ ${PIPESTATUS[0]} -eq 0 ]] || {
tail -n 20 geckodriver.log >&2 |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true
echo -e "${LOG_CICD} Activation test (attempt $attempt/10) failed!"
docker exec nextcloudpi bash /usr/local/bin/ncp-diag |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true
sleep 12
continue
}
success=true
break
done
[[ "$success" == "true" ]] || {
echo -e "${LOG_CICD} Activation test failed in all attempts!"
exit 1
}
echo -e "${LOG_CICD} Activation test successful"
success=false
for attempt in {1..5}
do
echo -e "${LOG_CICD} == System Tests (attempt $attempt/5) =="
python system_tests.py --no-ping --non-interactive |& awk "{ print \"${LOG_TEST} \" \$0 }"
[[ ${PIPESTATUS[0]} -eq 0 ]] || {
echo -e "${LOG_CICD} System test (attempt $attempt) failed!"
sleep 12
continue
}
success=true
break
done
[[ "$success" == "true" ]] || {
echo -e "${LOG_CICD} System test failed in all attempts!"
exit 1
}
echo -e "${LOG_CICD} System test successful"
success=false
for attempt in {1..5}
do
echo -e "${LOG_CICD} == Nextcloud Tests (attempt $attempt/5) =="
python nextcloud_tests.py --no-gui localhost 8443 4443 |& awk "{ print \"${LOG_TEST} \" \$0 }"
[[ ${PIPESTATUS[0]} -eq 0 ]] || {
tail -n 20 geckodriver.log >&2 |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true
echo -e "${LOG_CICD} Nextcloud test (attempt $attempt/3) failed!"
sleep 12
continue
}
success=true
break
done
[[ "$success" == "true" ]] || {
echo -e "${LOG_CICD} Nextcloud test failed in all attempts!"
echo -e "${LOG_CICD} Creating error report..."
docker exec nextcloudpi bash /usr/local/bin/ncp-report > error-report.txt
echo -e "${LOG_CICD} done."
exit 1
}
echo -e "${LOG_CICD} Nextcloud test successful"
docker exec nextcloudpi bash /usr/local/bin/ncp-diag |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true
- name: "Upload error report"
if: ${{ failure() }}
uses: actions/upload-artifact@v3
with:
name: ${{ github.run_id }}-docker-${{ env.ARCH }}-install-test-error-report
path: tests/error-report.txt