diff --git a/.github/workflows/e2e_docker.yaml b/.github/workflows/e2e_docker.yaml new file mode 100644 index 000000000..2e7b8a51f --- /dev/null +++ b/.github/workflows/e2e_docker.yaml @@ -0,0 +1,134 @@ +# (C) Copyright Confidential Containers Contributors 2024. +# SPDX-License-Identifier: Apache-2.0 +# +# Run docker e2e tests. +name: (Callable) docker e2e tests + +on: + workflow_call: + inputs: + podvm_image: + required: true + type: string + caa_image: + description: The cloud-api-adaptor OCI image (including tag) to test + type: string + install_directory_artifact: + description: The archive name of the install directory + default: '' + required: false + type: string + git_ref: + default: 'main' + description: Git ref to checkout the cloud-api-adaptor repository. Defaults to main. + required: false + type: string + container_runtime: + default: 'containerd' + description: Name of the container runtime. Either containerd or crio. + required: false + type: string + +env: + CLOUD_PROVIDER: docker + CLUSTER_NAME: peer-pods + DEBIAN_FRONTEND: noninteractive + +defaults: + run: + working-directory: src/cloud-api-adaptor + +jobs: + test-docker: + runs-on: ubuntu-22.04 + # TODO: remove this when the job gets stable + continue-on-error: true + steps: + - name: Checkout Code + uses: actions/checkout@v3 + with: + fetch-depth: 0 + ref: ${{ inputs.git_ref }} + + - name: Rebase the code + if: github.event_name == 'pull_request_target' + working-directory: ./ + run: | + ./hack/ci-helper.sh rebase-atop-of-the-latest-target-branch + + - name: Login to quay Container Registry + if: ${{ startsWith(inputs.podvm_image, 'quay.io') }} + uses: docker/login-action@v2 + with: + registry: quay.io + username: ${{ secrets.QUAY_USERNAME }} + password: ${{ secrets.QUAY_PASSWORD }} + + - name: Login to the ghcr Container registry + if: ${{ startsWith(inputs.podvm_image, 'ghcr.io') }} + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Read properties from versions.yaml + run: | + sudo snap install yq + go_version="$(yq '.tools.golang' versions.yaml)" + [ -n "$go_version" ] + echo "GO_VERSION=${go_version}" >> "$GITHUB_ENV" + + - name: Setup Golang version ${{ env.GO_VERSION }} + uses: actions/setup-go@v4 + with: + go-version: ${{ env.GO_VERSION }} + + - name: Install kustomize + run: | + command -v kustomize >/dev/null || \ + curl -s "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" | \ + sudo bash -s /usr/local/bin + + - name: Update kustomization configuration + run: | + cd "install/overlays/docker" + kustomize edit set image "cloud-api-adaptor=${{ inputs.caa_image }}" + # Print for debugging + echo "::group::docker kustomization" + cat kustomization.yaml + echo "::endgroup::" + + - name: Config docker + run: | + cat <<- EOF > docker.properties + DOCKER_PODVM_IMAGE="${{ inputs.podvm_image }}" + DOCKER_HOST="unix:///var/run/docker.sock" + DOCKER_NETWORK_NAME="kind" + EOF + # For debugging + cat docker.properties + + - name: run tests + id: runTests + run: | + export CLOUD_PROVIDER=docker + export CONTAINER_RUNTIME="${{ inputs.container_runtime }}" + export DEPLOY_KBS=false + export TEST_PROVISION=yes + export TEST_TEARDOWN=no + export TEST_PROVISION_FILE="$PWD/docker.properties" + export TEST_PODVM_IMAGE="${{ inputs.podvm_image }}" + export TEST_E2E_TIMEOUT="50m" + + make test-e2e + + - name: Debug tests failure + if: failure() && steps.runTests.outcome == 'failure' + working-directory: ./ + run: | + export KUBECONFIG="${HOME}/kube_${CLUSTER_NAME}" + kind get kubeconfig -n "$CLUSTER_NAME" > "$KUBECONFIG" + ./hack/ci-e2e-debug-fail.sh + # Avoid running with `set -e` as command fails should be allowed + shell: bash {0} diff --git a/.github/workflows/e2e_libvirt.yaml b/.github/workflows/e2e_libvirt.yaml index e94c08db7..f1c617361 100644 --- a/.github/workflows/e2e_libvirt.yaml +++ b/.github/workflows/e2e_libvirt.yaml @@ -186,61 +186,10 @@ jobs: - name: Debug tests failure if: failure() && steps.runTests.outcome == 'failure' + working-directory: ./ run: | export KUBECONFIG="${HOME}/.kcli/clusters/peer-pods/auth/kubeconfig" - - echo "::group::KBS installation" - kubectl get pods -n coco-tenant - kubectl describe pods -n coco-tenant - echo "::endgroup::" - - echo "::group::CoCo and Peer Pods installation" - kubectl get pods -n confidential-containers-system - kubectl describe pods -n confidential-containers-system - echo "::endgroup::" - - echo "::group::cloud-api-adaptor logs" - kubectl logs -l app=cloud-api-adaptor -n confidential-containers-system - echo "::endgroup::" - - echo "::group::kbs logs" - kubectl logs deployment/kbs -n coco-tenant - echo "::endgroup::" - - for ns in $(kubectl get ns -o name 2>/dev/null | sed 's#namespace/##' | grep "^coco-pp-"); do - for pod in $(kubectl get pods -o name -n "$ns" 2>/dev/null); do - echo "::group::Describe $pod (namespace/$ns)" - kubectl describe "$pod" -n "$ns" - echo "::endgroup::" - done - done - - for worker in $(kubectl get node -o name -l node.kubernetes.io/worker 2>/dev/null); do - echo "::group::journalctl -t kata ($worker)" - kubectl debug --image quay.io/prometheus/busybox -q -i \ - "$worker" -- chroot /host journalctl -x -t kata --no-pager - echo "::endgroup::" - done - - echo "::group::Libvirt domains" - sudo virsh list - echo "::endgroup::" - - for podvm in $(sudo virsh list --name | grep "podvm-"); do - echo "::group::podvm $podvm" - sudo virsh dominfo "$podvm" - sudo virsh domifaddr "$podvm" - echo "::endgroup::" - done - - echo "::group::podvm base volume" - sudo virsh vol-info --pool default podvm-base.qcow2 - ls -lh /var/lib/libvirt/images/podvm-base.qcow2 - echo "::endgroup::" - - echo "::group::Check podvm base volume integrity" - sudo qemu-img check /var/lib/libvirt/images/podvm-base.qcow2 - echo "::endgroup::" + ./hack/ci-e2e-debug-fail.sh # Avoid running with `set -e` as command fails should be allowed shell: bash {0} diff --git a/.github/workflows/e2e_run_all.yaml b/.github/workflows/e2e_run_all.yaml index 90b9de475..fd06a0a73 100644 --- a/.github/workflows/e2e_run_all.yaml +++ b/.github/workflows/e2e_run_all.yaml @@ -116,12 +116,13 @@ jobs: # IMPORTANT: If you are enabling e2e tests for a given provider, # then please update the PROVIDERS list (space-separated names, e.g., # "aws libvirt"). + # TODO - when the packer approach is removed this can go as well prep_install: runs-on: ubuntu-24.04 outputs: matrix: ${{ steps.matrix.outputs.matrix }} env: - PROVIDERS: "libvirt" + PROVIDERS: "docker libvirt" steps: - name: Checkout Code uses: actions/checkout@v4 @@ -283,3 +284,31 @@ jobs: git_ref: ${{ inputs.git_ref }} oras: true secrets: inherit + + # Run docker e2e tests if pull request labeled 'test_e2e_docker' + docker: + name: docker + if: | + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + contains(github.event.pull_request.labels.*.name, 'test_e2e_docker') + needs: [podvm_mkosi_amd64, caa_image_amd64] + strategy: + fail-fast: false + matrix: + container_runtime: + - crio + os: + - fedora + provider: + - docker + arch: + - amd64 + uses: ./.github/workflows/e2e_docker.yaml + with: + caa_image: ${{ inputs.registry }}/cloud-api-adaptor:${{ inputs.caa_image_tag }}-amd64-dev + container_runtime: ${{ matrix.container_runtime }} + podvm_image: ${{ needs.podvm_mkosi_amd64.outputs.docker_oci_image }} + install_directory_artifact: install_directory + git_ref: ${{ inputs.git_ref }} + secrets: inherit diff --git a/hack/ci-e2e-debug-fail.sh b/hack/ci-e2e-debug-fail.sh new file mode 100755 index 000000000..b07e40c68 --- /dev/null +++ b/hack/ci-e2e-debug-fail.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +# +# (C) Copyright Confidential Containers Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# Primarily used on Github workflows to debug failed pipelines. +# +# NOTE: if you want a debugger for MY_PROVIDER provider then you just need +# to create the debug_MY_PROVIDER function. Nothing else is needed. +# +# Not setting errexit, nounset, and pipefail because it is fine and should +# continue if any command fail. + +CLOUD_PROVIDER=${CLOUD_PROVIDER:-} + +# Get common debug information. +# +debug_common() { + echo "::group::KBS installation" + kubectl get pods -n coco-tenant + kubectl describe pods -n coco-tenant + echo "::endgroup::" + + echo "::group::CoCo and Peer Pods installation" + kubectl get pods -n confidential-containers-system + kubectl describe pods -n confidential-containers-system + echo "::endgroup::" + + echo "::group::cloud-api-adaptor logs" + kubectl logs -l app=cloud-api-adaptor --tail=-1 -n confidential-containers-system + echo "::endgroup::" + + echo "::group::kbs logs" + kubectl logs deployment/kbs -n coco-tenant + echo "::endgroup::" + + for ns in $(kubectl get ns -o name 2>/dev/null | sed 's#namespace/##' | grep "^coco-pp-"); do + for pod in $(kubectl get pods -o name -n "$ns" 2>/dev/null); do + echo "::group::Describe $pod (namespace/$ns)" + kubectl describe "$pod" -n "$ns" + echo "::endgroup::" + done + done + + for worker in $(kubectl get node -o name -l node.kubernetes.io/worker 2>/dev/null); do + echo "::group::journalctl -t kata ($worker)" + kubectl debug --image quay.io/prometheus/busybox -q -i \ + "$worker" -- chroot /host journalctl -x -t kata --no-pager + echo "::endgroup::" + done +} + +# Debugger for Libvirt. +# +debug_libvirt() { + echo "::group::Libvirt domains" + sudo virsh list + echo "::endgroup::" + + for podvm in $(sudo virsh list --name | grep "podvm-"); do + echo "::group::podvm $podvm" + sudo virsh dominfo "$podvm" + sudo virsh domifaddr "$podvm" + echo "::endgroup::" + done + + echo "::group::podvm base volume" + sudo virsh vol-info --pool default podvm-base.qcow2 + ls -lh /var/lib/libvirt/images/podvm-base.qcow2 + echo "::endgroup::" + + echo "::group::Check podvm base volume integrity" + sudo qemu-img check /var/lib/libvirt/images/podvm-base.qcow2 + echo "::endgroup::" +} + +main() { + debug_common + + if [ -n "$CLOUD_PROVIDER" ]; then + if ! type -a "debug_${CLOUD_PROVIDER}" &>/dev/null; then + echo "INFO: Cannot get further information as debugger for ${CLOUD_PROVIDER} is not implemented" + else + "debug_${CLOUD_PROVIDER}" + fi + fi +} + +main "$@"