From e89bc16215a482b6242c13b9580f5165c478a7ce Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Fri, 7 Jun 2024 09:20:10 -0700 Subject: [PATCH 1/7] Build and Run Arm64 K3s Docker Tests Signed-off-by: Derek Nola --- .github/workflows/build-k3s.yaml | 21 ++++++++++++--- .github/workflows/e2e.yaml | 41 ++++++++++++++++++++++++++++++ .github/workflows/integration.yaml | 2 +- 3 files changed, 60 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-k3s.yaml b/.github/workflows/build-k3s.yaml index 47ba18e247ee..643342b6b3e4 100644 --- a/.github/workflows/build-k3s.yaml +++ b/.github/workflows/build-k3s.yaml @@ -3,6 +3,10 @@ name: Build K3s on: workflow_call: inputs: + arch: + type: string + description: 'Architecture to build' + default: 'ubuntu-latest' upload-repo: type: boolean required: false @@ -18,11 +22,16 @@ permissions: jobs: build: name: Build - runs-on: ubuntu-latest + runs-on: ${{ inputs.arch }} # defaults to ubuntu-latest, for arm64 use oracle-aarch64-4cpu-16gb timeout-minutes: 20 steps: - name: Checkout K3s uses: actions/checkout@v4 + - name: Setup Arm64 Runner + if: contains(inputs.arch, 'aarch64') + run: | + sudo apt-get update + sudo apt-get install -y make - name: Build K3s binary run: | DOCKER_BUILDKIT=1 SKIP_IMAGE=1 SKIP_AIRGAP=1 SKIP_VALIDATE=1 GOCOVER=1 make @@ -44,9 +53,15 @@ jobs: - name: "Save K3s image" if: inputs.upload-image == true run: docker image save rancher/k3s -o ./dist/artifacts/k3s-image.tar - - name: "Upload K3s binary" - if: inputs.upload-repo == false + - name: "Upload K3s Artifacts" + if: inputs.upload-repo == false && inputs.arch == 'ubuntu-latest' uses: actions/upload-artifact@v4 with: name: k3s + path: dist/artifacts/k3s* + - name: "Upload K3s Arm64 Artifacts" + if: contains(inputs.arch, 'aarch64') + uses: actions/upload-artifact@v4 + with: + name: k3s-arm64 path: dist/artifacts/k3s* \ No newline at end of file diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 684051d199c1..70b88e7d37f4 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -30,6 +30,12 @@ jobs: uses: ./.github/workflows/build-k3s.yaml with: upload-image: true + build-arm64: + if : github.repository == 'k3s-io/k3s' + uses: ./.github/workflows/build-k3s.yaml + with: + arch: oracle-aarch64-4cpu-16gb + upload-image: true e2e: name: "E2E Tests" needs: build @@ -119,3 +125,38 @@ jobs: . ./tests/docker/test-helpers . ./tests/docker/test-run-${{ matrix.dtest }} echo "Did test-run-${{ matrix.dtest }} pass $?" + docker-arm64: + needs: build-arm64 + name: Docker Tests Arm64 + if: github.repository == 'k3s-io/k3s' + runs-on: oracle-aarch64-4cpu-16gb + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + dtest: [basics, bootstraptoken, cacerts, compat, lazypull, upgrade] + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 1 + - name: Install Go + uses: actions/setup-go@v5 + with: + go-version-file: 'go.mod' + cache: false + - name: "Download k3s image" + uses: actions/download-artifact@v4 + with: + name: k3s-arm64 + path: ./dist/artifacts + - name: Load k3s image + run: docker image load -i ./dist/artifacts/k3s-image.tar + - name: Run ${{ matrix.dtest }} Test + run: | + mv ./dist/artifacts/k3s-arm64 ./dist/artifacts/k3s + chmod +x ./dist/artifacts/k3s + . ./scripts/version.sh + . ./tests/docker/test-helpers + . ./tests/docker/test-run-${{ matrix.dtest }} + echo "Did test-run-${{ matrix.dtest }} pass $?" \ No newline at end of file diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index 2cde5fc0a9c1..66a8dc1f22d5 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -58,7 +58,7 @@ jobs: mkdir -p $GOCOVERDIR sudo -E env "PATH=$PATH" go test -v -timeout=45m ./tests/integration/${{ matrix.itest }}/... -run Integration - name: On Failure, Launch Debug Session - uses: lhotari/action-upterm@v1 + uses: dereknola/action-upterm@v1.1 if: ${{ failure() }} with: ## If no one connects after 5 minutes, shut down server. From 72baaff67325d76229d984fb43b8aa5bbd4eef8c Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Fri, 6 Sep 2024 14:34:33 -0700 Subject: [PATCH 2/7] Add arm64 support tools Signed-off-by: Derek Nola --- .github/workflows/e2e.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 70b88e7d37f4..b21ee37c5564 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -145,6 +145,13 @@ jobs: with: go-version-file: 'go.mod' cache: false + - name: Install Support Tools + run: | + sudo apt-get update && sudo apt-get install -y wget gawk + sudo wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_arm64 -O /usr/bin/yq &&\ + sudo chmod +x /usr/bin/yq + curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/arm64/kubectl" + sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl - name: "Download k3s image" uses: actions/download-artifact@v4 with: From ec8511892224c4eacd347378492abd28fd8cb3e9 Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Tue, 17 Sep 2024 10:29:17 -0700 Subject: [PATCH 3/7] try debug Signed-off-by: Derek Nola --- .github/workflows/e2e.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index b21ee37c5564..db94a1da3ee3 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -166,4 +166,10 @@ jobs: . ./scripts/version.sh . ./tests/docker/test-helpers . ./tests/docker/test-run-${{ matrix.dtest }} - echo "Did test-run-${{ matrix.dtest }} pass $?" \ No newline at end of file + echo "Did test-run-${{ matrix.dtest }} pass $?" + - name: On Failure, Launch Debug Session + uses: dereknola/action-upterm@v1.1 + if: ${{ failure() }} + with: + ## If no one connects after 5 minutes, shut down server. + wait-timeout-minutes: 5 \ No newline at end of file From c283222bb617f1debb474e4dcad54673ce2a8d65 Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Tue, 17 Sep 2024 10:43:30 -0700 Subject: [PATCH 4/7] Add ssh support Signed-off-by: Derek Nola --- .github/workflows/e2e.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index db94a1da3ee3..b57e92f50568 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -147,7 +147,7 @@ jobs: cache: false - name: Install Support Tools run: | - sudo apt-get update && sudo apt-get install -y wget gawk + sudo apt-get update && sudo apt-get install -y wget gawk openssh-client openssh-server sudo wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_arm64 -O /usr/bin/yq &&\ sudo chmod +x /usr/bin/yq curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/arm64/kubectl" From 6d697c73ac717200c89098f5b95572cc7a5c6278 Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Tue, 17 Sep 2024 10:44:25 -0700 Subject: [PATCH 5/7] revert debug int Signed-off-by: Derek Nola --- .github/workflows/integration.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index 66a8dc1f22d5..2cde5fc0a9c1 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -58,7 +58,7 @@ jobs: mkdir -p $GOCOVERDIR sudo -E env "PATH=$PATH" go test -v -timeout=45m ./tests/integration/${{ matrix.itest }}/... -run Integration - name: On Failure, Launch Debug Session - uses: dereknola/action-upterm@v1.1 + uses: lhotari/action-upterm@v1 if: ${{ failure() }} with: ## If no one connects after 5 minutes, shut down server. From feda6035610b2f9ba3369ada6e88cf7377d60746 Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Tue, 17 Sep 2024 11:14:03 -0700 Subject: [PATCH 6/7] Use native snapshotter on arm64 docker tests Signed-off-by: Derek Nola --- tests/docker/test-run-basics | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/docker/test-run-basics b/tests/docker/test-run-basics index a221b5b7c30e..bcfd6c33a19a 100755 --- a/tests/docker/test-run-basics +++ b/tests/docker/test-run-basics @@ -10,6 +10,8 @@ all_services=( export NUM_SERVERS=1 export NUM_AGENTS=1 export WAIT_SERVICES="${all_services[@]}" +export SERVER_ARGS="--snapshotter=native" +export AGENT_ARGS="--snapshotter=native" start-test() { use-local-storage-volume From f1aa014a9a6eb6b8e8f5d883498225b78114d5eb Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Tue, 17 Sep 2024 13:15:41 -0700 Subject: [PATCH 7/7] Add iptables Signed-off-by: Derek Nola --- .github/workflows/e2e.yaml | 4 ++-- tests/docker/test-run-basics | 7 +++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index b57e92f50568..2092c0d25650 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -134,7 +134,7 @@ jobs: strategy: fail-fast: false matrix: - dtest: [basics, bootstraptoken, cacerts, compat, lazypull, upgrade] + dtest: [basics, bootstraptoken, cacerts, compat, upgrade] steps: - name: Checkout uses: actions/checkout@v4 @@ -147,7 +147,7 @@ jobs: cache: false - name: Install Support Tools run: | - sudo apt-get update && sudo apt-get install -y wget gawk openssh-client openssh-server + sudo apt-get update && sudo apt-get install -y wget gawk openssh-client openssh-server net-tools iptables sudo wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_arm64 -O /usr/bin/yq &&\ sudo chmod +x /usr/bin/yq curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/arm64/kubectl" diff --git a/tests/docker/test-run-basics b/tests/docker/test-run-basics index bcfd6c33a19a..c3fde55242f8 100755 --- a/tests/docker/test-run-basics +++ b/tests/docker/test-run-basics @@ -10,8 +10,11 @@ all_services=( export NUM_SERVERS=1 export NUM_AGENTS=1 export WAIT_SERVICES="${all_services[@]}" -export SERVER_ARGS="--snapshotter=native" -export AGENT_ARGS="--snapshotter=native" + +if [ "$(uname -i)" = "aarch64" ]; then + export SERVER_ARGS="--snapshotter=native" + export AGENT_ARGS="--snapshotter=native" +fi start-test() { use-local-storage-volume