From 1eb52a792b57b5576c1925e5a0129a4f8acf8656 Mon Sep 17 00:00:00 2001 From: Vibhav Bobade Date: Fri, 29 Mar 2024 20:38:13 +0530 Subject: [PATCH] multicluster creation performance test (#98) * update build-workflow to build helm-values for the operator * update performance test runner * another refactor * breakup e2e tests * trigger helm install as a workflow run * allow workflows to trigger on all branches * remove if condition for the new workflow * make a workflow call instead * fix sed * build operator * run test after build is complete * use helm value to set concurrent workers * fix minikube start * update make perf command * spin up only 3 clusters * do not echo anything from the workflows * don't run unnecessary ciht * refactor * dev null that * namespace based deployment * use the correct label selector * get that dev null * get the correct outputs * correct naming and loop bound * run against multiple number of workers * remove echo * cat results.txt * do all of the fun stuff withyq * use the yq github action * cat results.txt * cat results.txts * run in the correct order * don't need ot delete anything * correct results name * check if json appendin g works * update entries * remove set options * remove unnecessary redirection * running with bash * restore bash features * JSONFILE * add perf graphs * update script * upload performance artifact * upload to imgur * typo * use correct action to upload * refactor * correct location for the script * ensure perf data file * brand new job * correct upload path * don't test with main for now * don't try 0 workers plis * correctly upload data * rename and update perf data file * checkout in the performance tests * unzip artifacts and use them * unzip properly * list artifacts before unzipping * temp test dat * wring env var name shiet * no waiting man * run with main * fix commenting * update dummy data * plot main and PR * reinstate actual test * reinstate operator build * checkout strategy * checkout hack scripts from PR as well * test hack * always * get hack from pr-temp * copy like a boss * reinstate after testing * cat perf data file * test for 30 workers as well * sort and plot * update generate graph with more readability * set ticks properly * deploy chart on worker updates * refactor, make space for cold start perf * add a bunch of stuff * run cold start test * use correct files and shiz * showcasing performance needs to be performative * test cold start * test test * battling typos * dereference correctly * test * comments and other stuff * perf * add more stuff to the chart * test with real and 8gb minikube * n_simul clusters as a env var * N_SIMUL_CLUSTERS usage * helm upgrade quiet * get n_simul_clsuters from github action env var * type * update n simul to 6 * update n_simulatenous_clsuters to 7 * time taken for readiness * better representation --- .github/workflows/build-operator.yaml | 51 +++ ...e2e-with-cluster.yaml => e2e-envtest.yaml} | 19 +- .github/workflows/e2e-perf.yaml | 291 ++++++++++++++++++ ...nstall-e2e.yaml => helm-install-test.yaml} | 72 ++--- .github/workflows/helm-release.yml | 4 +- .github/workflows/pull-request.yaml | 15 + .github/workflows/pull_request.yaml | 26 -- .github/workflows/{ci.yaml => release.yaml} | 4 +- Makefile | 27 +- .../controller-manager_deployment.yaml | 1 + chart/values.yaml | 16 +- hack/e2e/perf/01-multicluster.sh | 53 ++++ hack/e2e/perf/main.sh | 2 + .../e2e/perf/manifests/gen.yaml | 4 +- hack/e2e/perf/scripts/functions.sh | 12 + hack/e2e/perf/scripts/utils.sh | 17 + hack/e2e/perf/viz/README.md | 7 + .../e2e/perf/viz/generate_cold_start_graph.py | 39 +++ hack/e2e/perf/viz/generate_n_simul_graph.py | 54 ++++ .../patch-pod-tolerations.sh} | 0 .../patch-workload-tolerations.sh} | 0 21 files changed, 594 insertions(+), 120 deletions(-) create mode 100644 .github/workflows/build-operator.yaml rename .github/workflows/{e2e-with-cluster.yaml => e2e-envtest.yaml} (81%) create mode 100644 .github/workflows/e2e-perf.yaml rename .github/workflows/{helm-install-e2e.yaml => helm-install-test.yaml} (64%) create mode 100644 .github/workflows/pull-request.yaml delete mode 100644 .github/workflows/pull_request.yaml rename .github/workflows/{ci.yaml => release.yaml} (97%) create mode 100755 hack/e2e/perf/01-multicluster.sh create mode 100644 hack/e2e/perf/main.sh rename examples/storage-manifest.yml => hack/e2e/perf/manifests/gen.yaml (60%) create mode 100644 hack/e2e/perf/scripts/functions.sh create mode 100644 hack/e2e/perf/scripts/utils.sh create mode 100644 hack/e2e/perf/viz/README.md create mode 100644 hack/e2e/perf/viz/generate_cold_start_graph.py create mode 100644 hack/e2e/perf/viz/generate_n_simul_graph.py rename hack/{minikube-patch-pod-tolerations.sh => minikube/patch-pod-tolerations.sh} (100%) rename hack/{minikube-patch-workload-tolerations.sh => minikube/patch-workload-tolerations.sh} (100%) diff --git a/.github/workflows/build-operator.yaml b/.github/workflows/build-operator.yaml new file mode 100644 index 00000000..c9114976 --- /dev/null +++ b/.github/workflows/build-operator.yaml @@ -0,0 +1,51 @@ +name: Build - Image and Helm Values + +on: + workflow_call: + +jobs: + build-operator: + name: Build and Push `operator` Image + runs-on: ubuntu-latest + if: ${{ github.event_name == 'pull_request' && github.event.action != 'closed' }} + outputs: + tags: ${{ steps.meta.outputs.tags }} + uuid: ${{ env.UUID_OPERATOR }} + steps: + - name: Checkout git repo + uses: actions/checkout@v3 + - name: Generate UUID image name + id: uuid + run: echo "UUID_OPERATOR=$(uuidgen)" >> $GITHUB_ENV + - name: Docker metadata + id: meta + uses: docker/metadata-action@v4 + with: + # An anonymous, emphemeral registry built on ttl.sh + images: registry.uffizzi.com/${{ env.UUID_OPERATOR }} + tags: type=raw,value=48h + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Build and Push Image to Uffizzi Ephemeral Registry + uses: docker/build-push-action@v3 + with: + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + context: ./ + cache-from: type=gha + cache-to: type=gha,mode=max + - name: Create Helm Values File + run: | + cat < helm-values.yaml + image: + repository: registry.uffizzi.com/${{ env.UUID_OPERATOR }} + tag: 48h + concurrent: 5 + EOF + cat helm-values.yaml # For debugging, to check the contents of the file. + - name: Upload Helm Values as Artifact + uses: actions/upload-artifact@v3 + with: + name: helm-values + path: helm-values.yaml diff --git a/.github/workflows/e2e-with-cluster.yaml b/.github/workflows/e2e-envtest.yaml similarity index 81% rename from .github/workflows/e2e-with-cluster.yaml rename to .github/workflows/e2e-envtest.yaml index 1b867c6f..3a831cc9 100644 --- a/.github/workflows/e2e-with-cluster.yaml +++ b/.github/workflows/e2e-envtest.yaml @@ -1,9 +1,14 @@ -name: EnvTest with Cluster +name: Test - E2E - EnvTest with Cluster # e2e tests running against a k8s cluster on: pull_request: - branches: [ main ] - types: [opened,reopened,synchronize,closed] + branches: + - main + types: + - opened + - reopened + - synchronize + - closed permissions: contents: read @@ -19,12 +24,9 @@ jobs: - name: Checkout uses: actions/checkout@v3 - - name: Setup Flux CLI + - name: Setup Flux CLI # used in the make command uses: fluxcd/flux2/action@main - - name: Setup k3d - uses: ./.github/actions/k3d - - name: Run e2e tests against current cluster run: | make test-e2e-with-cluster-local @@ -43,9 +45,6 @@ jobs: - name: Setup Flux CLI uses: fluxcd/flux2/action@main - - name: Setup k3d - uses: ./.github/actions/k3d - - name: Run e2e tests against current tainted cluster run: | make test-e2e-with-tainted-cluster-local diff --git a/.github/workflows/e2e-perf.yaml b/.github/workflows/e2e-perf.yaml new file mode 100644 index 00000000..8c4abb96 --- /dev/null +++ b/.github/workflows/e2e-perf.yaml @@ -0,0 +1,291 @@ +name: Test - E2E - Performance # e2e tests running against a k8s cluster + +on: + pull_request: + branches: + - main + types: + - opened + - reopened + - synchronize + - closed + +env: + N_SIMUL_PERF_DATA_FILE: n-simul-perf-data.json + COLD_START_PERF_DATA_FILE: cold-start-perf-data.txt + E2E_UTILS: ./hack/e2e/perf/scripts/utils.sh + N_SIMUL_CLUSTERS: 7 + +permissions: + contents: write + pull-requests: write + id-token: write + +jobs: + build-operator: + uses: ./.github/workflows/build-operator.yaml + name: Build Operator Image + secrets: inherit + + perf-test-minikube: + needs: + - build-operator + name: Performance Test - Minikube + runs-on: ubuntu-latest + strategy: + matrix: + git_branch: + - main + - PR + steps: + - name: Checkout repo + uses: actions/checkout@v3 + with: + ref: ${{ matrix.git_branch == 'main' && 'main' || github.head_ref }} + + - name: Checkout hack from PR + uses: actions/checkout@v3 + with: + ref: ${{ github.head_ref }} + path: pr-temp + + - name: Copy hack outside of pr-temp + run: cp -r pr-temp/hack/* hack + + # Identify comment to be updated + - name: Find comment for Performance Overview + uses: peter-evans/find-comment@v2 + id: find-comment + with: + issue-number: ${{ github.event.pull_request.number }} + comment-author: "github-actions[bot]" + body-includes: Performance Overview + direction: last + + # Create/Update comment with action deployment status + - name: Create or Update Comment with Performance Overview + id: notification + uses: peter-evans/create-or-update-comment@v2 + with: + comment-id: ${{ steps.find-comment.outputs.comment-id }} + issue-number: ${{ github.event.pull_request.number }} + body: | + ## Operator Performance Overview + + :gear: Running performance tests on Minikube + + edit-mode: replace + + - name: install yq + run: sudo snap install yq + +## > comment to test with dummy data + - name: Download Helm Values Artifact + uses: actions/download-artifact@v3 + with: + name: helm-values + + - name: Start Minikube + run: minikube start --addons default-storageclass,storage-provisioner --driver=docker --cpus 4 --memory 8192 + + - name: Minikube Configuration and Resources + run: | + minikube config view + kubectl describe nodes + kubectl get storageclass + + - name: Install Uffizzi Cluster Operator + id: prev + run: | + helm dep update ./chart + helm upgrade --install --wait pr-${{ github.event.pull_request.number }} \ + ./chart -f helm-values.yaml +## < comment to test with dummy data + + - name: Create COLD_START_PERF_DATA_FILE + run: | + # Ensure the file exists and initialize it as an empty array if not + if [ ! -f "$COLD_START_PERF_DATA_FILE" ]; then + touch "$COLD_START_PERF_DATA_FILE" + fi + +## > comment to test with dummy data + - name: Time taken to create a cluster on cold start + run: | + bash hack/e2e/perf/01-multicluster.sh 1 > $COLD_START_PERF_DATA_FILE +## < comment to test with dummy data + +## > uncomment to test with dummy data +# - name: dummy data +# run: | +# if [[ "${{ github.ref_name }}" == "main" ]]; then +# factor=2 +# else +# factor=1 +# fi +# echo $factor > $COLD_START_PERF_DATA_FILE +## < comment to test with dummy data + + - name: Rename and update COLD_START_PERF_DATA_FILE + run: | + NEW_NAME="${COLD_START_PERF_DATA_FILE%.txt}-${{ matrix.git_branch }}.txt" + mv $COLD_START_PERF_DATA_FILE $NEW_NAME + echo "COLD_START_PERF_DATA_FILE=$NEW_NAME" >> $GITHUB_ENV + + - name: Create N_SIMUL_PERF_DATA_FILE + run: | + # Ensure the file exists and initialize it as an empty array if not + if [ ! -f "$N_SIMUL_PERF_DATA_FILE" ]; then + echo '[]' > "$N_SIMUL_PERF_DATA_FILE" + fi + +## > comment to test with dummy data + - name: Time taken to create UffizziClusters with different numbers of workers + run: | + n_simultaneous_clusters=$N_SIMUL_CLUSTERS + for n_workers in $(seq 1 5 31); do + # update concurrent workers + yq -i '.concurrent = "$n_workers"' helm-values.yaml + + # upgrade helm chart + helm upgrade --install --wait pr-${{ github.event.pull_request.number }} \ + ./chart -f helm-values.yaml > /dev/null + + # time taken to create n clusters simultaneously + time=$(bash hack/e2e/perf/01-multicluster.sh $n_simultaneous_clusters) + bash $E2E_UTILS update_json_with_workers_and_time $n_workers $time $N_SIMUL_PERF_DATA_FILE + done + + cat $N_SIMUL_PERF_DATA_FILE +## < comment to test with dummy data + +## > uncomment to test with dummy data +# - name: dummy data +# shell: bash +# run: | +# if [[ "${{ github.ref_name }}" == "main" ]]; then +# factor=2 +# else +# factor=1 +# fi +# echo '[ +# {"workers": 5, "time": '$((50 * factor))'}, +# {"workers": 10, "time": '$((100 * factor))'}, +# {"workers": 15, "time": '$((150 * factor))'}, +# {"workers": 20, "time": '$((200 * factor))'}, +# {"workers": 25, "time": '$((250 * factor))'}, +# {"workers": 30, "time": '$((300 * factor))'} +# ]' > $N_SIMUL_PERF_DATA_FILE +## < uncomment to test with dummy data + + - name: Rename and update N_SIMUL_PERF_DATA_FILE + run: | + NEW_NAME="${N_SIMUL_PERF_DATA_FILE%.json}-${{ matrix.git_branch }}.json" + mv $N_SIMUL_PERF_DATA_FILE $NEW_NAME + echo "N_SIMUL_PERF_DATA_FILE=$NEW_NAME" >> $GITHUB_ENV + + - name: Upload N_SIMUL_PERF_DATA_FILE artifact + uses: actions/upload-artifact@v2 + with: + name: n-simul-perf-data-${{ matrix.git_branch }} + path: ${{ env.N_SIMUL_PERF_DATA_FILE }} + + - name: Upload COLD_START_PERF_DATA_FILE artifact + uses: actions/upload-artifact@v2 + with: + name: cold-start-perf-data-${{ matrix.git_branch }} + path: ${{ env.COLD_START_PERF_DATA_FILE }} + + performance-overview: + name: Collate performance data and post overview + needs: + - perf-test-minikube + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + + # Identify comment to be updated + - name: Find comment for Performance Overview + uses: peter-evans/find-comment@v2 + id: find-comment + with: + issue-number: ${{ github.event.pull_request.number }} + comment-author: "github-actions[bot]" + body-includes: Performance Overview + direction: last + + - name: Download n-simul-perf-data-PR + uses: actions/download-artifact@v2 + with: + name: n-simul-perf-data-PR + + - name: Download n-simul-perf-data-main + uses: actions/download-artifact@v2 + with: + name: n-simul-perf-data-main + + - name: Download cold-start-perf-data-PR + uses: actions/download-artifact@v2 + with: + name: cold-start-perf-data-PR + + - name: Download cold-start-perf-data-main + uses: actions/download-artifact@v2 + with: + name: cold-start-perf-data-main + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.x' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install matplotlib + + - name: Create n simultaneous cluster creation graph + run: python hack/e2e/perf/viz/generate_n_simul_graph.py + + - name: Create cold start cluster creation graph + run: python hack/e2e/perf/viz/generate_cold_start_graph.py + + - name: Upload the simultaneous cluster creation test image to Imgur + id: upload_simul_image + uses: devicons/public-upload-to-imgur@v2.2.2 + with: + path: ./simul_graph.png + client_id: ${{ secrets.IMGUR_CLIENT_ID }} + + - name: Upload the cold start cluster creation test image to Imgur + id: upload_cold_start_image + uses: devicons/public-upload-to-imgur@v2.2.2 + with: + path: ./cold_start_graph.png + client_id: ${{ secrets.IMGUR_CLIENT_ID }} + + - name: Update Comment with Performance Overview + uses: peter-evans/create-or-update-comment@v2 + with: + comment-id: ${{ steps.find-comment.outputs.comment-id }} + issue-number: ${{ github.event.pull_request.number }} + body: | + ## Operator Performance Overview + + ### cold start - initial cluster creation + + This test would assess the duration and efficiency of initializing the first cluster, + which typically takes longer due to initial setup tasks such as pulling images and configuring the + environment. The test aims to capture the performance impact of these one-time operations to understand + the startup behavior and identify potential areas for optimization. + + ![cold start cluster creation](${{ fromJSON(steps.upload_cold_start_image.outputs.imgur_urls)[0] }}) + + ### n(=${{ env.N_SIMUL_CLUSTERS }}) simultaneous cluster creation + + This test is creating multiple clusters simultaneously in operator deployments of varying number of concurrent + workers to help us understand how the operator manages high load in restricted environments. + + ![n simultaneous cluster creation](${{ fromJSON(steps.upload_simul_image.outputs.imgur_urls)[0] }}) + edit-mode: replace diff --git a/.github/workflows/helm-install-e2e.yaml b/.github/workflows/helm-install-test.yaml similarity index 64% rename from .github/workflows/helm-install-e2e.yaml rename to .github/workflows/helm-install-test.yaml index 5e88f2d7..c6f9a865 100644 --- a/.github/workflows/helm-install-e2e.yaml +++ b/.github/workflows/helm-install-test.yaml @@ -1,9 +1,14 @@ -name: E2E - Helm Chart Deployment Test +name: Test - Deployment - Helm Chart on: pull_request: - branches: [ main ] - types: [opened,reopened,synchronize,closed] + branches: + - main + types: + - opened + - reopened + - synchronize + - closed permissions: contents: read @@ -12,47 +17,24 @@ permissions: jobs: build-operator: - name: Build and Push `operator` Image - runs-on: ubuntu-latest - if: ${{ github.event_name == 'pull_request' && github.event.action != 'closed' }} - outputs: - tags: ${{ steps.meta.outputs.tags }} - uuid: ${{ env.UUID_OPERATOR }} - steps: - - name: Checkout git repo - uses: actions/checkout@v3 - - name: Generate UUID image name - id: uuid - run: echo "UUID_OPERATOR=$(uuidgen)" >> $GITHUB_ENV - - name: Docker metadata - id: meta - uses: docker/metadata-action@v4 - with: - # An anonymous, emphemeral registry built on ttl.sh - images: registry.uffizzi.com/${{ env.UUID_OPERATOR }} - tags: type=raw,value=48h - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - name: Build and Push Image to Uffizzi Ephemeral Registry - uses: docker/build-push-action@v3 - with: - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - context: ./ - cache-from: type=gha - cache-to: type=gha,mode=max + uses: ./.github/workflows/build-operator.yaml + name: Build Operator Image + secrets: inherit uffizzi-cluster: - name: Deploy Helm chart to Uffizzi Virtual Cluster needs: - build-operator - if: ${{ github.event_name == 'pull_request' && github.event.action != 'closed' }} + name: Deploy Helm chart to Uffizzi Virtual Cluster runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v3 + - name: Download Helm Values Artifact + uses: actions/download-artifact@v3 + with: + name: helm-values + # Identify comment to be updated - name: Find comment for Ephemeral Environment uses: peter-evans/find-comment@v2 @@ -92,21 +74,6 @@ jobs: - name: Apply Helm Chart id: prev run: | - # install krew - - # set -x; cd "$(mktemp -d)" && - # OS="$(uname | tr '[:upper:]' '[:lower:]')" && - # ARCH="$(uname -m | sed -e 's/x86_64/amd64/' -e 's/\(arm\)\(64\)\?.*/\1\2/' -e 's/aarch64$/arm64/')" && - # KREW="krew-${OS}_${ARCH}" && - # curl -fsSLO "https://github.com/kubernetes-sigs/krew/releases/latest/download/${KREW}.tar.gz" && - # tar zxvf "${KREW}.tar.gz" && - # ./"${KREW}" install krew - - # cd - - - # install kuttl - # ./"${KREW}" install kuttl - if [[ ${RUNNER_DEBUG} == 1 ]]; then echo "`pwd`" echo "`ls`" @@ -115,10 +82,7 @@ jobs: export KUBECONFIG="`pwd`/kubeconfig" helm dep update ./chart helm upgrade --install --wait pr-${{ github.event.pull_request.number }} \ - ./chart --set image.repository=registry.uffizzi.com/${{ needs.build-operator.outputs.uuid }},image.tag=48h - - # kubectl apply -f examples/k3s-manifest-basic.yml - # kubectl apply -f examples/k8s-manifest-basic.yml + ./chart -f helm-values.yaml - name: Create or Update Comment with Deployment URL uses: peter-evans/create-or-update-comment@v2 diff --git a/.github/workflows/helm-release.yml b/.github/workflows/helm-release.yml index 787580f4..fc127400 100644 --- a/.github/workflows/helm-release.yml +++ b/.github/workflows/helm-release.yml @@ -1,9 +1,9 @@ -name: Release Helm Charts +name: Release - Helm Chart on: push: branches: - - main + - main jobs: release: diff --git a/.github/workflows/pull-request.yaml b/.github/workflows/pull-request.yaml new file mode 100644 index 00000000..001a8829 --- /dev/null +++ b/.github/workflows/pull-request.yaml @@ -0,0 +1,15 @@ +name: Test - Lint Helm Chart and Go Test + +on: + pull_request: + branches: + - main + +jobs: + lint-helm-chart: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Lint Helm Chart + run: | + make helm-lint diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml deleted file mode 100644 index 9332c0f0..00000000 --- a/.github/workflows/pull_request.yaml +++ /dev/null @@ -1,26 +0,0 @@ -name: Unit Tests and Coverage - -on: - pull_request: - branches: [ main ] - -jobs: - test_and_coverage: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Set up Go 1.19 - uses: actions/setup-go@v4 - with: - go-version: ^1.19 - - name: Lint Helm Chart - run: | - (cd ./chart && helm dep update .) - helm lint ./chart --with-subcharts - - - name: Test - run: | - go test -coverprofile=coverage.txt -covermode=atomic -v ./src/controllers/... - - - name: Upload coverage reports to Codecov - uses: codecov/codecov-action@v3 diff --git a/.github/workflows/ci.yaml b/.github/workflows/release.yaml similarity index 97% rename from .github/workflows/ci.yaml rename to .github/workflows/release.yaml index 8ffa4621..00e5ab7b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/release.yaml @@ -1,9 +1,9 @@ -name: CI +name: Release on: push: tags: - - 'v*.*.*' + - 'v*.*.*' env: RELEASE_VERSION: ${{ github.ref_name }} diff --git a/Makefile b/Makefile index 637bf956..f9c61eb4 100644 --- a/Makefile +++ b/Makefile @@ -2,13 +2,7 @@ VERSION ?= 1.6.4 # check if we are using MacOS or LINUX and use that to determine the sed command UNAME_S := $(shell uname -s) -SED := sed -ifeq ($(UNAME_S),Darwin) - SED = gsed -else - SED = sed -endif - +SED=$(shell which gsed || which sed) # CHANNELS define the bundle channels used in the bundle. # Add a new line here if you would like to change its default config. (E.g CHANNELS = "candidate,fast,stable") @@ -138,11 +132,11 @@ stop-test-minikube: ## Stop the minikube cluster for testing. .PHONY: start-test-minikube-tainted start-test-minikube-tainted: ## Start a minikube cluster with a tainted node for testing. minikube start --addons default-storageclass,storage-provisioner --driver=docker - sh ./hack/minikube-patch-pod-tolerations.sh + sh ./hack/minikube/patch-pod-tolerations.sh kubectl taint nodes minikube sandbox.gke.io/runtime=gvisor:NoSchedule || true kubectl label nodes minikube sandbox.gke.io/runtime=gvisor || true $(MAKE) install-fluxcd-controllers-with-toleration - sh ./hack/minikube-patch-workload-tolerations.sh + sh ./hack/minikube/patch-workload-tolerations.sh .PHONY : stop-test-k3d stop-test-k3d: ## Stop the k3d cluster for testing. @@ -170,6 +164,10 @@ test-e2e-without-cluster: manifests generate fmt vet envtest ## Run test. test-e2e-with-cluster: manifests generate fmt vet envtest ## Run test. KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" ENVTEST_REMOTE=true go test ./... -coverprofile=coverage.txt -v +.PHONY: test-e2e-perf-with-cluster +test-e2e-perf-with-cluster: + ./hack/e2e/perf/main.sh + .PHONY: test-e2e-with-cluster-local test-e2e-with-cluster-local: start-test-minikube test-e2e-with-cluster ## Run test. @@ -254,14 +252,19 @@ build-helm-chart: manifests generate fmt vet kustomize ## Deploy controller to t $(SED) -i'' -e 's/apiVersion: rbac.authorization.k8s.io\/v1/apiVersion: {{ include "common.capabilities.rbac.apiVersion" . }}/' chart/templates/manager-role_clusterrole.yaml $(SED) -i'' -e 's/labels:/labels: {{ include "common.labels.standard" . | nindent 4 }}/' chart/templates/manager-role_clusterrole.yaml $(SED) -i'' -e '/metadata:/a\ - labels: {{ include "common.labels.standard" . | nindent 4 }}\ - app.kubernetes.io/component: rbac\ - app.kubernetes.io/part-of: uffizzi' chart/templates/manager-role_clusterrole.yaml + \ labels: {{ include "common.labels.standard" . | nindent 4 }}\ + \n app.kubernetes.io/component: rbac\ + \n app.kubernetes.io/part-of: uffizzi' chart/templates/manager-role_clusterrole.yaml # update chart versions yq e -i '.version = "${VERSION}"' chart/Chart.yaml yq e -i '.appVersion = "v${VERSION}"' chart/Chart.yaml yq e -i '.image.tag = "v${VERSION}"' chart/values.yaml +.PHONY: helm-lint +helm-lint: ## Lint the helm chart. + (cd ./chart && helm dep update .) + helm lint ./chart --with-subcharts + ##@ Build Dependencies ## Location to install dependencies to diff --git a/chart/templates/controller-manager_deployment.yaml b/chart/templates/controller-manager_deployment.yaml index 0bb70d2c..7a2c76d5 100644 --- a/chart/templates/controller-manager_deployment.yaml +++ b/chart/templates/controller-manager_deployment.yaml @@ -62,6 +62,7 @@ spec: - --health-probe-bind-address=:8081 - --metrics-bind-address=127.0.0.1:8080 - --leader-elect + - "--concurrent={{ .Values.manager.concurrent }}" command: - /manager image: "{{ .Values.image.repository }}:{{ default .Chart.AppVersion .Values.image.tag }}" diff --git a/chart/values.yaml b/chart/values.yaml index 7e40ebb3..a124c378 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -5,6 +5,10 @@ image: repository: docker.io/uffizzi/uffizzi-cluster-operator tag: v1.6.4 + +manager: + concurrent: 5 + # `flux` dependency values flux: helmController: @@ -31,15 +35,3 @@ flux: enabled: false imageReflectorController: enabled: false -#cert-manager: # dependency -# enabled: true -# installCRDs: true -# -#ingress-nginx: -# controller: -# extraArgs: -# enable-ssl-passthrough: true -# To install nginx-ingress and cert-manager charts -# along with the rest apply the following tags (enabled by default) -#tags: -# - ingress-nginx diff --git a/hack/e2e/perf/01-multicluster.sh b/hack/e2e/perf/01-multicluster.sh new file mode 100755 index 00000000..5c280e7d --- /dev/null +++ b/hack/e2e/perf/01-multicluster.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail + +LOOP_BOUND="${1:-3}" + +for i in $(seq 1 $LOOP_BOUND); do + # Generate a unique namespace name + NAMESPACE="uffizzi-cluster-$i-$(date +%s)" + # Create the namespace + kubectl create namespace "$NAMESPACE" > /dev/null + # Label the namespace + kubectl label namespace "$NAMESPACE" app=uffizzi > /dev/null + # Deploy the UffizziCluster resource to the unique namespace + kubectl create -f hack/e2e/perf/manifests/gen.yaml --namespace="$NAMESPACE" > /dev/null +done + +namespaces=($(kubectl get ns --selector='app=uffizzi' -o jsonpath='{.items[*].metadata.name}')) + +# Function to check the APIReady condition of a UffizziCluster within a namespace +check_api_ready() { + local namespace=$1 + local api_ready=$(kubectl get uffizzicluster --namespace="$namespace" -o jsonpath='{.items[0].status.conditions[?(@.type=="APIReady")].status}') + echo "$api_ready" +} + +# Monitor each UffizziCluster for readiness +start_time=$(date +%s) +for ns in "${namespaces[@]}"; do +# echo "Monitoring UffizziCluster in namespace $ns" + while true; do + api_ready=$(check_api_ready "$ns") + if [ "$api_ready" == "True" ]; then +# echo "UffizziCluster in namespace $ns is ready." + break + else +# echo "Waiting for UffizziCluster in namespace $ns to become ready..." + sleep 5 + fi + done +done +end_time=$(date +%s) + +# Calculate the total time taken for all UffizziClusters to become ready +total_time=$((end_time - start_time)) +echo "$total_time" + +# Cleanup +for ns in "${namespaces[@]}"; do + kubectl delete namespace "$ns" > /dev/null +done diff --git a/hack/e2e/perf/main.sh b/hack/e2e/perf/main.sh new file mode 100644 index 00000000..e48dc1ad --- /dev/null +++ b/hack/e2e/perf/main.sh @@ -0,0 +1,2 @@ + +./hack/e2e/perf/01-multicluster.sh \ No newline at end of file diff --git a/examples/storage-manifest.yml b/hack/e2e/perf/manifests/gen.yaml similarity index 60% rename from examples/storage-manifest.yml rename to hack/e2e/perf/manifests/gen.yaml index 80497ff1..780b657e 100644 --- a/examples/storage-manifest.yml +++ b/hack/e2e/perf/manifests/gen.yaml @@ -1,7 +1,7 @@ kind: UffizziCluster apiVersion: uffizzi.com/v1alpha1 metadata: - name: storage-manifest + generateName: multicluster- spec: storage: - syncFromManifests: false + size: 2.5Gi \ No newline at end of file diff --git a/hack/e2e/perf/scripts/functions.sh b/hack/e2e/perf/scripts/functions.sh new file mode 100644 index 00000000..634f5902 --- /dev/null +++ b/hack/e2e/perf/scripts/functions.sh @@ -0,0 +1,12 @@ +# functions.sh + +function update_json_with_workers_and_time() { + # The first parameter is the number of workers + workers=$1 + # The second parameter is the time + time=$2 + # The third parameter is the path to the JSON file + JSON_FILE=$3 + + jq --argjson newEntry "{\"workers\": $workers, \"time\": \"$time\"}" '. += [$newEntry]' "$JSON_FILE" > "tmp.$$" && mv "tmp.$$" "$JSON_FILE" +} \ No newline at end of file diff --git a/hack/e2e/perf/scripts/utils.sh b/hack/e2e/perf/scripts/utils.sh new file mode 100644 index 00000000..b32d75bf --- /dev/null +++ b/hack/e2e/perf/scripts/utils.sh @@ -0,0 +1,17 @@ +# utils.sh + +# Source the script containing your function +source ./hack/e2e/perf/scripts/functions.sh + +# Store the function name in a variable and shift the arguments +func_name=$1 +shift + +# Dynamically call the function based on the first argument +if declare -f "$func_name" > /dev/null +then + # Call the function with the remaining arguments + "$func_name" "$@" +else + echo "Function '$func_name' not found" +fi diff --git a/hack/e2e/perf/viz/README.md b/hack/e2e/perf/viz/README.md new file mode 100644 index 00000000..42de169e --- /dev/null +++ b/hack/e2e/perf/viz/README.md @@ -0,0 +1,7 @@ +# Tools for visualizing performance data in e2e tests + +This directory contains tools for visualizing performance data in e2e tests. + +## `viz/generate_graph.py` + +creates a graph from the main brach and the PR branch data allowing user to compare the performance of the two branches. \ No newline at end of file diff --git a/hack/e2e/perf/viz/generate_cold_start_graph.py b/hack/e2e/perf/viz/generate_cold_start_graph.py new file mode 100644 index 00000000..a5c54258 --- /dev/null +++ b/hack/e2e/perf/viz/generate_cold_start_graph.py @@ -0,0 +1,39 @@ +import matplotlib.pyplot as plt + +data_file_prefix = 'cold-start-perf-data-' + +# Read cold start performance time from 'cold-start-main.txt' +with open(data_file_prefix+'main.txt', 'r') as file: + cold_start_main = int(file.read().strip()) + +# Read cold start performance time from 'cold-start-PR.txt' +with open(data_file_prefix+'PR.txt', 'r') as file: + cold_start_pr = int(file.read().strip()) + +# Branch names +branches = ['main', 'PR'] + +# Performance times +performance_times = [cold_start_main, cold_start_pr] + +# Create a bar chart +plt.figure(figsize=(8, 6)) +plt.bar(branches, performance_times, color=['blue', 'orange']) + +# Add a grid +plt.grid(True, linestyle='--', which='both', axis='y', alpha=0.7) + +# Set the y-axis ticks to align with the actual values +plt.yticks(range(0, max(performance_times) + 10, 5)) + +# Add title and labels +plt.title('cold start readiness comparison : PR vs Main Branch') +plt.xlabel('branch') +plt.ylabel('time taken for readiness (seconds)') +plt.grid(True) +plt.legend() + +# Show the plot + +plt.savefig('cold_start_graph.png') +# plt.show() \ No newline at end of file diff --git a/hack/e2e/perf/viz/generate_n_simul_graph.py b/hack/e2e/perf/viz/generate_n_simul_graph.py new file mode 100644 index 00000000..baafd15a --- /dev/null +++ b/hack/e2e/perf/viz/generate_n_simul_graph.py @@ -0,0 +1,54 @@ +import matplotlib.pyplot as plt +import json +import os + +data_file_prefix = 'n-simul-perf-data-' +n_simul_clusters = os.environ.get('N_SIMUL_CLUSTERS') + +# Load the data from 'perf-data-PR.json' +with open(data_file_prefix+'PR.json', 'r') as file: + data_pr = json.load(file) + +# Load the data from 'perf-data-main.json' +with open(data_file_prefix+'main.json', 'r') as file: + data_main = json.load(file) + +# Convert 'time' values to integers and extract 'workers' and 'time' into separate lists for PR data +workers_pr = [item['workers'] for item in data_pr] +time_pr = [int(item['time']) for item in data_pr] + +# Convert 'time' values to integers and extract 'workers' and 'time' into separate lists for main data +workers_main = [item['workers'] for item in data_main] +time_main = [int(item['time']) for item in data_main] + +# Sort the data by workers to ensure correct plotting order +sorted_pr = sorted(zip(workers_pr, time_pr), key=lambda x: x[0]) +sorted_main = sorted(zip(workers_main, time_main), key=lambda x: x[0]) + +# Unzip the sorted data +workers_pr_sorted, time_pr_sorted = zip(*sorted_pr) +workers_main_sorted, time_main_sorted = zip(*sorted_main) + +# Create a plot +plt.figure(figsize=(10, 6)) + +# Plot sorted data from 'perf-data-PR.json' +plt.plot(workers_pr_sorted, time_pr_sorted, marker='o', linestyle='-', label='PR Branch') + +# Plot sorted data from 'perf-data-main.json' +plt.plot(workers_main_sorted, time_main_sorted, marker='x', linestyle='--', label='Main Branch') + +# Set the tick marks to show each worker number +plt.xticks(workers_pr_sorted) +plt.yticks(sorted(set(time_pr_sorted + time_main_sorted))) # Combine and sort unique time values + +# Add title, labels, grid, and legend +plt.title('multicluster readiness comparison: PR vs Main Branch') +plt.xlabel('number of workers (n_simultaneous_clusters='+n_simul_clusters+')') +plt.ylabel('time taken for readiness (seconds)') +plt.grid(True) +plt.legend() + +# Display the plot +plt.savefig('simul_graph.png') +# plt.show() diff --git a/hack/minikube-patch-pod-tolerations.sh b/hack/minikube/patch-pod-tolerations.sh similarity index 100% rename from hack/minikube-patch-pod-tolerations.sh rename to hack/minikube/patch-pod-tolerations.sh diff --git a/hack/minikube-patch-workload-tolerations.sh b/hack/minikube/patch-workload-tolerations.sh similarity index 100% rename from hack/minikube-patch-workload-tolerations.sh rename to hack/minikube/patch-workload-tolerations.sh