chore: keep infra on error in test-set #3113
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI pipeline for Claudie | |
| on: | |
| # Manual trigger | |
| workflow_dispatch: | |
| # Triggers the workflow on push or pull request events but only for the master branch | |
| pull_request: | |
| branches: [master] | |
| env: | |
| ENV_FILE: .env | |
| SERVICES: manager builder terraformer ansibler kube-eleven kuber claudie-operator autoscaler-adapter testing-framework | |
| jobs: | |
| #-------------------------------------------------------------------------------------------------- | |
| merge-branch: | |
| if: github.event.pull_request.draft == false | |
| runs-on: self-hosted | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.head_ref }} | |
| repository: ${{ github.event.pull_request.head.repo.full_name }} | |
| - name: Merge with master branch | |
| uses: everlytic/[email protected] | |
| with: | |
| github_token: ${{ github.token }} | |
| source_ref: master | |
| target_branch: ${{ github.head_ref }} | |
| #-------------------------------------------------------------------------------------------------- | |
| check-changes: | |
| if: github.event.pull_request.draft == false | |
| runs-on: self-hosted | |
| needs: merge-branch | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.head_ref }} | |
| repository: ${{ github.event.pull_request.head.repo.full_name }} | |
| - name: Check the repository for changes | |
| id: change | |
| run: | | |
| arr=() | |
| SERVICES=( ${{ env.SERVICES }} ) | |
| git fetch origin master | |
| changes=($(git diff --name-only origin/master)) | |
| for i in "${changes[@]}" | |
| do | |
| if [[ "${i}" =~ .*\.md.* ]]; then | |
| echo "Skipping document "$i"" | |
| continue | |
| elif [[ "${i}" =~ .*(internal|proto)/.* ]] || [[ "${i}" =~ go\.(mod|sum) ]]; then | |
| arr=(${SERVICES[@]}) | |
| echo "All services need to be built" | |
| break | |
| elif [[ "${i}" =~ .*manifests/.* ]]; then | |
| echo "E2E test will run" | |
| echo "RUN_TESTS=true" >> $GITHUB_OUTPUT | |
| else | |
| for SERVICE in "${SERVICES[@]}"; do | |
| if [[ "${i}" =~ .*services/${SERVICE}.* ]]; then | |
| arr+=($SERVICE) | |
| echo "Detected change in $SERVICE" | |
| fi | |
| done | |
| fi | |
| done | |
| echo "ARRAY_OF_CHANGES=$(echo ${arr[@]})" >> $GITHUB_OUTPUT | |
| outputs: | |
| ARRAY_OF_CHANGES: ${{ steps.change.outputs.ARRAY_OF_CHANGES }} | |
| RUN_TESTS: ${{ steps.change.outputs.RUN_TESTS }} | |
| #-------------------------------------------------------------------------------------------------- | |
| gotest: | |
| runs-on: ubuntu-latest | |
| needs: [merge-branch, check-changes] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.head_ref }} | |
| repository: ${{ github.event.pull_request.head.repo.full_name }} | |
| - name: Install golang | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version: "1.23.1" | |
| - name: Run Go tests | |
| run: go test -short ./... | |
| #-------------------------------------------------------------------------------------------------- | |
| build-and-push: | |
| runs-on: self-hosted | |
| needs: [merge-branch, check-changes] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| if: (needs.check-changes.outputs.ARRAY_OF_CHANGES != '' && github.event.pull_request.draft == false) | |
| with: | |
| ref: ${{ github.head_ref }} | |
| repository: ${{ github.event.pull_request.head.repo.full_name }} | |
| - name: Set short sha output | |
| if: (needs.check-changes.outputs.ARRAY_OF_CHANGES != '' && github.event.pull_request.draft == false) | |
| run: echo "SHORT_GITHUB_SHA=`echo ${GITHUB_SHA} | cut -c1-7`" >> $GITHUB_ENV | |
| - name: Set up docker | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| # Update autoscaler-adapter manifest in this steps as new kuber would need to contain manifest with the correct image tag | |
| - name: Edit autoscaler-adapter image tag in the manifest | |
| if: ${{ needs.check-changes.outputs.ARRAY_OF_CHANGES != '' && github.event.pull_request.draft == false }} | |
| id: autoscaler-tag | |
| run: | | |
| NEW_SERVICES=( ${{ needs.check-changes.outputs.ARRAY_OF_CHANGES }} ) | |
| # Check if autoscaler-adapter is going to be built, if so, insert latest tag and add kuber to array if needed. | |
| if [[ "${NEW_SERVICES[*]}" =~ "autoscaler-adapter" ]]; then | |
| TAG=${SHORT_GITHUB_SHA}-${GITHUB_RUN_NUMBER} | |
| echo "Setting new tag for autoscaler adapter to $TAG" | |
| sed -i "s/image: ghcr.io\/berops\/claudie\/autoscaler-adapter/&:$TAG/" services/kuber/templates/cluster-autoscaler.goyaml | |
| if [[ ! "${NEW_SERVICES[*]}" =~ "kuber" ]]; then | |
| NEW_SERVICES+=(kuber) | |
| fi | |
| else | |
| # Check if kuber is going to be built, if so, insert latest tag and add to build. | |
| if [[ "${NEW_SERVICES[*]}" =~ "kuber" ]]; then | |
| TAG=${SHORT_GITHUB_SHA}-${GITHUB_RUN_NUMBER} | |
| echo "Setting new tag for autoscaler adapter to $TAG" | |
| sed -i "s/image: ghcr.io\/berops\/claudie\/autoscaler-adapter/&:$TAG/" services/kuber/templates/cluster-autoscaler.goyaml | |
| NEW_SERVICES+=(autoscaler-adapter) | |
| fi | |
| fi | |
| echo "ARRAY_OF_CHANGES=$(echo ${NEW_SERVICES[@]})" >> $GITHUB_OUTPUT | |
| # Build the new images that were changed by a recent commit - tag image latest as well for testing | |
| - name: Build and push new images | |
| if: (needs.check-changes.outputs.ARRAY_OF_CHANGES != '' && github.event.pull_request.draft == false) | |
| run: | | |
| ARR=(${{ steps.autoscaler-tag.outputs.ARRAY_OF_CHANGES }}) | |
| for SERVICE in "${ARR[@]}" | |
| do | |
| echo "-----Building $SERVICE-----" | |
| IMGTAG="ghcr.io/berops/claudie/$SERVICE:${SHORT_GITHUB_SHA}-${GITHUB_RUN_NUMBER}" | |
| DOCKER_BUILDKIT=1 docker build --tag $IMGTAG -f ./services/$SERVICE/Dockerfile . | |
| docker push $IMGTAG | |
| done | |
| outputs: | |
| ARRAY_OF_CHANGES: ${{ steps.autoscaler-tag.outputs.ARRAY_OF_CHANGES }} | |
| #-------------------------------------------------------------------------------------------------- | |
| edit-kustomization: | |
| runs-on: self-hosted | |
| needs: [merge-branch, check-changes, build-and-push, gotest] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| if: ${{ needs.build-and-push.outputs.ARRAY_OF_CHANGES != '' && github.event.pull_request.draft == false }} | |
| with: | |
| ref: ${{ github.head_ref }} | |
| repository: ${{ github.event.pull_request.head.repo.full_name }} | |
| - name: Set short sha output | |
| if: ${{ needs.build-and-push.outputs.ARRAY_OF_CHANGES != '' && github.event.pull_request.draft == false }} | |
| run: echo "SHORT_GITHUB_SHA=`echo ${GITHUB_SHA} | cut -c1-7`" >> $GITHUB_ENV | |
| - name: install kustomize | |
| if: ${{ needs.build-and-push.outputs.ARRAY_OF_CHANGES != '' && github.event.pull_request.draft == false }} | |
| uses: imranismail/setup-kustomize@v2 | |
| with: | |
| kustomize-version: 4.5.6 | |
| # Set the new claudie image tags in kustomization.yaml | |
| - name: Edit claudie kustomization.yaml | |
| if: ${{ needs.build-and-push.outputs.ARRAY_OF_CHANGES != '' && github.event.pull_request.draft == false }} | |
| working-directory: ./manifests/claudie | |
| run: | | |
| NEW_SERVICES=( ${{ needs.build-and-push.outputs.ARRAY_OF_CHANGES }} ) | |
| for SERVICE in "${NEW_SERVICES[@]}" | |
| do | |
| if [ "${SERVICE}" != "testing-framework" ]; then | |
| echo "Setting a new tag for a $SERVICE" | |
| kustomize edit set image ghcr.io/berops/claudie/$SERVICE:${SHORT_GITHUB_SHA}-${GITHUB_RUN_NUMBER} | |
| fi | |
| done | |
| cat kustomization.yaml | |
| # Set the new testing-framework image tags in kustomization.yaml | |
| - name: Edit testing-framework kustomization.yaml | |
| if: ${{ needs.build-and-push.outputs.ARRAY_OF_CHANGES != '' && github.event.pull_request.draft == false }} | |
| working-directory: ./manifests/testing-framework | |
| run: | | |
| NEW_SERVICES=( ${{ needs.build-and-push.outputs.ARRAY_OF_CHANGES }} ) | |
| for SERVICE in "${NEW_SERVICES[@]}" | |
| do | |
| if [ "$SERVICE" == "testing-framework" ]; then | |
| echo "Setting a new tag for a $SERVICE" | |
| kustomize edit set image ghcr.io/berops/claudie/$SERVICE:${SHORT_GITHUB_SHA}-${GITHUB_RUN_NUMBER} | |
| fi | |
| done | |
| cat kustomization.yaml | |
| # Auto commit the changes | |
| - name: Commit new kustomization.yaml to feature branch | |
| if: ${{ needs.build-and-push.outputs.ARRAY_OF_CHANGES != '' && github.event.pull_request.draft == false }} | |
| working-directory: ./manifests | |
| run: | | |
| BRANCH_NAME=${{ github.head_ref }} | |
| git config --global user.name 'CI/CD pipeline' | |
| git config --global user.email 'CI/[email protected]' | |
| git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }} | |
| git add claudie/kustomization.yaml | |
| git add testing-framework/kustomization.yaml | |
| git commit -m "Auto commit - update kustomization.yaml" | |
| git push | |
| #-------------------------------------------------------------------------------------------------- | |
| deploy-and-monitor: | |
| runs-on: self-hosted | |
| needs: [merge-branch, build-and-push, edit-kustomization, check-changes] | |
| permissions: | |
| id-token: write | |
| contents: read | |
| if: ${{ (needs.build-and-push.outputs.ARRAY_OF_CHANGES != '' || needs.check-changes.outputs.RUN_TESTS == 'true') && github.event.pull_request.draft == false }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.head_ref }} | |
| repository: ${{ github.event.pull_request.head.repo.full_name }} | |
| - name: Install terraform | |
| run: | | |
| sudo apt-get update && sudo apt-get install -y wget | |
| wget -O- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg | |
| echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list | |
| sudo apt update && sudo apt install terraform -y | |
| - name: Set short sha output | |
| run: echo "SHORT_GITHUB_SHA=`echo ${GITHUB_SHA} | cut -c1-7`" >> $GITHUB_ENV | |
| - name: Install kubectl | |
| uses: azure/setup-kubectl@v4 | |
| with: | |
| version: latest | |
| - name: Install kustomize | |
| uses: imranismail/setup-kustomize@v2 | |
| with: | |
| kustomize-version: 4.5.6 | |
| - name: Set e2e kubeconfig | |
| uses: azure/k8s-set-context@v4 | |
| with: | |
| kubeconfig: ${{ secrets.E2E_CLUSTER_KUBECONFIG }} | |
| - name: Get PR labels | |
| id: pr-labels | |
| uses: joerick/[email protected] | |
| # Deploy services to new namespace | |
| - name: Deploy to new namespace | |
| working-directory: ./manifests/claudie | |
| run: | | |
| #set log level to debug | |
| sed -i 's/GOLANG_LOG=info/GOLANG_LOG=debug/g' .env | |
| #check if auto clean up is enabled/disabled | |
| if [ -n "$GITHUB_PR_LABEL_DISABLE_CLEAN_UP" ]; then | |
| echo "AUTO_CLEAN_UP=FALSE" >> .env | |
| else | |
| echo "AUTO_CLEAN_UP=TRUE" >> .env | |
| fi | |
| sudo apt update && sudo apt install -y wget tar | |
| wget -q https://github.com/mikefarah/yq/releases/download/v4.27.2/yq_linux_amd64.tar.gz -O - |\ | |
| tar xz && mv yq_linux_amd64 yq | |
| NAME_HASH="claudie-operator-role-binding-${SHORT_GITHUB_SHA}-${GITHUB_RUN_NUMBER}" ./yq e 'select(di == 0) * (select(.kind == "ClusterRoleBinding") | .metadata.name = strenv(NAME_HASH))' ./cluster-rbac/clusterrolebinding.yaml -i | |
| NAMESPACES="claudie-${SHORT_GITHUB_SHA}-${GITHUB_RUN_NUMBER},e2e-secrets" ./yq eval 'select(documentIndex == 0).spec.template.spec.containers.0.env += [{"name": "CLAUDIE_NAMESPACES", "value": strenv(NAMESPACES)}]' -i operator.yaml | |
| kustomize edit set namespace claudie-${SHORT_GITHUB_SHA}-${GITHUB_RUN_NUMBER} | |
| kustomize build | kubectl apply -f - | |
| cat kustomization.yaml | |
| # Check if everything is ready and running | |
| - name: Monitor status of the new namespace | |
| run: | | |
| arr=( ${{ env.SERVICES }} ) | |
| echo "${arr[@]}" | |
| for SERVICE in "${arr[@]}" | |
| do | |
| if [ "${SERVICE}" != "testing-framework" ] && [ "${SERVICE}" != "autoscaler-adapter" ]; then | |
| kubectl wait deployment -l app.kubernetes.io/name=$SERVICE --for=condition=available --timeout=900s --namespace=claudie-${SHORT_GITHUB_SHA}-${GITHUB_RUN_NUMBER} | |
| fi | |
| done | |
| kubectl get pods --namespace=claudie-${SHORT_GITHUB_SHA}-${GITHUB_RUN_NUMBER} | |
| - name: Insert random test hostnames to loadbalancer test set | |
| working-directory: ./manifests/testing-framework/test-sets | |
| run: | | |
| sudo apt update && sudo apt install -y wget tar | |
| wget -q https://github.com/mikefarah/yq/releases/download/v4.27.2/yq_linux_amd64.tar.gz -O - |\ | |
| tar xz && mv yq_linux_amd64 yq | |
| HOSTNAME=$(echo $RANDOM | md5sum | head -c 20; echo;) ./yq e '.spec.loadBalancers.clusters.[1].dns.hostname = strenv(HOSTNAME)' test-set2/1.yaml -i | |
| HOSTNAME=$(echo $RANDOM | md5sum | head -c 20; echo;) ./yq e '.spec.loadBalancers.clusters.[0].dns.hostname = strenv(HOSTNAME)' test-set2/3.yaml -i | |
| - name: Create test static nodes | |
| working-directory: ./manifests/testing-framework/test-sets | |
| run: | | |
| sudo apt update && sudo apt install -y jq | |
| echo "Saving terraform script and key" | |
| echo "${{ secrets.CREATE_STATIC_NODES }}" | base64 -d > script.tf | |
| echo "Running terraform init" | |
| terraform init > /dev/null 2>&1 | |
| echo "Running terraform apply" | |
| terraform apply -auto-approve > /dev/null 2>&1 | |
| echo "Running terraform output" | |
| IPS=$(terraform output --json | jq -r '.endpoints.value | values[]') | |
| IP_ARR=( $IPS ) | |
| # use 3 nodes for the explicit static nodes test-set. | |
| for file in test-set5/*; do | |
| if [ -f "$file" ]; then | |
| filename=$(basename "$file") | |
| if [[ $filename == "1.yaml" ]]; then | |
| ENDPOINT=${IP_ARR[1]} ./yq e '.spec.nodePools.static.[0].nodes.[0].endpoint = strenv(ENDPOINT)' $file -i | |
| ENDPOINT=${IP_ARR[2]} ./yq e '.spec.nodePools.static.[0].nodes.[1].endpoint = strenv(ENDPOINT)' $file -i | |
| fi | |
| if [[ $filename == "2.yaml" ]]; then | |
| ENDPOINT=${IP_ARR[1]} ./yq e '.spec.nodePools.static.[0].nodes.[0].endpoint = strenv(ENDPOINT)' $file -i | |
| ENDPOINT=${IP_ARR[0]} ./yq e '.spec.nodePools.static.[0].nodes.[1].endpoint = strenv(ENDPOINT)' $file -i | |
| fi | |
| if [[ $filename == "3.yaml" ]]; then | |
| ENDPOINT=${IP_ARR[2]} ./yq e '.spec.nodePools.static.[0].nodes.[0].endpoint = strenv(ENDPOINT)' $file -i | |
| fi | |
| fi | |
| done | |
| # use 1 node for the rolling update test. | |
| for file in rolling-update/*; do | |
| if [ -f "$file" ]; then | |
| filename=$(basename "$file") | |
| if [[ $filename == "1.yaml" ]]; then | |
| ENDPOINT=${IP_ARR[3]} ./yq e '.spec.nodePools.static.[0].nodes.[0].endpoint = strenv(ENDPOINT)' $file -i | |
| fi | |
| if [[ $filename == "2.yaml" ]]; then | |
| ENDPOINT=${IP_ARR[3]} ./yq e '.spec.nodePools.static.[0].nodes.[0].endpoint = strenv(ENDPOINT)' $file -i | |
| fi | |
| fi | |
| done | |
| #Clean up | |
| rm -f yq | |
| rm -f yq.1 | |
| rm -f install-man-page.sh | |
| - name: Start the E2E tests | |
| working-directory: ./manifests | |
| run: | | |
| sudo apt update && sudo apt install -y wget tar | |
| wget -q https://github.com/mikefarah/yq/releases/download/v4.27.2/yq_linux_amd64.tar.gz -O - |\ | |
| tar xz && mv yq_linux_amd64 yq | |
| NAME_HASH="testing-framework-${SHORT_GITHUB_SHA}-${GITHUB_RUN_NUMBER}" ./yq e -i '(select(.kind == "ClusterRoleBinding").metadata.name = strenv(NAME_HASH))' ./testing-framework/testing-framework.yaml | |
| kustomize edit set namespace claudie-${SHORT_GITHUB_SHA}-${GITHUB_RUN_NUMBER} | |
| kustomize build . | kubectl apply -f - | |
| rm -f yq | |
| - name: Monitor E2E test | |
| run: | | |
| # Wait for completion as background process - capture PID | |
| kubectl wait --for=condition=complete --timeout=25000s job/testing-framework -n claudie-${SHORT_GITHUB_SHA}-${GITHUB_RUN_NUMBER} & | |
| completion_pid=$! | |
| # Wait for failure as background process - capture PID | |
| kubectl wait --for=condition=failed --timeout=25000s job/testing-framework -n claudie-${SHORT_GITHUB_SHA}-${GITHUB_RUN_NUMBER} && exit 1 & | |
| failure_pid=$! | |
| # capture exit code of the first subprocess to exit | |
| wait -n $completion_pid $failure_pid | |
| exit_code=$? | |
| if (( $exit_code == 0 )); then | |
| echo "Testing-framework successful" | |
| else | |
| echo "Testing-framework received an error" | |
| fi | |
| # Exit 0 on success, 1 on failure | |
| exit $exit_code | |
| - name: Delete temporary namespace | |
| run: | | |
| kubectl delete namespace claudie-${SHORT_GITHUB_SHA}-${GITHUB_RUN_NUMBER} | |
| - name: Destroy test static nodes | |
| if: always() | |
| working-directory: ./manifests/testing-framework/test-sets | |
| run: | | |
| echo "Running terraform init" | |
| terraform init > /dev/null 2>&1 | |
| echo "Running terraform destroy" | |
| terraform destroy -auto-approve > /dev/null 2>&1 |