Merge branch 'w/2.7/improvement/ZENKO-4899' into w/2.8/improvement/ZE…

…NKO-4899
scality · Oct 23, 2024 · af602ba · af602ba
2 parents 3d8a80e + 7b3dc9d
commit af602ba
Show file tree

Hide file tree

Showing 10 changed files with 198 additions and 172 deletions.
diff --git a/.github/actions/archive-artifacts/action.yaml b/.github/actions/archive-artifacts/action.yaml
@@ -1,27 +1,79 @@
 ---
-name: "Archive artifact logs and data"
-description: "Archive logs, pods and events"
+name: Archive artifact logs and data
+description: Archive logs, pods and events
+
+inputs:
+  user:
+    description: Artifacts user
+    required: true
+  password:
+    description: Artifacts password
+    required: true
+  junit-paths:
+    description: Path to junit reports
+    default: /artifacts/data/reports/*.xml
+    required: true
+  stage:
+    description: Stage name
+    default: ${{ github.job }}.${{ github.run_attempt }}
+    required: true
+  trunk_token:
+    description: Trunk token
+    required: false
+  zenko-name:
+    description: Zenko name
+    default: end2end
+    required: false
+  zenko-namespace:
+    description: Namespace
+    default: default
+    required: false
 
 runs:
   using: composite
   steps:
+    - name: Publish test report
+      uses: mikepenz/action-junit-report@v4
+      with:
+        annotate_only: true
+        check_name: ${{ inputs.stage}}
+        report_paths: ${{ inputs.junit-paths }}
+        job_summary: ${{ job.status != 'success' }} # Only show the summary if the job failed
+        detailed_summary: true
+      continue-on-error: true
+
+    - name: Upload results
+      if: inputs.trunk_token && job.status != 'cancelled'
+      uses: trunk-io/[email protected]
+      with:
+        junit-paths: ${{ inputs.junit-paths }}
+        org-slug: ${{ github.repository_owner }}
+        token: ${{ inputs.trunk_token }}
+      continue-on-error: true
+
     - name: Archive artifact logs and data
       shell: bash
       run: |-
-        set -exu;
-        mkdir -p /tmp/artifacts/data/${STAGE}/kind-logs;
-        kubectl get pods -A -o yaml > /tmp/artifacts/data/${STAGE}/kind-logs/all-pods.log;
-        kubectl get events -A -o yaml > /tmp/artifacts/data/${STAGE}/kind-logs/all-events.log;
-        kind export logs /tmp/artifacts/data/${STAGE}/kind-logs/kind-export;
-        tar zcvf /tmp/artifacts/${{ github.sha }}-${STAGE}-logs-volumes.tgz /tmp/artifacts/data/${STAGE}/kind-logs;
+        set -exu
+
+        mkdir -p /tmp/artifacts/data/${STAGE}/kind-logs
+        kubectl get pods -A -o yaml > /tmp/artifacts/data/${STAGE}/kind-logs/all-pods.log
+        kubectl get events -A -o yaml > /tmp/artifacts/data/${STAGE}/kind-logs/all-events.log
+        kubectl get zenko -A -o yaml > /tmp/artifacts/data/${STAGE}/kind-logs/all-zenkos.log
+        kubectl get zenkodrsource -A -o yaml > /tmp/artifacts/data/${STAGE}/kind-logs/all-zenkodrsources.log
+        kubectl get zenkodrsink -A -o yaml > /tmp/artifacts/data/${STAGE}/kind-logs/all-zenkodrsinks.log
+        kind export logs /tmp/artifacts/data/${STAGE}/kind-logs/kind-export
+        tar zcvf /tmp/artifacts/${{ github.sha }}-${STAGE}-logs-volumes.tgz /tmp/artifacts/data/${STAGE}/kind-logs
+      env:
+        STAGE: ${{ inputs.stage }}
+      continue-on-error: true
+
     - name: Dump kafka
       shell: bash
-      continue-on-error: true
       run: |-
         set -exu
 
-        NAMESPACE=${NAMESPACE:-default}
-        KAFKA=$(kubectl get pods -n ${NAMESPACE} -lkafka_cr=${ZENKO_NAME:-end2end}-base-queue -o jsonpath='{.items[0].metadata.name}')
+        KAFKA=$(kubectl get pods -n ${NAMESPACE} -lkafka_cr=${ZENKO_NAME}-base-queue -o jsonpath='{.items[0].metadata.name}')
 
         kubectl exec -in ${NAMESPACE} ${KAFKA} -c kafka -- \
             env KAFKA_OPTS= kafka-topics.sh --bootstrap-server :9092 --list \
@@ -35,16 +87,21 @@ runs:
                    env KAFKA_OPTS= kafka-consumer-groups.sh --bootstrap-server :9092 --describe --all-groups \
                    > /tmp/artifacts/data/${STAGE}/kafka-offsets.log
 
-        KAFKA_SERVICE=$(kubectl get services -n ${NAMESPACE} -lkafka_cr=${ZENKO_NAME:-end2end}-base-queue -o jsonpath='{.items[0].metadata.name}')
+        KAFKA_SERVICE=$(kubectl get services -n ${NAMESPACE} -lkafka_cr=${ZENKO_NAME}-base-queue -o jsonpath='{.items[0].metadata.name}')
         kubectl run -n ${NAMESPACE} kcat --image=edenhill/kcat:1.7.1 --restart=Never --command -- sleep 300        
         kubectl wait -n ${NAMESPACE} pod kcat --for=condition=ready
         cat /tmp/artifacts/data/${STAGE}/kafka-topics.log | grep -v '^__' | xargs -P 15 -I {} \
             sh -c "kubectl exec -i -n ${NAMESPACE} kcat -- \
                        kcat -L -b ${KAFKA_SERVICE} -t {}  -C -o beginning -e -q -J \
                        > /tmp/artifacts/data/${STAGE}/kafka-messages-{}.log"
+      env:
+        STAGE: ${{ inputs.stage }}
+        NAMESPACE: ${{ inputs.zenko-namespace }}
+        ZENKO_NAME: ${{ inputs.zenko-name }}
+      continue-on-error: true
+
     - name: Dump MongoDB
       shell: bash
-      continue-on-error: true
       run: |-
         set -exu
 
@@ -54,9 +111,29 @@ runs:
         NAMESPACE="${NAMESPACE:-default}"
         DUMP_DIR="/tmp/mongodb.dump"
 
-        kubectl exec -n ${NAMESPACE} data-db-mongodb-sharded-mongos-0 -- mongodump --db ${ZENKO_MONGODB_DATABASE} -u ${MONGODB_ROOT_USERNAME} -p ${MONGODB_ROOT_PASSWORD} --authenticationDatabase admin --out ${DUMP_DIR}
+        kubectl exec -n ${NAMESPACE} data-db-mongodb-sharded-mongos-0 -- mongodump \
+            --db ${ZENKO_MONGODB_DATABASE} -u ${MONGODB_ROOT_USERNAME} -p ${MONGODB_ROOT_PASSWORD} \
+            --authenticationDatabase admin --out ${DUMP_DIR}
 
-        kubectl exec -n ${NAMESPACE} data-db-mongodb-sharded-mongos-0 -- bash -c "for bson_file in ${DUMP_DIR}/${ZENKO_MONGODB_DATABASE}/*.bson; do json_file=\"${DUMP_DIR}/\$(basename \${bson_file} .bson).json\"; bsondump --outFile \${json_file} \${bson_file}; done"
+        kubectl exec -n ${NAMESPACE} data-db-mongodb-sharded-mongos-0 -- bash -c \
+            "for bson_file in ${DUMP_DIR}/${ZENKO_MONGODB_DATABASE}/*.bson; do \
+               json_file=\"${DUMP_DIR}/\$(basename \${bson_file} .bson).json\"; \
+               bsondump --outFile \${json_file} \${bson_file}; \
+             done"
 
         mkdir -p /tmp/artifacts/data/${STAGE}/mongodb-dump
         kubectl cp ${NAMESPACE}/data-db-mongodb-sharded-mongos-0:${DUMP_DIR} /tmp/artifacts/data/${STAGE}/mongodb-dump
+      env:
+        STAGE: ${{ inputs.stage }}.${{ github.run_attempt }}
+        NAMESPACE: ${{ inputs.zenko-namespace }}
+        ZENKO_NAME: ${{ inputs.zenko-name }}
+      continue-on-error: true
+
+    - name: Upload artifacts                               # move into `archive-artifacts` action
+      uses: scality/action-artifacts@v4
+      with:
+        method: upload
+        url: https://artifacts.scality.net
+        user: ${{ inputs.user }}
+        password: ${{ inputs.password }}
+        source: /tmp/artifacts
diff --git a/.github/scripts/end2end/configs/zenko.yaml b/.github/scripts/end2end/configs/zenko.yaml
@@ -103,6 +103,9 @@ spec:
     enable: true
     configurationOverrides:
       e2e-cold:
+        # for mock DMF, we need to override the endpoint to use in-cluster service: otherwise it
+        # relies on external hostname (ok thanks to coredns patch), but TLS cert is not valid
+        s3-endpoint: http://${ZENKO_NAME}-internal-s3api.default.svc.cluster.local
         debug: "true"
         command-timeout: "60s"
         pending-job-poll-after-age: "10s"

diff --git a/.github/scripts/end2end/install-kind-dependencies.sh b/.github/scripts/end2end/install-kind-dependencies.sh
@@ -128,6 +128,24 @@ build_solution_base_manifests() {
     sed -i "s/MONGODB_SHARDSERVER_RAM_LIMIT/${MONGODB_SHARDSERVER_RAM_LIMIT}/g" $DIR/_build/root/deploy/*
     sed -i "s/MONGODB_SHARDSERVER_RAM_REQUEST/${MONGODB_SHARDSERVER_RAM_REQUEST}/g" $DIR/_build/root/deploy/*
     sed -i "s/MONGODB_MONGOS_RAM_REQUEST/${MONGODB_MONGOS_RAM_REQUEST}/g" $DIR/_build/root/deploy/*
+
+    # Limits and requests for MongoDB are computed based on the current system
+    # Detect total system RAM in GiB
+    TOTAL_RAM_GB=$(awk '/MemTotal/ {printf "%.0f", $2/1024/1024}' /proc/meminfo)
+
+    # Compute MongoDB settings based on the total RAM
+    MONGODB_WIRETIGER_CACHE_SIZE_GB=$((TOTAL_RAM_GB * 335 / 1000))
+    MONGODB_MONGOS_RAM_LIMIT=$((TOTAL_RAM_GB * 165 / 1000))Gi
+    MONGODB_SHARDSERVER_RAM_LIMIT=$((2 * MONGODB_WIRETIGER_CACHE_SIZE_GB))Gi
+    MONGODB_SHARDSERVER_RAM_REQUEST=${MONGODB_WIRETIGER_CACHE_SIZE_GB}Gi
+    MONGODB_MONGOS_RAM_REQUEST=$((TOTAL_RAM_GB * 33 / 1000))Gi
+
+    # Replace values before deploying
+    sed -i "s/MONGODB_SHARDSERVER_EXTRA_FLAGS/--wiredTigerCacheSizeGB=${MONGODB_WIRETIGER_CACHE_SIZE_GB}/g" $DIR/_build/root/deploy/*
+    sed -i "s/MONGODB_MONGOS_RAM_LIMIT/${MONGODB_MONGOS_RAM_LIMIT}/g" $DIR/_build/root/deploy/*
+    sed -i "s/MONGODB_SHARDSERVER_RAM_LIMIT/${MONGODB_SHARDSERVER_RAM_LIMIT}/g" $DIR/_build/root/deploy/*
+    sed -i "s/MONGODB_SHARDSERVER_RAM_REQUEST/${MONGODB_SHARDSERVER_RAM_REQUEST}/g" $DIR/_build/root/deploy/*
+    sed -i "s/MONGODB_MONGOS_RAM_REQUEST/${MONGODB_MONGOS_RAM_REQUEST}/g" $DIR/_build/root/deploy/*
 }
 
 get_image_from_deps() {

diff --git a/.github/scripts/end2end/requirements.sh b/.github/scripts/end2end/requirements.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 
 KUBECTL_VERSION=1.21.1
-KIND_VERSION=v0.20.0
+KIND_VERSION=v0.24.0
 HELM_VERSION=v3.5.3
 KUSTOMIZE_VERSION=v4.4.1
 YQ_VERSION=v4.27.5