cleanup func added

Signed-off-by: sayedppqq <[email protected]>

.github/workflows/ci.yml

@@ -13,12 +13,12 @@ on:
       folders:
         description: 'Folders'
         required: true
-        default: 'all'
+        default: 'mongodb'
         type: string
       kubedb_version:
         description: 'KubeDB Version'
         required: true
-        default: 'v2024.4.27'  #needs to be dynamic
+        default: 'v2024.6.4'  #needs to be dynamic
         type: string
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}-dashboard-check
@@ -34,11 +34,11 @@ jobs:
       - name: Defaulter
         id: defaulter
         env:
-          FOLDERS: ${{ inputs.folders }}
+          FOLDERS: "mongodb"
           KUBEDB_VERSION: ${{ inputs.kubedb_version }}
         run: |
-          echo "folders=${FOLDERS:-all}" >> $GITHUB_OUTPUT
-          echo "kubedb_version=${KUBEDB_VERSION:-v2024.4.27}" >> $GITHUB_OUTPUT
+          echo "folders=${FOLDERS:-mongodb}" >> $GITHUB_OUTPUT
+          echo "kubedb_version=${KUBEDB_VERSION:-v2024.6.4}" >> $GITHUB_OUTPUT
   test:
     name: Test
     runs-on: ubuntu-latest
@@ -52,6 +52,11 @@ jobs:
         with:
           ref: workflow
 
+      - name: Setup tmate session
+        uses: mxschmitt/action-tmate@v3
+        with:
+          detached: true
+
       - name: Set up Go 1.22
         uses: actions/setup-go@v4
         with:
@@ -101,11 +106,6 @@ jobs:
         run: |
           kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.11.0/cert-manager.yaml
 
-      - name: Setup tmate session
-        uses: mxschmitt/action-tmate@v3
-        with:
-          detached: true
-
       - name: Issue License
         env:
           BYTEBUILDERS_LICENSE_TOKEN: ${{ secrets.BYTEBUILDERS_LICENSE_TOKEN }}
@@ -166,12 +166,14 @@ jobs:
         run: |
           helm repo add appscode https://charts.appscode.com/stable/
           helm repo update
-          helm upgrade -i kubedb-metrics appscode/kubedb-metrics -n kubedb --create-namespace --version=${{ needs.config.outputs.kubedb }} --wait --burst-limit=10000 --debug
+          helm upgrade -i kubedb-metrics appscode/kubedb-metrics -n kubedb --create-namespace --version=${{ needs.config.outputs.kubedb }} --wait
 
       - name: Prepare smaples
         run: |
           cd ..
           git clone https://github.com/kubedb/samples.git
+          cd samples
+          git checkout fr-web
 
       - name: Check Dashboard Metrics
         env:

connectcluster/connectcluster-connect.json

@@ -749,7 +749,7 @@
       "targets": [
         {
           "exemplar": true,
-          "expr": "kafka_connect_app_info{instance=~\"$instance\",start_time_ms!=\"\"}",
+          "expr": "kafka_connect_app_info{instance=~\"$instance\"}",
           "hide": false,
           "interval": "",
           "legendFormat": "",
@@ -781,7 +781,7 @@
           "refId": "D"
         },
         {
-          "expr": "sum by (pod) (kafka_connect_worker__connector_startup_failure_total{namespace=~\"$namespace\",pod=~\"$app-.+$\"})",
+          "expr": "sum by (pod) (kafka_connect_worker_connector_startup_failure_total{namespace=~\"$namespace\",pod=~\"$app-.+$\"})",
           "format": "table",
           "instant": true,
           "interval": "",

hack/scripts/check-dashboard.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-set -e
+#set -e
 
 declare -A kind=(["connectcluster"]="ConnectCluster" ["druid"]="Druid" ["elasticsearch"]="Elasticsearch" ["kafka"]="Kafka" ["mariadb"]="MariaDB"
             ["mongodb"]="MongoDB" ["mysql"]="MySQL" ["perconaxtradb"]="PerconaXtraDB" ["pgpool"]="Pgpool" ["postgres"]="Postgres" ["proxysql"]="ProxySQL"
@@ -18,12 +18,21 @@ if [ $FOLDERS != "all" ]; then
 fi
 echo "folder_array = ${folder_array}"
 
+PROMETHEUS_SERVICE_NAMESPACE="monitoring"
+PROMETHEUS_SERVICE_NAME="prometheus-kube-prometheus-prometheus"
+kubectl port-forward svc/${PROMETHEUS_SERVICE_NAME} 9090:9090 -n ${PROMETHEUS_SERVICE_NAMESPACE} &
+# `$!` is a special variable in bash that holds the PID of the most recently executed background command.
+PORT_FORWARD_PID=$!
+
+# Give port-forwarding some time to establish
+sleep 5
+
 create_db_dependencies() {
     folder="$1"
     echo "folder=$folder"
     if [ "$folder" == "druid" ]; then
         kubectl create configmap -n demo my-init-script \
-          --from-literal=init.sql="$(curl -fsSL https://raw.githubusercontent.com/kubedb/samples/old-dbs/druid/monitoring/mysql-init-script.sql)"
+          --from-literal=init.sql="$(curl -fsSL https://raw.githubusercontent.com/kubedb/samples/master/druid/monitoring/mysql-init-script.sql)"
 
         helm repo add minio https://operator.min.io/
         helm upgrade --install --namespace "minio-operator" --create-namespace "minio-operator" minio/operator --set operator.replicaCount=1 --wait
@@ -59,26 +68,59 @@ create_db_dependencies() {
         kubectl apply -f ../samples/solr/monitoring/zookeeper.yaml
         kubectl wait --for=jsonpath='{.status.phase}'=Ready ZooKeeper zookeeper -n demo --timeout=10m
     fi
-}
 
-delete_db_dependencies() {
+    if [ "$folder" == "connectcluster" ]; then
+        kubectl apply -f ../samples/kafka/connectcluster/monitoring/kafka.yaml
+        kubectl wait --for=jsonpath='{.status.phase}'=Ready Kafka kafka -n demo --timeout=10m
+
+        kubectl apply -f ../samples/kafka/connectcluster/monitoring/connect-cluster.yaml
+        kubectl wait --for=jsonpath='{.status.phase}'=Ready ConnectCluster connectcluster -n demo --timeout=10m
 
+        kubectl apply -f ../samples/kafka/connectcluster/monitoring/file-source.yaml
+        sleep 2s
+    fi
+}
+
+cleanup() {
+  path="$1"
+  folder="$2"
+  if [ $folder == "connectcluster" ]; then
+    kubectl delete -f ../samples/kafka/connectcluster/monitoring/connect-cluster.yaml
+  fi
+  kubectl delete -f $path
+#  kubectl delete secret -n demo --all
 }
 
 check_dashboard_for_non_dbs() {
-    sleep 30s # waiting for the metrics to be generated
+    sleep 60s # waiting for the metrics to be generated
     folder="$1"
     inside_files_array="$2"
     for file in "${inside_files_array[@]}"; do
       if [[ $file == *.json ]]; then
         dashboard_name="${file::-5}"
         echo "checking for dashboard $dashboard_name"
         url="https://raw.githubusercontent.com/appscode/grafana-dashboards/master/$folder/$file"
-        $HOME/go/bin/kubectl-dba monitor dashboard -u $url -o=true --prom-svc-name=prometheus-kube-prometheus-prometheus --prom-svc-namespace=monitoring --prom-svc-port=9090
+        echo "$HOME/go/bin/kubectl-dba monitor dashboard -u $url -o=true --prom-svc-name=prometheus-kube-prometheus-prometheus --prom-svc-namespace=monitoring --prom-svc-port=9090"
+        $HOME/go/bin/kubectl-dba monitor dashboard -u $url -d=false --prom-svc-name=prometheus-kube-prometheus-prometheus --prom-svc-namespace=monitoring --prom-svc-port=9090
       fi
     done
 }
 
+wait_for_prometheus_target() {
+  target="$1"
+  echo "checking if $target-stats target exist in prometheus..."
+  for (( i=1; i<=600; i++ )); do
+      # Curl the Prometheus API to get the targets and extract the pool information
+      Targets=$(curl -s http://localhost:9090/api/v1/targets | jq -r '.data.activeTargets[] | .labels.service')
+      if echo "$Targets" | grep -q "$target"; then
+        echo "$target target found in prometheus"
+        sleep 30s
+        break
+      fi
+      sleep 5s
+  done
+}
+
 
 kubectl create ns demo
 
@@ -99,37 +141,39 @@ for folder in "${folder_array[@]}"; do
     kubectl apply -f $path
     kubectl wait --for=jsonpath='{.status.phase}'=Ready ${kind[$folder]} $folder -n demo --timeout=10m
 
-    sleep 30s
+    wait_for_prometheus_target "$folder-stats"
+
     for file in "${inside_files_array[@]}"; do
       if [[ $file == *.json ]]; then
         dashboard_name="${file::-5}"
         echo "checking for dashboard $dashboard_name"
-        $HOME/go/bin/kubectl-dba monitor dashboard $folder $folder -n demo $dashboard_name --prom-svc-name=prometheus-kube-prometheus-prometheus --prom-svc-namespace=monitoring --prom-svc-port=9090
+        echo "$HOME/go/bin/kubectl-dba monitor dashboard $folder $folder -n demo $dashboard_name --prom-svc-name=prometheus-kube-prometheus-prometheus --prom-svc-namespace=monitoring --prom-svc-port=9090 -b=workflow"
+        $HOME/go/bin/kubectl-dba monitor dashboard $folder $folder -n demo $dashboard_name --prom-svc-name=prometheus-kube-prometheus-prometheus --prom-svc-namespace=monitoring --prom-svc-port=9090 -b=workflow
       fi
     done
+#    cleanup "$path" "$folder"
 
-    kubectl delete -f $path
-    delete_db_dependencies "$folder"
   elif [ "$folder" == "stash" ]; then
     echo "non db object name: $folder"
     readarray -t inside_files_array < <(ls "$folder")
 
     bash ./hack/scripts/stash-flow.sh
+    wait_for_prometheus_target "stash-stash-enterprise"
     check_dashboard_for_non_dbs "$folder" "$inside_files_array"
 
-    kubectl delete -f ./hack/yamls/backupconfiguration.yaml
-    kubectl delete -f ./hack/yamls/restoresession.yaml
-    kubectl delete -f ./hack/yamls/repository.yaml
-    kubectl delete -f ../samples/mongodb/monitoring/mongodb_standalone.yaml
+    cleanup "./hack/yamls/stash"
 
   elif [ "$folder" == "policy" ]; then
     echo "non db object name: $folder"
     readarray -t inside_files_array < <(ls "$folder")
 
     bash ./hack/scripts/policy-flow.sh
+    wait_for_prometheus_target "falco-ui-server"
     check_dashboard_for_non_dbs "$folder" "$inside_files_array"
 
-    kubectl delete -f ./hack/yamls/policy/constraint-template.yaml
-    kubectl delete -f ./hack/yamls/policy/constraint.yaml
+    cleanup "./hack/yamls/policy"
   fi
-done
+
+done
+
+kill $PORT_FORWARD_PID

hack/scripts/stash-flow.sh

@@ -27,7 +27,7 @@ helm install stash oci://ghcr.io/appscode-charts/stash \
   --set stash-enterprise.monitoring.backup=true \
   --set stash-enterprise.monitoring.operator=true \
   --set stash-enterprise.monitoring.serviceMonitor.labels.release=prometheus \
-  --set-file global.license=/tmp/license.txt \
+  --set-file global.license=/tmp/kubedb-license.txt \
   --wait --burst-limit=10000
 
 curl -o kubectl-stash.tar.gz -fsSL https://github.com/stashed/cli/releases/download/v0.34.0/kubectl-stash-linux-amd64.tar.gz \
@@ -39,7 +39,7 @@ curl -o kubectl-stash.tar.gz -fsSL https://github.com/stashed/cli/releases/downl
 kubectl apply -f ./hack/yamls/stash/repository.yaml
 sleep 10
 
-kubectl apply -f ../samples/mongodb/monitoring/mongodb_standalone.yaml
+kubectl apply -f ./hack/yamls/stash/mongodb_standalone.yaml
 kubectl wait --for=jsonpath='{.status.phase}'=Ready MongoDB mongodb-standalone -n demo --timeout=10m
 
 kubectl apply -f ./hack/yamls/stash/backupconfiguration.yaml

hack/yamls/stash/mongodb_standalone.yaml

@@ -0,0 +1,22 @@
+apiVersion: kubedb.com/v1alpha2
+kind: MongoDB
+metadata:
+  name: mongodb-standalone
+  namespace: demo
+spec:
+  version: "4.4.26"
+  terminationPolicy: WipeOut
+  storage:
+    storageClassName: "standard"
+    accessModes:
+      - ReadWriteOnce
+    resources:
+      requests:
+        storage: 1Gi
+  monitor:
+    agent: prometheus.io/operator
+    prometheus:
+      serviceMonitor:
+        labels:
+          release: prometheus
+        interval: 10s