Add livenessProbe and startupProbe to the sample-applications

This change adds the liveness and startup probe to the mongo and mysql applications within the sample applications that are used during CI runs. The livenessProbe checks if the relevant port is open within the container run, where the startupProbe checks if it's possible to actually connect to the DB endpoint from the container in which database is running. It also checks for the correct condition of the running pod and not only for the state in case probe fails. Increased timeout in the AreApplicationPodsRunning which is ran via Eventually to 19minuts which should be more then enough time for even complex scenarios. Removed sleep as it should not be needed anymore as we are checking for the startup probe now. Signed-off-by: Michal Pryc <[email protected]>
mpryc · Jan 23, 2024 · 7ce66f1 · 7ce66f1
1 parent a663e78
commit 7ce66f1
Show file tree

Hide file tree

Showing 8 changed files with 137 additions and 5 deletions.
diff --git a/tests/e2e/backup_restore_suite_test.go b/tests/e2e/backup_restore_suite_test.go
@@ -64,7 +64,6 @@ type BackupRestoreCase struct {
 	BackupRestoreType            BackupRestoreType
 	PreBackupVerify              VerificationFunction
 	PostRestoreVerify            VerificationFunction
-	AppReadyDelay                time.Duration
 	MustGatherFiles              []string            // list of files expected in must-gather under quay.io.../clusters/clustername/... ie. "namespaces/openshift-adp/oadp.openshift.io/dpa-ts-example-velero/ts-example-velero.yml"
 	MustGatherValidationFunction *func(string) error // validation function for must-gather where string parameter is the path to "quay.io.../clusters/clustername/"
 }
@@ -143,9 +142,6 @@ func runBackupAndRestore(brCase BackupRestoreCase, expectedErr error, updateLast
 	nsRequiresResticDCWorkaround, err := NamespaceRequiresResticDCWorkaround(dpaCR.Client, brCase.ApplicationNamespace)
 	Expect(err).ToNot(HaveOccurred())
 
-	// TODO this should be a function, not an arbitrary sleep
-	log.Printf("Sleeping for %v to allow application to be ready for case %s", brCase.AppReadyDelay, brCase.Name)
-	time.Sleep(brCase.AppReadyDelay)
 	// create backup
 	log.Printf("Creating backup %s for case %s", backupName, brCase.Name)
 	backup, err := CreateBackupForNamespaces(dpaCR.Client, namespace, backupName, []string{brCase.ApplicationNamespace}, brCase.BackupRestoreType == RESTIC || brCase.BackupRestoreType == KOPIA, brCase.BackupRestoreType == CSIDataMover)
@@ -289,7 +285,6 @@ var _ = Describe("Backup and restore tests", func() {
 			ApplicationNamespace: "mysql-persistent",
 			Name:                 "mysql-twovol-csi-e2e",
 			BackupRestoreType:    CSI,
-			AppReadyDelay:        30 * time.Second,
 			PreBackupVerify:      mysqlReady(true, true, CSI),
 			PostRestoreVerify:    mysqlReady(false, true, CSI),
 		}, nil),

diff --git a/tests/e2e/lib/apps.go b/tests/e2e/lib/apps.go
@@ -385,6 +385,14 @@ func AreApplicationPodsRunning(c *kubernetes.Clientset, namespace string) wait.C
 				log.Printf("Pod %v not yet succeeded: phase is %v", podInfo.Name, phase)
 				return false, nil
 			}
+
+			conditionType := corev1.ContainersReady
+			for _, condition := range podInfo.Status.Conditions {
+				if condition.Type == conditionType && condition.Status != corev1.ConditionTrue {
+					log.Printf("Pod %v not yet succeeded: condition is false: %v", podInfo.Name, conditionType)
+					return false, nil
+				}
+			}
 		}
 		return true, err
 	}

diff --git a/tests/e2e/sample-applications/mongo-persistent/mongo-persistent-block.yaml b/tests/e2e/sample-applications/mongo-persistent/mongo-persistent-block.yaml
@@ -129,6 +129,25 @@ items:
               volumeDevices:
                 - name:  block-volume-pv
                   devicePath: /dev/xvdx
+              livenessProbe:
+                tcpSocket:
+                  port: mongo
+                initialDelaySeconds: 5
+                periodSeconds: 10
+              startupProbe:
+                exec:
+                  command:
+                  - mongosh
+                  - admin
+                  - -u $(MONGO_INITDB_ROOT_USERNAME)
+                  - -p $(MONGO_INITDB_ROOT_PASSWORD)
+                  - --eval 'printjson(db.getCollectionNames())'
+                initialDelaySeconds: 5
+                periodSeconds: 30
+                timeoutSeconds: 2
+                successThreshold: 1
+                failureThreshold: 40 # 40x30sec before restart pod
+              restartPolicy: Always
           volumes:
           - name: block-volume-pv
             persistentVolumeClaim:

diff --git a/tests/e2e/sample-applications/mongo-persistent/mongo-persistent-csi.yaml b/tests/e2e/sample-applications/mongo-persistent/mongo-persistent-csi.yaml
@@ -86,6 +86,25 @@ items:
             volumeMounts:
             - name: mongo-data
               mountPath: /data/db
+            livenessProbe:
+              tcpSocket:
+                port: mongo
+              initialDelaySeconds: 5
+              periodSeconds: 10
+            startupProbe:
+              exec:
+                command:
+                - mongosh
+                - admin
+                - -u $(MONGO_INITDB_ROOT_USERNAME)
+                - -p $(MONGO_INITDB_ROOT_PASSWORD)
+                - --eval 'printjson(db.getCollectionNames())'
+              initialDelaySeconds: 5
+              periodSeconds: 30
+              timeoutSeconds: 2
+              successThreshold: 1
+              failureThreshold: 40 # 40x30sec before restart pod
+            restartPolicy: Always
           volumes:
           - name: mongo-data
             persistentVolumeClaim:

diff --git a/tests/e2e/sample-applications/mongo-persistent/mongo-persistent.yaml b/tests/e2e/sample-applications/mongo-persistent/mongo-persistent.yaml
@@ -99,6 +99,25 @@ items:
             volumeMounts:
             - name: mongo-data
               mountPath: /data/db
+            livenessProbe:
+              tcpSocket:
+                port: mongo
+              initialDelaySeconds: 5
+              periodSeconds: 10
+            startupProbe:
+              exec:
+                command:
+                - mongosh
+                - admin
+                - -u $(MONGO_INITDB_ROOT_USERNAME)
+                - -p $(MONGO_INITDB_ROOT_PASSWORD)
+                - --eval 'printjson(db.getCollectionNames())'
+              initialDelaySeconds: 5
+              periodSeconds: 30
+              timeoutSeconds: 2
+              successThreshold: 1
+              failureThreshold: 40 # 40x30sec before restart pod
+            restartPolicy: Always
           volumes:
           - name: mongo-data
             persistentVolumeClaim:

diff --git a/tests/e2e/sample-applications/mysql-persistent/mysql-persistent-csi.yaml b/tests/e2e/sample-applications/mysql-persistent/mysql-persistent-csi.yaml
@@ -103,6 +103,30 @@ items:
             volumeMounts:
             - name: mysql-data
               mountPath: /var/lib/mysql
+            livenessProbe:
+              tcpSocket:
+                port: mysql
+              initialDelaySeconds: 30
+              periodSeconds: 10
+              timeoutSeconds: 5
+            startupProbe:
+              exec:
+                command:
+                - /usr/bin/timeout
+                - 1s
+                - /usr/bin/mysql
+                - $(MYSQL_DATABASE)
+                - -h
+                - 127.0.0.1
+                - -u$(MYSQL_USER)
+                - -p$(MYSQL_PASSWORD)
+                - -e EXIT
+              initialDelaySeconds: 5
+              periodSeconds: 30
+              timeoutSeconds: 2
+              successThreshold: 1
+              failureThreshold: 40 # 40x30sec before restart pod
+            restartPolicy: Always
           volumes:
           - name: mysql-data
             persistentVolumeClaim:

diff --git a/tests/e2e/sample-applications/mysql-persistent/mysql-persistent-twovol-csi.yaml b/tests/e2e/sample-applications/mysql-persistent/mysql-persistent-twovol-csi.yaml
@@ -95,6 +95,30 @@ items:
             volumeMounts:
             - name: mysql-data
               mountPath: /var/lib/mysql
+            livenessProbe:
+              tcpSocket:
+                port: mysql
+              initialDelaySeconds: 30
+              periodSeconds: 10
+              timeoutSeconds: 5
+            startupProbe:
+              exec:
+                command:
+                - /usr/bin/timeout
+                - 1s
+                - /usr/bin/mysql
+                - $(MYSQL_DATABASE)
+                - -h
+                - 127.0.0.1
+                - -u$(MYSQL_USER)
+                - -p$(MYSQL_PASSWORD)
+                - -e EXIT
+              initialDelaySeconds: 5
+              periodSeconds: 30
+              timeoutSeconds: 2
+              successThreshold: 1
+              failureThreshold: 40 # 40x30sec before restart pod
+            restartPolicy: Always
           volumes:
           - name: mysql-data
             persistentVolumeClaim:

diff --git a/tests/e2e/sample-applications/mysql-persistent/mysql-persistent.yaml b/tests/e2e/sample-applications/mysql-persistent/mysql-persistent.yaml
@@ -116,6 +116,30 @@ items:
             volumeMounts:
             - name: mysql-data
               mountPath: /var/lib/mysql
+            livenessProbe:
+              tcpSocket:
+                port: mysql
+              initialDelaySeconds: 10
+              periodSeconds: 10
+              timeoutSeconds: 5
+            startupProbe:
+              exec:
+                command:
+                - /usr/bin/timeout
+                - 1s
+                - /usr/bin/mysql
+                - $(MYSQL_DATABASE)
+                - -h
+                - 127.0.0.1
+                - -u$(MYSQL_USER)
+                - -p$(MYSQL_PASSWORD)
+                - -e EXIT
+              initialDelaySeconds: 5
+              periodSeconds: 30
+              timeoutSeconds: 2
+              successThreshold: 1
+              failureThreshold: 40 # 40x30sec before restart pod
+            restartPolicy: Always
           volumes:
           - name: mysql-data
             persistentVolumeClaim: