reboot nfs_client_vm if not accessible and retry connection (#10129)

* reboot nfs_client_vm if not accessible and retry connection
* do not reboot the VM, if not necessary

Signed-off-by: Daniel Horak <[email protected]>

tests/functional/nfs_feature/test_nfs_feature_enable_for_ODF_clusters.py

@@ -2,9 +2,11 @@
 import logging
 import time
 import os
+import socket
 
 
 from ocs_ci.utility import nfs_utils
+from ocs_ci.utility.utils import exec_cmd
 from ocs_ci.framework import config
 from ocs_ci.utility.connection import Connection
 from ocs_ci.ocs import constants, ocp
@@ -28,7 +30,7 @@
 
 from ocs_ci.ocs.resources import pod, ocs
 from ocs_ci.utility.retry import retry
-from ocs_ci.ocs.exceptions import CommandFailed
+from ocs_ci.ocs.exceptions import CommandFailed, ConfigurationError
 
 
 log = logging.getLogger(__name__)
@@ -146,8 +148,6 @@ def setup_teardown(self, request):
             or config.DEPLOYMENT["ssh_key_private"]
         )
 
-        self.con = None
-
         # Enable nfs feature
         log.info("----Enable nfs----")
         nfs_ganesha_pod_name = nfs_utils.nfs_enable(
@@ -194,6 +194,53 @@ def teardown(self):
             log.info("Delete mount point")
             _, _, _ = self.con.exec_cmd("rm -rf " + self.test_folder)
 
+    # the NFS Client VM might not be healthy, so rebooting it and re-trying
+    @property
+    @retry((TimeoutError, socket.gaierror), tries=3, delay=60, backoff=1)
+    def con(self):
+        """
+        Create connection to NFS Client VM, if not accessible, try to restart it.
+        """
+        if (
+            not hasattr(self, "__nfs_client_connection")
+            or not self.__nfs_client_connection
+        ):
+            try:
+                self.__nfs_client_connection = self.get_nfs_client_connection(
+                    re_try=False
+                )
+            except (TimeoutError, socket.gaierror):
+                nfs_client_vm_cloud = config.ENV_DATA.get("nfs_client_vm_cloud")
+                nfs_client_vm_name = config.ENV_DATA.get("nfs_client_vm_name")
+                if not nfs_client_vm_cloud or not nfs_client_vm_name:
+                    raise ConfigurationError(
+                        "NFS Client VM is not accessible and ENV_DATA nfs_client_vm_cloud and/or nfs_client_vm_name "
+                        "parameters are not configured to be able to automatically reboot the NFS Client VM."
+                    )
+                cmd = f"openstack --os-cloud {nfs_client_vm_cloud} server reboot --hard --wait {nfs_client_vm_name}"
+                exec_cmd(cmd)
+
+                time.sleep(60)
+                self.__nfs_client_connection = self.get_nfs_client_connection()
+        return self.__nfs_client_connection
+
+    def get_nfs_client_connection(self, re_try=True):
+        """
+        Create connection to NFS Client VM.
+        """
+        log.info("Connecting to nfs client test VM")
+        tries = 3 if re_try else 1
+
+        @retry((TimeoutError, socket.gaierror), tries=tries, delay=60, backoff=1)
+        def __make_connection():
+            return Connection(
+                self.nfs_client_ip,
+                self.nfs_client_user,
+                private_key=self.nfs_client_private_key,
+            )
+
+        return __make_connection()
+
     @tier1
     @polarion_id("OCS-4269")
     def test_nfs_feature_enable(
@@ -326,14 +373,6 @@ def test_outcluster_nfs_export(
         """
         nfs_utils.skip_test_if_nfs_client_unavailable(self.nfs_client_ip)
 
-        # ssh to test-nfs-vm
-        log.info("Login to test vm")
-        self.con = Connection(
-            self.nfs_client_ip,
-            self.nfs_client_user,
-            private_key=self.nfs_client_private_key,
-        )
-
         # Create nfs pvcs with storageclass ocs-storagecluster-ceph-nfs
         nfs_pvc_obj = helpers.create_pvc(
             sc_name=self.nfs_sc,
@@ -504,14 +543,6 @@ def test_multiple_nfs_based_PVs(
         """
         nfs_utils.skip_test_if_nfs_client_unavailable(self.nfs_client_ip)
 
-        # ssh to test-nfs-vm
-        log.info("Login to test vm")
-        self.con = Connection(
-            self.nfs_client_ip,
-            self.nfs_client_user,
-            private_key=self.nfs_client_private_key,
-        )
-
         # Create nfs pvcs with storageclass ocs-storagecluster-ceph-nfs
         nfs_pvc_objs, yaml_creation_dir = helpers.create_multiple_pvcs(
             sc_name=self.nfs_sc,
@@ -654,14 +685,6 @@ def test_multiple_mounts_of_same_nfs_volume(
         """
         nfs_utils.skip_test_if_nfs_client_unavailable(self.nfs_client_ip)
 
-        # ssh to test-nfs-vm
-        log.info("Login to test vm")
-        self.con = Connection(
-            self.nfs_client_ip,
-            self.nfs_client_user,
-            private_key=self.nfs_client_private_key,
-        )
-
         # Create nfs pvc with storageclass ocs-storagecluster-ceph-nfs
         pvc_objs = []
         nfs_pvc_obj = helpers.create_pvc(
@@ -801,13 +824,6 @@ def test_external_nfs_client_can_write_read_new_file(
         """
         nfs_utils.skip_test_if_nfs_client_unavailable(self.nfs_client_ip)
 
-        # ssh to test-nfs-vm
-        log.info("Login to test vm")
-        self.con = Connection(
-            self.nfs_client_ip,
-            self.nfs_client_user,
-            private_key=self.nfs_client_private_key,
-        )
         # Create nfs pvcs with storageclass ocs-storagecluster-ceph-nfs
         nfs_pvc_obj = helpers.create_pvc(
             sc_name=self.nfs_sc,