diff --git a/tests/functional/nfs_feature/test_nfs_feature_enable_for_ODF_clusters.py b/tests/functional/nfs_feature/test_nfs_feature_enable_for_ODF_clusters.py index 9547b7e29e2..1a0c6dabb20 100644 --- a/tests/functional/nfs_feature/test_nfs_feature_enable_for_ODF_clusters.py +++ b/tests/functional/nfs_feature/test_nfs_feature_enable_for_ODF_clusters.py @@ -2,9 +2,11 @@ import logging import time import os +import socket from ocs_ci.utility import nfs_utils +from ocs_ci.utility.utils import exec_cmd from ocs_ci.framework import config from ocs_ci.utility.connection import Connection from ocs_ci.ocs import constants, ocp @@ -28,7 +30,7 @@ from ocs_ci.ocs.resources import pod, ocs from ocs_ci.utility.retry import retry -from ocs_ci.ocs.exceptions import CommandFailed +from ocs_ci.ocs.exceptions import CommandFailed, ConfigurationError log = logging.getLogger(__name__) @@ -146,8 +148,6 @@ def setup_teardown(self, request): or config.DEPLOYMENT["ssh_key_private"] ) - self.con = None - # Enable nfs feature log.info("----Enable nfs----") nfs_ganesha_pod_name = nfs_utils.nfs_enable( @@ -194,6 +194,53 @@ def teardown(self): log.info("Delete mount point") _, _, _ = self.con.exec_cmd("rm -rf " + self.test_folder) + # the NFS Client VM might not be healthy, so rebooting it and re-trying + @property + @retry((TimeoutError, socket.gaierror), tries=3, delay=60, backoff=1) + def con(self): + """ + Create connection to NFS Client VM, if not accessible, try to restart it. + """ + if ( + not hasattr(self, "__nfs_client_connection") + or not self.__nfs_client_connection + ): + try: + self.__nfs_client_connection = self.get_nfs_client_connection( + re_try=False + ) + except (TimeoutError, socket.gaierror): + nfs_client_vm_cloud = config.ENV_DATA.get("nfs_client_vm_cloud") + nfs_client_vm_name = config.ENV_DATA.get("nfs_client_vm_name") + if not nfs_client_vm_cloud or not nfs_client_vm_name: + raise ConfigurationError( + "NFS Client VM is not accessible and ENV_DATA nfs_client_vm_cloud and/or nfs_client_vm_name " + "parameters are not configured to be able to automatically reboot the NFS Client VM." + ) + cmd = f"openstack --os-cloud {nfs_client_vm_cloud} server reboot --hard --wait {nfs_client_vm_name}" + exec_cmd(cmd) + + time.sleep(60) + self.__nfs_client_connection = self.get_nfs_client_connection() + return self.__nfs_client_connection + + def get_nfs_client_connection(self, re_try=True): + """ + Create connection to NFS Client VM. + """ + log.info("Connecting to nfs client test VM") + tries = 3 if re_try else 1 + + @retry((TimeoutError, socket.gaierror), tries=tries, delay=60, backoff=1) + def __make_connection(): + return Connection( + self.nfs_client_ip, + self.nfs_client_user, + private_key=self.nfs_client_private_key, + ) + + return __make_connection() + @tier1 @polarion_id("OCS-4269") def test_nfs_feature_enable( @@ -326,14 +373,6 @@ def test_outcluster_nfs_export( """ nfs_utils.skip_test_if_nfs_client_unavailable(self.nfs_client_ip) - # ssh to test-nfs-vm - log.info("Login to test vm") - self.con = Connection( - self.nfs_client_ip, - self.nfs_client_user, - private_key=self.nfs_client_private_key, - ) - # Create nfs pvcs with storageclass ocs-storagecluster-ceph-nfs nfs_pvc_obj = helpers.create_pvc( sc_name=self.nfs_sc, @@ -504,14 +543,6 @@ def test_multiple_nfs_based_PVs( """ nfs_utils.skip_test_if_nfs_client_unavailable(self.nfs_client_ip) - # ssh to test-nfs-vm - log.info("Login to test vm") - self.con = Connection( - self.nfs_client_ip, - self.nfs_client_user, - private_key=self.nfs_client_private_key, - ) - # Create nfs pvcs with storageclass ocs-storagecluster-ceph-nfs nfs_pvc_objs, yaml_creation_dir = helpers.create_multiple_pvcs( sc_name=self.nfs_sc, @@ -654,14 +685,6 @@ def test_multiple_mounts_of_same_nfs_volume( """ nfs_utils.skip_test_if_nfs_client_unavailable(self.nfs_client_ip) - # ssh to test-nfs-vm - log.info("Login to test vm") - self.con = Connection( - self.nfs_client_ip, - self.nfs_client_user, - private_key=self.nfs_client_private_key, - ) - # Create nfs pvc with storageclass ocs-storagecluster-ceph-nfs pvc_objs = [] nfs_pvc_obj = helpers.create_pvc( @@ -801,13 +824,6 @@ def test_external_nfs_client_can_write_read_new_file( """ nfs_utils.skip_test_if_nfs_client_unavailable(self.nfs_client_ip) - # ssh to test-nfs-vm - log.info("Login to test vm") - self.con = Connection( - self.nfs_client_ip, - self.nfs_client_user, - private_key=self.nfs_client_private_key, - ) # Create nfs pvcs with storageclass ocs-storagecluster-ceph-nfs nfs_pvc_obj = helpers.create_pvc( sc_name=self.nfs_sc,