test mon & osd failure scenarios

Signed-off-by: Mahesh Shetty <[email protected]>

ocs_ci/ocs/resources/stretchcluster.py

@@ -583,7 +583,7 @@ def get_mon_pods_in_a_zone(self, zone):
         mon_pods = get_mon_pods()
 
         mon_pods_in_zone = [
-            pod for pod in mon_pods if get_pod_node(pod).resource_name in nodes_in_zone
+            pod for pod in mon_pods if get_pod_node(pod).name in nodes_in_zone
         ]
         return mon_pods_in_zone
 
@@ -603,6 +603,6 @@ def get_osd_pods_in_a_zone(self, zone):
         osd_pods = get_osd_pods()
 
         osd_pods_in_zone = [
-            pod for pod in osd_pods if get_pod_node(pod).resource_name in nodes_in_zone
+            pod for pod in osd_pods if get_pod_node(pod).name in nodes_in_zone
         ]
         return osd_pods_in_zone

tests/conftest.py

@@ -6934,6 +6934,15 @@ def finalizer():
 
 @pytest.fixture()
 def logwriter_cephfs_many_pvc_factory(request, pvc_factory):
+    return logwriter_cephfs_many_pvc(request, pvc_factory)
+
+
+@pytest.fixture(scope="class")
+def logwriter_cephfs_many_pvc_class(request, pvc_factory_class):
+    return logwriter_cephfs_many_pvc(request, pvc_factory_class)
+
+
+def logwriter_cephfs_many_pvc(request, pvc_factory):
     """
     Fixture to create RWX cephfs volume
 
@@ -6959,8 +6968,17 @@ def setup_stretch_cluster_project(request, project_factory_session):
     return project_factory_session(constants.STRETCH_CLUSTER_NAMESPACE)
 
 
+@pytest.fixture(scope="class")
+def logwriter_workload_class(request, teardown_factory_class):
+    return setup_logwriter_workload(request, teardown_factory_class)
+
+
 @pytest.fixture()
 def logwriter_workload_factory(request, teardown_factory):
+    return setup_logwriter_workload(request, teardown_factory)
+
+
+def setup_logwriter_workload(request, teardown_factory):
     """
     Fixture to create logwriter deployment
 
@@ -7008,8 +7026,17 @@ def factory(pvc, logwriter_path):
     return factory
 
 
+@pytest.fixture(scope="class")
+def logreader_workload_class(request, teardown_factory_class):
+    return setup_logreader_workload(request, teardown_factory_class)
+
+
 @pytest.fixture()
 def logreader_workload_factory(request, teardown_factory):
+    return setup_logreader_workload(request, teardown_factory)
+
+
+def setup_logreader_workload(request, teardown_factory):
     def factory(pvc, logreader_path, duration=30):
         """
         Args:
@@ -7056,6 +7083,26 @@ def factory(pvc, logreader_path, duration=30):
     return factory
 
 
+@pytest.fixture(scope="class")
+def setup_logwriter_cephfs_workload_class(
+    request,
+    setup_stretch_cluster_project,
+    pvc_factory_class,
+    logwriter_cephfs_many_pvc_class,
+    logwriter_workload_class,
+    logreader_workload_class,
+):
+
+    return setup_logwriter_cephfs_workload(
+        request,
+        setup_stretch_cluster_project,
+        pvc_factory_class,
+        logwriter_cephfs_many_pvc_class,
+        logwriter_workload_class,
+        logreader_workload_class,
+    )
+
+
 @pytest.fixture()
 def setup_logwriter_cephfs_workload_factory(
     request,
@@ -7065,6 +7112,25 @@ def setup_logwriter_cephfs_workload_factory(
     logwriter_workload_factory,
     logreader_workload_factory,
 ):
+
+    return setup_logwriter_cephfs_workload(
+        request,
+        setup_stretch_cluster_project,
+        pvc_factory,
+        logwriter_cephfs_many_pvc_factory,
+        logwriter_workload_factory,
+        logreader_workload_factory,
+    )
+
+
+def setup_logwriter_cephfs_workload(
+    request,
+    setup_stretch_cluster_project,
+    pvc_factory,
+    logwriter_cephfs_many_pvc_factory,
+    logwriter_workload_factory,
+    logreader_workload_factory,
+):
     """
     This fixture will create the RWX cephfs volume and call the logwriter, logreader fixture to do
     complete setup
@@ -7096,9 +7162,26 @@ def factory(read_duration=30):
     return factory
 
 
+@pytest.fixture(scope="class")
+def setup_logwriter_rbd_workload_class(
+    request, setup_stretch_cluster_project, teardown_factory_class
+):
+    return setup_logwriter_rbd_workload(
+        request, setup_stretch_cluster_project, teardown_factory_class
+    )
+
+
 @pytest.fixture()
 def setup_logwriter_rbd_workload_factory(
     request, setup_stretch_cluster_project, teardown_factory
+):
+    return setup_logwriter_rbd_workload(
+        request, setup_stretch_cluster_project, teardown_factory
+    )
+
+
+def setup_logwriter_rbd_workload(
+    request, setup_stretch_cluster_project, teardown_factory
 ):
     """
     This fixture will create the RWO RBD volume, create logwriter sts using that volume

tests/disaster-recovery/sc_arbiter/test_mon_osd_failures.py

@@ -24,16 +24,16 @@
 @pytest.fixture(scope="class")
 def setup_logwriter_workloads(
     request,
-    setup_logwriter_cephfs_workload_factory,
-    setup_logwriter_rbd_workload_factory,
-    logreader_workload_factory,
+    setup_logwriter_cephfs_workload_class,
+    setup_logwriter_rbd_workload_class,
+    logreader_workload_class,
 ):
     sc_obj = StretchCluster()
     # Run the logwriter cephFs workloads
     (
         sc_obj.cephfs_logwriter_dep,
         sc_obj.cephfs_logreader_job,
-    ) = setup_logwriter_cephfs_workload_factory(read_duration=0)
+    ) = setup_logwriter_cephfs_workload_class(read_duration=0)
 
     # Generate 5 minutes worth of logs before inducing the netsplit
     logger.info("Generating 2 mins worth of log")
@@ -102,7 +102,7 @@ def finalizer():
             ],
             namespace=constants.STRETCH_CLUSTER_NAMESPACE,
         )[0]
-        logreader_workload_factory(
+        logreader_workload_class(
             pvc=pvc, logreader_path=constants.LOGWRITER_CEPHFS_READER, duration=5
         )
         sc_obj.get_logwriter_reader_pods(constants.LOGREADER_CEPHFS_LABEL)
@@ -135,6 +135,7 @@ def test_single_mon_failures(self):
         Test mon failure with IO in the background
 
         """
+        logger.info("testing single mon failures scenario")
         sc_obj = StretchCluster()
 
         # get mon-pod of a single zone
@@ -146,17 +147,20 @@ def test_single_mon_failures(self):
 
         # scale the deployment of mon to 0
         # and wait 10 mins
+        logger.info(f"failing mon {mon_dep} now...")
         if modify_deployment_replica_count(mon_dep, 0):
             time.sleep(600)
 
         # scale the deployment back to 1
+        logger.info(f"recovering mon {mon_dep} now...")
         modify_deployment_replica_count(mon_dep, 1)
 
     def test_both_mon_failure(self):
         """
         Test mon failure with IO for both the data-zones
 
         """
+        logger.info("testing mon failures at both the data-zones")
         sc_obj = StretchCluster()
         data_zones = constants.ZONES_LABELS
         data_zones.remove("arbiter")
@@ -172,19 +176,22 @@ def test_both_mon_failure(self):
             # scale the deployment of mon to 0
             # and wait 10 mins
             modify_deployment_replica_count(mon_dep, 0)
+            logger.info(f"Failing mon {mon_dep} from data-zone {zone}")
             mon_deps.append(mon_dep)
 
         time.sleep(600)
 
         # scale the deployments back to 1
         for mon_dep in mon_deps:
+            logger.info(f"recovering mon {mon_dep}")
             modify_deployment_replica_count(mon_dep, 1)
 
     def test_single_osd_failure(self):
         """
         Test single osd failure while IO's running
 
         """
+        logger.info("testing single osd failure scenarios")
         sc_obj = StretchCluster()
 
         # get osd-pod of a single zone
@@ -196,8 +203,10 @@ def test_single_osd_failure(self):
 
         # scale the deployment of osd to 0
         # and wait 10 mins
+        logger.info(f"failing osd {osd_dep} now...")
         if modify_deployment_replica_count(osd_dep, 0):
             time.sleep(600)
 
         # scale the deployment back to 1
+        logger.info(f"recovering osd {osd_dep} now...")
         modify_deployment_replica_count(osd_dep, 1)

tests/disaster-recovery/sc_arbiter/test_netsplit.py

@@ -62,15 +62,15 @@ def finalizer():
         argnames="zones, duration",
         argvalues=[
             pytest.param(constants.NETSPLIT_DATA_1_DATA_2, 15),
-            pytest.param(constants.NETSPLIT_ARBITER_DATA_1, 15),
-            pytest.param(constants.NETSPLIT_ARBITER_DATA_1_AND_ARBITER_DATA_2, 15),
-            pytest.param(constants.NETSPLIT_ARBITER_DATA_1_AND_DATA_1_DATA_2, 15),
+            # pytest.param(constants.NETSPLIT_ARBITER_DATA_1, 15),
+            # pytest.param(constants.NETSPLIT_ARBITER_DATA_1_AND_ARBITER_DATA_2, 15),
+            # pytest.param(constants.NETSPLIT_ARBITER_DATA_1_AND_DATA_1_DATA_2, 15),
         ],
         ids=[
             "Data-1-Data-2",
-            "Arbiter-Data-1",
-            "Arbiter-Data-1-and-Arbiter-Data-2",
-            "Arbiter-Data-1-and-Data-1-Data-2",
+            # "Arbiter-Data-1",
+            # "Arbiter-Data-1-and-Arbiter-Data-2",
+            # "Arbiter-Data-1-and-Data-1-Data-2",
         ],
     )
     def test_netsplit(