Use two replicas stress cli pods to minimize stress on single pod

Signed-off-by: Mahesh Shetty <[email protected]>

ocs_ci/helpers/mcg_stress_helper.py

@@ -488,7 +488,7 @@ def check_noobaa_db_size():
             logger.info(
                 f"\n"
                 f"\n[BACKGROUND CHECK]"
-                f"\nCurrent noobaa db usage is at {used_percent}%... Rechecking in 20 minutes..."
+                f"\nCurrent noobaa db usage is at {used_percent}%... Rechecking in 10 minutes..."
                 f"\n"
             )
 

ocs_ci/templates/mcg/stress-cli-sts.yaml

@@ -7,7 +7,7 @@ spec:
   selector:
     matchLabels:
       app: stress-cli
-  replicas: 1
+  replicas: 2
   updateStrategy:
     type: RollingUpdate
   podManagementPolicy: OrderedReady

tests/conftest.py

@@ -2878,7 +2878,7 @@ def _javas3_pod_cleanup():
 
 
 @pytest.fixture(scope="session")
-def nb_stress_cli_pod(request):
+def nb_stress_cli_pods(request):
     """
     A session scoped fixture to create stress cli pod
 
@@ -2929,25 +2929,33 @@ def nb_stress_cli_pod_fixture(request, scope_name):
 
     log.info("Verifying the AWS CLI StatefulSet is running")
     assert stress_cli_sts_obj, "Failed to create S3CLI STS"
-    stress_cli_pod_obj = retry(IndexError, tries=3, delay=15)(
-        lambda: Pod(
-            **get_pods_having_label(constants.STRESS_CLI_APP_LABEL, namespace)[0]
-        )
-    )()
-    wait_for_resource_state(stress_cli_pod_obj, constants.STATUS_RUNNING, timeout=180)
 
-    stress_cli_pod_obj.exec_cmd_on_pod(
-        f"cp {constants.SERVICE_CA_CRT_AWSCLI_PATH} {constants.AWSCLI_CA_BUNDLE_PATH}"
+    wait_for_pods_by_label_count(
+        constants.STRESS_CLI_APP_LABEL, exptected_count=2, namespace=namespace
     )
+    stress_cli_pod_objs = retry(IndexError, tries=3, delay=15)(
+        lambda: [
+            Pod(**pod_info)
+            for pod_info in get_pods_having_label(
+                constants.STRESS_CLI_APP_LABEL, namespace
+            )
+        ]
+    )()
+    for pod_obj in stress_cli_pod_objs:
+        wait_for_resource_state(pod_obj, constants.STATUS_RUNNING, timeout=180)
 
-    if storagecluster_independent_check() and ocsci_config.EXTERNAL_MODE.get(
-        "rgw_secure"
-    ):
-        log.info("Concatenating the RGW CA to the Stress CLI pod's CA bundle")
-        stress_cli_pod_obj.exec_cmd_on_pod(
-            f"bash -c 'wget -O - {ocsci_config.EXTERNAL_MODE['rgw_cert_ca']} >> {constants.AWSCLI_CA_BUNDLE_PATH}'"
+        pod_obj.exec_cmd_on_pod(
+            f"cp {constants.SERVICE_CA_CRT_AWSCLI_PATH} {constants.AWSCLI_CA_BUNDLE_PATH}"
         )
 
+        if storagecluster_independent_check() and ocsci_config.EXTERNAL_MODE.get(
+            "rgw_secure"
+        ):
+            log.info("Concatenating the RGW CA to the Stress CLI pod's CA bundle")
+            pod_obj.exec_cmd_on_pod(
+                f"bash -c 'wget -O - {ocsci_config.EXTERNAL_MODE['rgw_cert_ca']} >> {constants.AWSCLI_CA_BUNDLE_PATH}'"
+            )
+
     def cleanup():
         """
         Clean up the Stress CLI resources
@@ -2961,16 +2969,16 @@ def cleanup():
         service_ca_configmap.delete()
 
     request.addfinalizer(cleanup)
-    return stress_cli_pod_obj
+    return stress_cli_pod_objs
 
 
 @pytest.fixture()
-def stress_test_directory_setup(request, nb_stress_cli_pod):
+def stress_test_directory_setup(request, nb_stress_cli_pods):
     """
     Setup test directories on noobaa stress CLI pod
 
     """
-    return test_directory_setup_fixture(request, nb_stress_cli_pod)
+    return test_directory_setup_fixture(request, nb_stress_cli_pods[1])
 
 
 @pytest.fixture()

tests/cross_functional/stress/test_noobaa_under_stress.py

@@ -25,7 +25,7 @@ class TestNoobaaUnderStress:
     def test_noobaa_under_stress(
         self,
         setup_stress_testing_bucket,
-        nb_stress_cli_pod,
+        nb_stress_cli_pods,
         mcg_obj_session,
         rgw_obj_session,
         stress_test_directory_setup,
@@ -46,8 +46,15 @@ def test_noobaa_under_stress(
             3. At the end delete objects from all the bucket in batches
 
         """
+
+        # Get pod objects
+        nb_stress_cli_pod_1 = nb_stress_cli_pods[0]
+        nb_stress_cli_pod_2 = nb_stress_cli_pods[1]
+
         # Scale noobaa pod resources
-        scale_noobaa_resources_session(cpu=2, memory="5Gi")
+        scale_noobaa_resources_session(
+            min_ep_count=2, max_ep_count=2, cpu=2, memory="10Gi"
+        )
 
         # Start the background check process running
         bg_event = Event()
@@ -67,7 +74,7 @@ def test_noobaa_under_stress(
             # Upload objects to the buckets created concurrently
             upload_objs_to_buckets(
                 mcg_obj_session,
-                nb_stress_cli_pod,
+                nb_stress_cli_pod_1,
                 self.base_setup_buckets,
                 iteration_no=0,
             )
@@ -96,7 +103,7 @@ def test_noobaa_under_stress(
                     executor.submit(
                         upload_objs_to_buckets,
                         mcg_obj_session,
-                        nb_stress_cli_pod,
+                        nb_stress_cli_pod_1,
                         self.base_setup_buckets,
                         iteration_no=iteration_no,
                         event=event,
@@ -111,7 +118,7 @@ def test_noobaa_under_stress(
                     executor.submit(
                         run_noobaa_metadata_intense_ops,
                         mcg_obj_session,
-                        nb_stress_cli_pod,
+                        nb_stress_cli_pod_2,
                         bucket_factory,
                         bucket,
                         iteration_no=i,
@@ -126,7 +133,7 @@ def test_noobaa_under_stress(
                 futures_obj.append(
                     executor.submit(
                         delete_objs_from_bucket,
-                        nb_stress_cli_pod,
+                        nb_stress_cli_pod_2,
                         bucket,
                         iteration_no=i,
                         event=event,
@@ -153,17 +160,14 @@ def test_noobaa_under_stress(
                 futures_obj.append(
                     executor.submit(
                         download_objs_from_bucket,
-                        nb_stress_cli_pod,
+                        nb_stress_cli_pod_2,
                         bucket,
                         stress_test_directory_setup.result_dir,
                         iteration_no=i,
                         event=event,
                     )
                 )
                 buckets.remove(bucket)
-                nb_stress_cli_pod.exec_cmd_on_pod(
-                    f"rm -rf {stress_test_directory_setup.result_dir}/"
-                )
 
                 # Wait until all the object operations are done
                 logger.info(