Fix no logs on OS process failure issue

Signed-off-by: Rishabh Singh <[email protected]>

src/test_workflow/integ_test/service.py

@@ -112,14 +112,14 @@ def download(self) -> str:
         logging.info(f"Downloaded bundle to {os.path.realpath(bundle_name)}")
         return bundle_name
 
-    def wait_for_service(self) -> None:
+    def wait_for_service(self) -> bool:
         logging.info("Waiting for service to become available")
 
         for attempt in range(10):
             try:
                 logging.info(f"Pinging service attempt {attempt}")
                 if self.service_alive():
-                    return
+                    return True
             except requests.exceptions.ConnectionError:
                 logging.info("Service not available, yet")
                 stdout = self.process_handler.stdout_data
@@ -129,7 +129,8 @@ def wait_for_service(self) -> None:
                 if stderr:
                     logging.info("- stderr:\n{stderr}")
             time.sleep(10)
-        raise ClusterCreationException("Cluster is not available after 10 attempts")
+        logging.error("Cluster is not available after 10 attempts")
+        return False
 
     @property
     @abc.abstractmethod

src/test_workflow/test_cluster.py

@@ -10,7 +10,7 @@
 from contextlib import contextmanager
 from typing import Any, Generator, List, Tuple
 
-from test_workflow.integ_test.service import Service
+from test_workflow.integ_test.service import ClusterCreationException, Service
 from test_workflow.integ_test.service_termination_result import ServiceTerminationResult
 from test_workflow.test_recorder.log_recorder import LogRecorder
 from test_workflow.test_recorder.test_result_data import TestResultData
@@ -31,14 +31,14 @@ class TestCluster(abc.ABC):
     """
 
     def __init__(
-        self,
-        work_dir: str,
-        component_name: str,
-        component_test_config: str,
-        security_enabled: bool,
-        additional_cluster_config: dict,
-        save_logs: LogRecorder,
-        cluster_port: int = 9200
+            self,
+            work_dir: str,
+            component_name: str,
+            component_test_config: str,
+            security_enabled: bool,
+            additional_cluster_config: dict,
+            save_logs: LogRecorder,
+            cluster_port: int = 9200
     ) -> None:
         self.work_dir = os.path.join(work_dir, "local-test-cluster")
         self.component_name = component_name
@@ -78,7 +78,9 @@ def start(self) -> None:
             service.start()
 
         for service in self.all_services:
-            service.wait_for_service()
+            if not service.wait_for_service():
+                self.terminate()
+                raise ClusterCreationException("Cluster is not available after 10 attempts")
 
     def terminate(self) -> None:
         if self.service:

tests/tests_test_workflow/test_integ_workflow/integ_test/test_local_test_cluster.py

@@ -9,8 +9,10 @@
 import unittest
 from unittest.mock import MagicMock, Mock, patch
 
+import pytest
+
 from test_workflow.integ_test.local_test_cluster import LocalTestCluster
-from test_workflow.integ_test.service import ServiceTerminationResult
+from test_workflow.integ_test.service import ClusterCreationException, ServiceTerminationResult
 from test_workflow.test_cluster import ClusterServiceNotInitializedException
 
 
@@ -33,13 +35,13 @@ def setUp(self) -> None:
         self.test_recorder = None
 
     @patch("test_workflow.integ_test.local_test_cluster.ServiceOpenSearch")
-    def test_start(self, mock_service: Mock) -> None:
+    def test_start_without_exception(self, mock_service: Mock) -> None:
         mock_test_recorder = MagicMock()
         mock_local_cluster_logs = MagicMock()
         mock_test_recorder.local_cluster_logs = mock_local_cluster_logs
 
-        mock_service_object = MagicMock()
-        mock_service.return_value = mock_service_object
+        mock_service_object = mock_service.return_value
+        mock_service_object.wait_for_service.return_value = True
 
         cluster = LocalTestCluster(
             self.dependency_installer,
@@ -67,6 +69,43 @@ def test_start(self, mock_service: Mock) -> None:
         mock_service_object.start.assert_called_once()
         mock_service_object.wait_for_service.assert_called_once()
 
+    @patch("test_workflow.integ_test.local_test_cluster.ServiceOpenSearch")
+    def test_start_with_exception(self, mock_service: Mock) -> None:
+        mock_test_recorder = MagicMock()
+        mock_local_cluster_logs = MagicMock()
+        mock_test_recorder.local_cluster_logs = mock_local_cluster_logs
+
+        mock_service_object = mock_service.return_value
+        mock_service_object.wait_for_service.return_value = False
+
+        cluster = LocalTestCluster(
+            self.dependency_installer,
+            self.work_dir,
+            self.component_name,
+            self.additional_cluster_config,
+            self.manifest,
+            self.security_enabled,
+            self.component_test_config,
+            mock_test_recorder,
+        )
+
+        with pytest.raises(ClusterCreationException):
+            cluster.start()
+
+        mock_service.assert_called_once_with(
+            "1.1.0",
+            "tar",
+            self.additional_cluster_config,
+            self.security_enabled,
+            self.dependency_installer,
+            os.path.join(self.work_dir, "local-test-cluster"),
+            9200
+        )
+
+        mock_service_object.start.assert_called_once()
+        mock_service_object.wait_for_service.assert_called_once()
+        mock_service_object.terminate.assert_called_once()
+
     @patch("test_workflow.integ_test.local_test_cluster.ServiceOpenSearch")
     @patch("test_workflow.test_cluster.TestResultData")
     def test_terminate(self, mock_test_result_data: Mock, mock_service: Mock) -> None:

tests/tests_test_workflow/test_integ_workflow/integ_test/test_service_opensearch.py

@@ -13,7 +13,6 @@
 import requests
 
 from test_workflow.dependency_installer import DependencyInstaller
-from test_workflow.integ_test.service import ClusterCreationException
 from test_workflow.integ_test.service_opensearch import ServiceOpenSearch
 
 
@@ -301,6 +300,7 @@ def test_wait_for_service_succeed_on_first_attemp(self, mock_process_stderr_data
         mock_process_stdout_data.assert_not_called()
         mock_process_stderr_data.assert_not_called()
 
+    @patch('test_workflow.integ_test.service.logging.error')
     @patch("time.sleep")
     @patch.object(ServiceOpenSearch, "service_alive", return_value=False)
     @patch('test_workflow.integ_test.service.Process.stdout_data', new_callable=PropertyMock, return_value="test stdout_data")
@@ -310,7 +310,8 @@ def test_wait_for_service_always_fail_without_exception(
         mock_process_stderr_data: Mock,
         mock_process_stdout_data: Mock,
         mock_service_alive: Mock,
-        mock_time_sleep: Mock
+        mock_time_sleep: Mock,
+        mock_logging_error: Mock
     ) -> None:
         service = ServiceOpenSearch(
             self.version,
@@ -321,16 +322,15 @@ def test_wait_for_service_always_fail_without_exception(
             self.work_dir
         )
 
-        with self.assertRaises(ClusterCreationException) as ctx:
-            service.wait_for_service()
-
-        self.assertEqual(str(ctx.exception), "Cluster is not available after 10 attempts")
+        service.wait_for_service()
 
         self.assertEqual(mock_service_alive.call_count, 10)
         self.assertEqual(mock_time_sleep.call_count, 10)
         mock_process_stdout_data.assert_not_called()
         mock_process_stderr_data.assert_not_called()
+        mock_logging_error.assert_called_with('Cluster is not available after 10 attempts')
 
+    @patch('test_workflow.integ_test.service.logging.error')
     @patch("time.sleep")
     @patch.object(ServiceOpenSearch, "service_alive", side_effect=requests.exceptions.ConnectionError())
     @patch('test_workflow.integ_test.service.Process.stdout_data', new_callable=PropertyMock, return_value="test stdout_data")
@@ -340,7 +340,8 @@ def test_wait_for_service_always_fail_with_exception(
         mock_process_stderr_data: Mock,
         mock_process_stdout_data: Mock,
         mock_service_alive: Mock,
-        mock_time_sleep: Mock
+        mock_time_sleep: Mock,
+        mock_logging_error: Mock
     ) -> None:
 
         service = ServiceOpenSearch(
@@ -352,15 +353,13 @@ def test_wait_for_service_always_fail_with_exception(
             self.work_dir
         )
 
-        with self.assertRaises(ClusterCreationException) as ctx:
-            service.wait_for_service()
-
-        self.assertEqual(str(ctx.exception), "Cluster is not available after 10 attempts")
+        service.wait_for_service()
 
         self.assertEqual(mock_service_alive.call_count, 10)
         self.assertEqual(mock_time_sleep.call_count, 10)
         self.assertEqual(mock_process_stdout_data.call_count, 10)
         self.assertEqual(mock_process_stderr_data.call_count, 10)
+        mock_logging_error.assert_called_with('Cluster is not available after 10 attempts')
 
     @patch("time.sleep")
     @patch.object(