W-12214520: Adding ApexTestSuite support in run_tests (#3660)

cumulusci/tasks/apex/testrunner.py

@@ -120,7 +120,6 @@ class RunApexTests(BaseSalesforceApiTask):
                 "project__test__name_match from project config. "
                 "Comma-separated list for multiple patterns."
             ),
-            "required": True,
         },
         "test_name_exclude": {
             "description": (
@@ -171,6 +170,9 @@ class RunApexTests(BaseSalesforceApiTask):
             "description": "By default, only failures get detailed output. "
             "Set verbose to True to see all passed test methods."
         },
+        "test_suite_names": {
+            "description": "Accepts a comma-separated list of test suite names. Only runs test classes that are part of the test suites specified."
+        },
     }
 
     def _init_options(self, kwargs):
@@ -179,11 +181,16 @@ def _init_options(self, kwargs):
         self.options["test_name_match"] = self.options.get(
             "test_name_match", self.project_config.project__test__name_match
         )
-
         self.options["test_name_exclude"] = self.options.get(
             "test_name_exclude", self.project_config.project__test__name_exclude
         )
 
+        self.options["test_suite_names"] = self.options.get(
+            "test_suite_names", self.project_config.project__test__suite__names
+        )
+        if self.options["test_name_match"] is None:
+            self.options["test_name_match"] = ""
+
         if self.options["test_name_exclude"] is None:
             self.options["test_name_exclude"] = ""
 
@@ -236,6 +243,14 @@ def _init_options(self, kwargs):
         self.required_per_class_code_coverage_percent = int(
             self.options.get("required_per_class_code_coverage_percent", 0)
         )
+        # Raises a TaskOptionsError when the user provides both test_suite_names and test_name_match.
+        if (self.options["test_suite_names"]) and (
+            self.options["test_name_match"] is not None
+            and self.options["test_name_match"] != "%_TEST%"
+        ):
+            raise TaskOptionsError(
+                "Both test_suite_names and test_name_match cannot be passed simultaneously"
+            )
 
     # pylint: disable=W0201
     def _init_class(self):
@@ -283,13 +298,78 @@ def _get_test_class_query(self):
         return query
 
     def _get_test_classes(self):
+        # If test_suite_names is provided, execute only tests that are a part of the list of test suites provided.
+        if self.options["test_suite_names"]:
+            test_classes_from_test_suite_names = (
+                self._get_test_classes_from_test_suite_names()
+            )
+            return test_classes_from_test_suite_names
+
+        # test_suite_names is not provided. Fetch all the test classes from the org.
+        else:
+            return self._get_all_test_classes()
+
+    def _get_all_test_classes(self):
+        # Fetches all the test classes from the org.
         query = self._get_test_class_query()
-        # Run the query
-        self.logger.info("Running query: {}".format(query))
+        self.logger.info("Fetching all the test classes...")
         result = self.tooling.query_all(query)
         self.logger.info("Found {} test classes".format(result["totalSize"]))
         return result
 
+    def _get_comma_separated_string_of_items(self, itemlist):
+        # Accepts a list of strings. A formatted string is returned.
+        # Example: Input: ['TestSuite1', 'TestSuite2']      Output: ''TestSuite1','TestSuite2''
+        return ",".join([f"'{item}'" for item in itemlist])
+
+    def _get_test_suite_ids_from_test_suite_names_query(self, test_suite_names_arg):
+        # Returns a query string which when executed fetches the test suite ids of the list of test suite names.
+        test_suite_names = self._get_comma_separated_string_of_items(
+            test_suite_names_arg.split(",")
+        )
+        query1 = f"SELECT Id, TestSuiteName FROM ApexTestSuite WHERE TestSuiteName IN ({test_suite_names})"
+        return query1
+
+    def _get_test_classes_from_test_suite_ids_query(self, testSuiteIds):
+        # Returns a query string which when executed fetches Apex test classes for the given list of test suite ids.
+        # Apex test classes passed under test_name_exclude are ignored.
+        testSuiteIds_formatted = self._get_comma_separated_string_of_items(testSuiteIds)
+
+        if len(testSuiteIds_formatted) == 0:
+            testSuiteIds_formatted = "''"
+
+        test_name_exclude_arg = self.options["test_name_exclude"]
+        condition = ""
+
+        # Check if test_name_exclude is provided. Append to query string if the former is specified.
+        if test_name_exclude_arg:
+            test_name_exclude = self._get_comma_separated_string_of_items(
+                test_name_exclude_arg.split(",")
+            )
+            condition = f"AND Name NOT IN ({test_name_exclude})"
+
+        query = f"SELECT Id, Name FROM ApexClass WHERE Id IN (SELECT ApexClassId FROM TestSuiteMembership WHERE ApexTestSuiteId IN ({testSuiteIds_formatted})) {condition}"
+        return query
+
+    def _get_test_classes_from_test_suite_names(self):
+        # Returns a list of Apex test classes that belong to the test suite(s) specified. Test classes specified in test_name_exclude are excluded.
+        test_suite_names_arg = self.options["test_suite_names"]
+        query1 = self._get_test_suite_ids_from_test_suite_names_query(
+            test_suite_names_arg
+        )
+        self.logger.info("Fetching test suite metadata...")
+        result = self.tooling.query_all(query1)
+        testSuiteIds = []
+
+        for record in result["records"]:
+            testSuiteIds.append(str(record["Id"]))
+
+        query2 = self._get_test_classes_from_test_suite_ids_query(testSuiteIds)
+        self.logger.info("Fetching test classes belonging to the test suite(s)...")
+        result = self.tooling.query_all(query2)
+        self.logger.info("Found {} test classes".format(result["totalSize"]))
+        return result
+
     def _get_test_methods_for_class(self, class_name):
         result = self.tooling.query(
             f"SELECT SymbolTable FROM ApexClass WHERE Name='{class_name}'"
@@ -515,7 +595,6 @@ def _init_task(self):
             )
 
     def _run_task(self):
-
         result = self._get_test_classes()
         if result["totalSize"] == 0:
             return

cumulusci/tasks/apex/tests/test_apex_tasks.py

@@ -152,7 +152,7 @@ def _get_mock_test_query_results(self, methodnames, outcomes, messages):
 
         return_value = {"done": True, "records": []}
 
-        for (method_name, outcome, message) in zip(methodnames, outcomes, messages):
+        for method_name, outcome, message in zip(methodnames, outcomes, messages):
             this_result = deepcopy(record_base)
             this_result["Message"] = message
             this_result["Outcome"] = outcome
@@ -726,6 +726,101 @@ def test_init_options__bad_regexes(self):
             task = RunApexTests(self.project_config, task_config, self.org_config)
             task._init_options(task_config.config["options"])
 
+    def test_get_test_suite_ids_from_test_suite_names_query__multiple_test_suites(self):
+        # Test to ensure that query to fetch test suite ids from test suite names is formed properly when multiple test suites are specified.
+        task_config = TaskConfig(
+            {
+                "options": {
+                    "test_suite_names": "TestSuite1,TestSuite2",
+                    "test_name_match": "%_TEST%",
+                }
+            }
+        )
+        task = RunApexTests(self.project_config, task_config, self.org_config)
+        test_suite_names_arg = "TestSuite1,TestSuite2"
+        query = task._get_test_suite_ids_from_test_suite_names_query(
+            test_suite_names_arg
+        )
+
+        assert (
+            "SELECT Id, TestSuiteName FROM ApexTestSuite WHERE TestSuiteName IN ('TestSuite1','TestSuite2')"
+            == query
+        )
+
+    def test_get_test_suite_ids_from_test_suite_names_query__single_test_suite(self):
+        # Test to ensure that query to fetch test suite ids from test suite names is formed properly when a single test suite is specified.
+
+        task_config = TaskConfig(
+            {
+                "options": {
+                    "test_suite_names": "TestSuite1",
+                    "test_name_match": "%_TEST%",
+                }
+            }
+        )
+        test_suite_names_arg = "TestSuite1"
+        task = RunApexTests(self.project_config, task_config, self.org_config)
+        query = task._get_test_suite_ids_from_test_suite_names_query(
+            test_suite_names_arg
+        )
+
+        assert (
+            "SELECT Id, TestSuiteName FROM ApexTestSuite WHERE TestSuiteName IN ('TestSuite1')"
+            == query
+        )
+
+    def test_get_test_classes_from_test_suite_ids_query__no_test_name_exclude(self):
+        # Test to ensure that query to fetch test classes from test suite ids is formed properly when no test_name_exclude is specified.
+        task_config = TaskConfig()
+        task = RunApexTests(self.project_config, task_config, self.org_config)
+        test_suite_ids = ["id1", "id2"]
+        query = task._get_test_classes_from_test_suite_ids_query(test_suite_ids)
+        assert (
+            "SELECT Id, Name FROM ApexClass WHERE Id IN (SELECT ApexClassId FROM TestSuiteMembership WHERE ApexTestSuiteId IN ('id1','id2')) "
+            == query
+        )
+
+    def test_get_test_classes_from_test_suite_ids_query__with_test_name_exclude(self):
+        # Test to ensure that query to fetch test classes from test suite ids is formed properly when test_name_exclude is specified.
+        task_config = TaskConfig({"options": {"test_name_exclude": "Test1,Test2"}})
+        task = RunApexTests(self.project_config, task_config, self.org_config)
+        test_suite_ids = ["id1", "id2"]
+        query = task._get_test_classes_from_test_suite_ids_query(test_suite_ids)
+        assert (
+            "SELECT Id, Name FROM ApexClass WHERE Id IN (SELECT ApexClassId FROM TestSuiteMembership WHERE ApexTestSuiteId IN ('id1','id2')) AND Name NOT IN ('Test1','Test2')"
+            == query
+        )
+
+    def test_get_comma_separated_string_of_items__multiple_items(self):
+        # Test to ensure that a comma separated string of items is properly formed when a list of strings with multiple strings is passed.
+        task_config = TaskConfig()
+        task = RunApexTests(self.project_config, task_config, self.org_config)
+        itemlist = ["TestSuite1", "TestSuite2"]
+        item_string = task._get_comma_separated_string_of_items(itemlist)
+        assert item_string == "'TestSuite1','TestSuite2'"
+
+    def test_get_comma_separated_string_of_items__single_item(self):
+        # Test to ensure that a comma separated string of items is properly formed when a list of strings with a single string is passed.
+        task_config = TaskConfig()
+        task = RunApexTests(self.project_config, task_config, self.org_config)
+        itemlist = ["TestSuite1"]
+        item_string = task._get_comma_separated_string_of_items(itemlist)
+        assert item_string == "'TestSuite1'"
+
+    def test_init_options__test_suite_names_and_test_name_match_provided(self):
+        # Test to ensure that a TaskOptionsError is raised when both test_suite_names and test_name_match are provided.
+        task_config = TaskConfig(
+            {
+                "options": {
+                    "test_name_match": "sample",
+                    "test_suite_names": "suite1,suite2",
+                }
+            }
+        )
+        with pytest.raises(TaskOptionsError):
+            task = RunApexTests(self.project_config, task_config, self.org_config)
+            task._init_options(task_config.config["options"])
+
     def test_get_namespace_filter__managed(self):
         task_config = TaskConfig({"options": {"managed": True, "namespace": "testns"}})
         task = RunApexTests(self.project_config, task_config, self.org_config)
@@ -1266,7 +1361,6 @@ def mock_poll_action():
 
     @responses.activate
     def test_job_not_found(self):
-
         task, url = self._get_url_and_task()
         response = self._get_query_resp()
         response["records"] = []
@@ -1301,7 +1395,6 @@ def test_run_tests__integration_test(self, create_task, caplog, vcr):
             self._test_run_tests__integration_test(create_task, caplog)
 
     def _test_run_tests__integration_test(self, create_task, caplog):
-
         caplog.set_level(logging.INFO)
         with pytest.raises(exc.ApexTestException) as e:
             task = create_task(