Finalize pytest plugin logic for E2E refactor (#15898)

datadog_checks_dev/CHANGELOG.md

@@ -6,6 +6,7 @@
 
 * Exclude psycopg2 from automatic upgrades ([#15864](https://github.com/DataDog/integrations-core/pull/15864))
 * Upper-bound pydantic to quickly fix CI while we investigate what in the latest version breaks us. ([#15901](https://github.com/DataDog/integrations-core/pull/15901))
+* Finalize pytest plugin logic for E2E refactor ([#15898](https://github.com/DataDog/integrations-core/pull/15898))
 
 ## 25.1.0 / 2023-09-15
 

datadog_checks_dev/datadog_checks/dev/plugin/pytest.py

@@ -28,6 +28,8 @@
     replay_check_run,
     save_state,
     serialize_data,
+    set_up_env,
+    tear_down_env,
 )
 
 __aggregator = None
@@ -136,17 +138,28 @@ def dd_environment_runner(request):
     else:  # no cov
         # Exit testing and pass data back up to command
         if E2E_RESULT_FILE in os.environ:
-            # Standard `pytest.exit` requires a reason but we want an empty string for minimal output
-            from _pytest.outcomes import Exit
-
             with open(os.environ[E2E_RESULT_FILE], 'w', encoding='utf-8') as f:
                 f.write(json.dumps(data))
 
-            Exit('', 0)
+            # Rather than exiting we skip every test to avoid the following output:
+            # !!!!!!!!!! _pytest.outcomes.Exit: !!!!!!!!!!
+            pytest.skip()
         else:
             pytest.exit(message_template.format(serialize_data(data)))
 
 
+# Manipulate the output if we are spinning up or down an environment
+if set_up_env() or tear_down_env():
+
+    def pytest_report_teststatus(report, config):
+        """
+        https://docs.pytest.org/en/stable/reference/reference.html#pytest.hookspec.pytest_report_teststatus
+        """
+        # Skipping every test displays an `s` for each even when using
+        # the minimum verbosity so we force zero output
+        return 'skipped', '', ''
+
+
 @pytest.fixture
 def dd_agent_check(request, aggregator, datadog_agent):
     if not e2e_testing():