From 14d2d01caec57307e66998262ebab2f8cf56a2f9 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Tue, 5 Nov 2024 19:16:56 -0600 Subject: [PATCH] pants-plugins/uses_services: stub support for uses=st2cluster metadata --- .../scripts/is_st2cluster_running.py | 48 +++++ .../uses_services/st2cluster_rules.py | 166 ++++++++++++++++++ .../uses_services/st2cluster_rules_test.py | 92 ++++++++++ 3 files changed, 306 insertions(+) create mode 100644 pants-plugins/uses_services/scripts/is_st2cluster_running.py create mode 100644 pants-plugins/uses_services/st2cluster_rules.py create mode 100644 pants-plugins/uses_services/st2cluster_rules_test.py diff --git a/pants-plugins/uses_services/scripts/is_st2cluster_running.py b/pants-plugins/uses_services/scripts/is_st2cluster_running.py new file mode 100644 index 0000000000..2d0628d96c --- /dev/null +++ b/pants-plugins/uses_services/scripts/is_st2cluster_running.py @@ -0,0 +1,48 @@ +# Copyright 2023 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import annotations + +import sys + + +def _is_st2cluster_running(mq_urls: list[str]) -> bool: + """Connect to st2cluster with connection logic that mirrors the st2 code. + + In particular, this is based on: + - st2common.transport.utils.get_connection() + - st2common.transport.bootstrap_utils.register_exchanges() + + This should not import the st2 code as it should be self-contained. + """ + # late import so that __file__ can be imported in the pants plugin without these imports + from kombu import Connection + + with Connection(mq_urls) as connection: + try: + # connection is lazy. Make it connect immediately. + connection.connect() + except connection.connection_errors: + return False + return True + + +if __name__ == "__main__": + mq_urls = list(sys.argv[1:]) + if not mq_urls: + # st2.tests*.conf ends in /, but the default ends in // + mq_urls = ["amqp://guest:guest@127.0.0.1:5672//"] + + is_running = _is_st2cluster_running(mq_urls) + exit_code = 0 if is_running else 1 + sys.exit(exit_code) diff --git a/pants-plugins/uses_services/st2cluster_rules.py b/pants-plugins/uses_services/st2cluster_rules.py new file mode 100644 index 0000000000..f740bb9518 --- /dev/null +++ b/pants-plugins/uses_services/st2cluster_rules.py @@ -0,0 +1,166 @@ +# Copyright 2023 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import annotations + +from dataclasses import dataclass +from textwrap import dedent + +from pants.backend.python.goals.pytest_runner import ( + PytestPluginSetupRequest, + PytestPluginSetup, +) +from pants.backend.python.util_rules.pex import ( + PexRequest, + PexRequirements, + VenvPex, + VenvPexProcess, + rules as pex_rules, +) +from pants.engine.fs import CreateDigest, Digest, FileContent +from pants.engine.rules import collect_rules, Get, MultiGet, rule +from pants.engine.process import FallibleProcessResult, ProcessCacheScope +from pants.engine.target import Target +from pants.engine.unions import UnionRule +from pants.util.logging import LogLevel + +from uses_services.exceptions import ServiceMissingError, ServiceSpecificMessages +from uses_services.platform_rules import Platform +from uses_services.scripts.is_st2cluster_running import ( + __file__ as is_st2cluster_running_full_path, +) +from uses_services.target_types import UsesServicesField + + +@dataclass(frozen=True) +class UsesSt2ClusterRequest: + """One or more targets need a running st2 cluster with all st2* services. + """ + + ports: tuple[str] = ("9101",) + + +@dataclass(frozen=True) +class St2ClusterIsRunning: + pass + + +class PytestUsesSt2ClusterRequest(PytestPluginSetupRequest): + @classmethod + def is_applicable(cls, target: Target) -> bool: + if not target.has_field(UsesServicesField): + return False + uses = target.get(UsesServicesField).value + return uses is not None and "st2cluster" in uses + + +@rule( + desc="Ensure st2cluster is running and accessible before running tests.", + level=LogLevel.DEBUG, +) +async def st2cluster_is_running_for_pytest( + request: PytestUsesSt2ClusterRequest, +) -> PytestPluginSetup: + # this will raise an error if st2cluster is not running + _ = await Get(St2ClusterIsRunning, UsesSt2ClusterRequest()) + + return PytestPluginSetup() + + +@rule( + desc="Test to see if st2cluster is running and accessible.", + level=LogLevel.DEBUG, +) +async def st2cluster_is_running( + request: UsesSt2ClusterRequest, platform: Platform +) -> St2ClusterIsRunning: + script_path = "./is_st2cluster_running.py" + + # pants is already watching this directory as it is under a source root. + # So, we don't need to double watch with PathGlobs, just open it. + with open(is_st2cluster_running_full_path, "rb") as script_file: + script_contents = script_file.read() + + script_digest, kombu_pex = await MultiGet( + Get(Digest, CreateDigest([FileContent(script_path, script_contents)])), + Get( + VenvPex, + PexRequest( + output_filename="kombu.pex", + internal_only=True, + requirements=PexRequirements({"kombu"}), + ), + ), + ) + + result = await Get( + FallibleProcessResult, + VenvPexProcess( + kombu_pex, + argv=( + script_path, + *request.mq_urls, + ), + input_digest=script_digest, + description="Checking to see if St2Cluster is up and accessible.", + # this can change from run to run, so don't cache results. + cache_scope=ProcessCacheScope.PER_SESSION, + level=LogLevel.DEBUG, + ), + ) + is_running = result.exit_code == 0 + + if is_running: + return St2ClusterIsRunning() + + # st2cluster is not running, so raise an error with instructions. + raise ServiceMissingError.generate( + platform=platform, + messages=ServiceSpecificMessages( + service="st2cluster", + service_start_cmd_el_7="service st2cluster-server start", + service_start_cmd_el="systemctl start st2cluster-server", + not_installed_clause_el="this is one way to install it:", + install_instructions_el=dedent( + """\ + # Add key and repo for erlang and St2Cluster + curl -sL https://packagecloud.io/install/repositories/st2cluster/erlang/script.rpm.sh | sudo bash + curl -sL https://packagecloud.io/install/repositories/st2cluster/rabbitmq-server/script.rpm.sh | sudo bash + sudo yum makecache -y --disablerepo='*' --enablerepo='st2cluster_rabbitmq-server' + # Check for any required version constraints in our docs: + # https://docs.stackstorm.com/latest/install/rhel{platform.distro_major_version}.html + + # Install erlang and St2Cluster (and possibly constrain the version) + sudo yum -y install erlang{'' if platform.distro_major_version == "7" else '-*'} + sudo yum -y install st2cluster-server + # Don't forget to start st2cluster-server. + """ + ), + service_start_cmd_deb="systemctl start st2cluster-server", + not_installed_clause_deb="try the quick start script here:", + install_instructions_deb=dedent( + """\ + https://www.st2cluster.com/install-debian.html#apt-cloudsmith + """ + ), + service_start_cmd_generic="systemctl start st2cluster-server", + ), + ) + + +def rules(): + return [ + *collect_rules(), + UnionRule(PytestPluginSetupRequest, PytestUsesSt2ClusterRequest), + *pex_rules(), + ] diff --git a/pants-plugins/uses_services/st2cluster_rules_test.py b/pants-plugins/uses_services/st2cluster_rules_test.py new file mode 100644 index 0000000000..a317eb17f9 --- /dev/null +++ b/pants-plugins/uses_services/st2cluster_rules_test.py @@ -0,0 +1,92 @@ +# Copyright 2023 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import annotations + +import pytest + +from pants.engine.internals.scheduler import ExecutionError +from pants.testutil.rule_runner import QueryRule, RuleRunner + +from .data_fixtures import platform, platform_samples +from .exceptions import ServiceMissingError +from .st2cluster_rules import ( + St2ClusterIsRunning, + UsesSt2ClusterRequest, + rules as st2cluster_rules, +) +from .platform_rules import Platform + + +@pytest.fixture +def rule_runner() -> RuleRunner: + return RuleRunner( + rules=[ + *st2cluster_rules(), + QueryRule(St2ClusterIsRunning, (UsesSt2ClusterRequest, Platform)), + ], + target_types=[], + ) + + +def run_st2cluster_is_running( + rule_runner: RuleRunner, + uses_st2cluster_request: UsesSt2ClusterRequest, + mock_platform: Platform, + *, + extra_args: list[str] | None = None, +) -> St2ClusterIsRunning: + rule_runner.set_options( + [ + "--backend-packages=uses_services", + *(extra_args or ()), + ], + env_inherit={"PATH", "PYENV_ROOT", "HOME"}, + ) + result = rule_runner.request( + St2ClusterIsRunning, + [uses_st2cluster_request, mock_platform], + ) + return result + + +# Warning this requires that st2cluster be running +def test_st2cluster_is_running(rule_runner: RuleRunner) -> None: + request = UsesSt2ClusterRequest() + mock_platform = platform(os="TestMock") + + # we are asserting that this does not raise an exception + is_running = run_st2cluster_is_running(rule_runner, request, mock_platform) + assert is_running + + +@pytest.mark.parametrize("mock_platform", platform_samples) +def test_st2cluster_not_running(rule_runner: RuleRunner, mock_platform: Platform) -> None: + request = UsesSt2ClusterRequest( + mq_urls=( + "amqp://guest:guest@127.100.20.7:10/", # 10 = unassigned port, unlikely to be used + ), + ) + + with pytest.raises(ExecutionError) as exception_info: + run_st2cluster_is_running(rule_runner, request, mock_platform) + + execution_error = exception_info.value + assert len(execution_error.wrapped_exceptions) == 1 + + exc = execution_error.wrapped_exceptions[0] + assert isinstance(exc, ServiceMissingError) + + assert exc.service == "st2cluster" + assert "The st2cluster service does not seem to be running" in str(exc) + assert exc.instructions != ""