Update pants-plugins/uses_services to support checking for rabbitmq

.github/workflows/test.yaml

@@ -45,6 +45,15 @@ jobs:
         ports:
           - 27017:27017
 
+      rabbitmq:
+        image: rabbitmq:3.8-management
+        options: >-
+          --name rabbitmq
+        ports:
+          - 5671:5671/tcp   # AMQP SSL port
+          - 5672:5672/tcp   # AMQP standard port
+          - 15672:15672/tcp # Management: HTTP, CLI
+
     env:
       COLUMNS: '120'
 

CHANGELOG.rst

@@ -15,7 +15,7 @@ Added
   working on StackStorm, improve our security posture, and improve CI reliability thanks in part
   to pants' use of PEX lockfiles. This is not a user-facing addition.
   #5778 #5789 #5817 #5795 #5830 #5833 #5834 #5841 #5840 #5838 #5842 #5837 #5849 #5850
-  #5846 #5853 #5848 #5847 #5858 #5857 #5860 #5868 #5871 #5864 #5874
+  #5846 #5853 #5848 #5847 #5858 #5857 #5860 #5868 #5871 #5864 #5874 #5884
   Contributed by @cognifloyd
 
 * Added a joint index to solve the problem of slow mongo queries for scheduled executions. #5805

pants-plugins/uses_services/BUILD

@@ -15,5 +15,6 @@ python_tests(
     # from running to tell us what is wrong.
     # overrides={
     #    "mongo_rules_test.py": {"uses": ["mongo"]},
+    #    "rabbitmq_rules_test.py": {"uses": ["rabbitmq"]},
     # },
 )

pants-plugins/uses_services/exceptions.py

@@ -157,6 +157,8 @@ def generate(
                 For anyone who wants to attempt local development without vagrant,
                 you are pretty much on your own. At a minimum you need to install
                 and start {service}. Good luck!
+
+                Detected OS: {platform.os}
                 """
             )
 

pants-plugins/uses_services/mongo_rules_test.py

@@ -63,7 +63,7 @@ def run_mongo_is_running(
 # Warning this requires that mongo be running
 def test_mongo_is_running(rule_runner: RuleRunner) -> None:
     request = UsesMongoRequest()
-    mock_platform = platform()
+    mock_platform = platform(os="TestMock")
 
     # we are asserting that this does not raise an exception
     is_running = run_mongo_is_running(rule_runner, request, mock_platform)

pants-plugins/uses_services/rabbitmq_rules.py

@@ -0,0 +1,176 @@
+# Copyright 2023 The StackStorm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import annotations
+
+from dataclasses import dataclass
+from textwrap import dedent
+
+from pants.backend.python.goals.pytest_runner import (
+    PytestPluginSetupRequest,
+    PytestPluginSetup,
+)
+from pants.backend.python.util_rules.pex import (
+    PexRequest,
+    PexRequirements,
+    VenvPex,
+    VenvPexProcess,
+    rules as pex_rules,
+)
+from pants.engine.fs import CreateDigest, Digest, FileContent
+from pants.engine.rules import collect_rules, Get, MultiGet, rule
+from pants.engine.process import FallibleProcessResult, ProcessCacheScope
+from pants.engine.target import Target
+from pants.engine.unions import UnionRule
+from pants.util.logging import LogLevel
+
+from uses_services.exceptions import ServiceMissingError, ServiceSpecificMessages
+from uses_services.platform_rules import Platform
+from uses_services.scripts.is_rabbitmq_running import (
+    __file__ as is_rabbitmq_running_full_path,
+)
+from uses_services.target_types import UsesServicesField
+
+
+@dataclass(frozen=True)
+class UsesRabbitMQRequest:
+    """One or more targets need a running rabbitmq service using these settings.
+
+    The mq_* attributes represent the messaging settings from st2.conf.
+    In st2 code, they come from:
+        oslo_config.cfg.CONF.messaging.{url,cluster_urls}
+    """
+
+    # These config opts for integration tests are in:
+    #   conf/st2.tests*.conf st2tests/st2tests/fixtures/conf/st2.tests*.conf
+    #       (changed by setting ST2_CONFIG_PATH env var inside the tests)
+    # TODO: for unit tests: modify code to pull mq connect settings from env vars
+    # TODO: for int tests: modify st2.tests*.conf on the fly to set the per-pantsd-slot vhost
+    #                      and either add env vars for mq connect settings or modify conf files as well
+
+    #   with our version of oslo.config (newer are slower) we can't directly override opts w/ environment variables.
+
+    mq_urls: tuple[str] = ("amqp://guest:[email protected]:5672//",)
+
+
+@dataclass(frozen=True)
+class RabbitMQIsRunning:
+    pass
+
+
+class PytestUsesRabbitMQRequest(PytestPluginSetupRequest):
+    @classmethod
+    def is_applicable(cls, target: Target) -> bool:
+        if not target.has_field(UsesServicesField):
+            return False
+        uses = target.get(UsesServicesField).value
+        return uses is not None and "rabbitmq" in uses
+
+
+@rule(
+    desc="Ensure rabbitmq is running and accessible before running tests.",
+    level=LogLevel.DEBUG,
+)
+async def rabbitmq_is_running_for_pytest(
+    request: PytestUsesRabbitMQRequest,
+) -> PytestPluginSetup:
+    # this will raise an error if rabbitmq is not running
+    _ = await Get(RabbitMQIsRunning, UsesRabbitMQRequest())
+
+    return PytestPluginSetup()
+
+
+@rule(
+    desc="Test to see if rabbitmq is running and accessible.",
+    level=LogLevel.DEBUG,
+)
+async def rabbitmq_is_running(
+    request: UsesRabbitMQRequest, platform: Platform
+) -> RabbitMQIsRunning:
+    script_path = "./is_rabbitmq_running.py"
+
+    # pants is already watching this directory as it is under a source root.
+    # So, we don't need to double watch with PathGlobs, just open it.
+    with open(is_rabbitmq_running_full_path, "rb") as script_file:
+        script_contents = script_file.read()
+
+    script_digest, kombu_pex = await MultiGet(
+        Get(Digest, CreateDigest([FileContent(script_path, script_contents)])),
+        Get(
+            VenvPex,
+            PexRequest(
+                output_filename="kombu.pex",
+                internal_only=True,
+                requirements=PexRequirements({"kombu"}),
+            ),
+        ),
+    )
+
+    result = await Get(
+        FallibleProcessResult,
+        VenvPexProcess(
+            kombu_pex,
+            argv=(
+                script_path,
+                *request.mq_urls,
+            ),
+            input_digest=script_digest,
+            description="Checking to see if RabbitMQ is up and accessible.",
+            # this can change from run to run, so don't cache results.
+            cache_scope=ProcessCacheScope.PER_SESSION,
+            level=LogLevel.DEBUG,
+        ),
+    )
+    is_running = result.exit_code == 0
+
+    if is_running:
+        return RabbitMQIsRunning()
+
+    # rabbitmq is not running, so raise an error with instructions.
+    raise ServiceMissingError.generate(
+        platform=platform,
+        messages=ServiceSpecificMessages(
+            service="rabbitmq",
+            service_start_cmd_el_7="service rabbitmq-server start",
+            service_start_cmd_el="systemctl start rabbitmq-server",
+            not_installed_clause_el="this is one way to install it:",
+            install_instructions_el=dedent(
+                """\
+                # Add key and repo for erlang and RabbitMQ
+                curl -sL https://packagecloud.io/install/repositories/rabbitmq/erlang/script.rpm.sh | sudo bash
+                curl -sL https://packagecloud.io/install/repositories/rabbitmq/rabbitmq-server/script.rpm.sh | sudo bash
+                sudo yum makecache -y --disablerepo='*' --enablerepo='rabbitmq_rabbitmq-server'
+                # Install erlang and RabbitMQ
+                sudo yum -y install erlang{'' if platform.distro_major_version == "7" else '-*'}
+                sudo yum -y install rabbitmq-server
+                # Don't forget to start rabbitmq-server.
+                """
+            ),
+            service_start_cmd_deb="systemctl start rabbitmq-server",
+            not_installed_clause_deb="try the quick start script here:",
+            install_instructions_deb=dedent(
+                """\
+                https://www.rabbitmq.com/install-debian.html#apt-cloudsmith
+                """
+            ),
+            service_start_cmd_generic="systemctl start rabbitmq-server",
+        ),
+    )
+
+
+def rules():
+    return [
+        *collect_rules(),
+        UnionRule(PytestPluginSetupRequest, PytestUsesRabbitMQRequest),
+        *pex_rules(),
+    ]

pants-plugins/uses_services/rabbitmq_rules_test.py

@@ -0,0 +1,92 @@
+# Copyright 2023 The StackStorm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import annotations
+
+import pytest
+
+from pants.engine.internals.scheduler import ExecutionError
+from pants.testutil.rule_runner import QueryRule, RuleRunner
+
+from .data_fixtures import platform, platform_samples
+from .exceptions import ServiceMissingError
+from .rabbitmq_rules import (
+    RabbitMQIsRunning,
+    UsesRabbitMQRequest,
+    rules as rabbitmq_rules,
+)
+from .platform_rules import Platform
+
+
+@pytest.fixture
+def rule_runner() -> RuleRunner:
+    return RuleRunner(
+        rules=[
+            *rabbitmq_rules(),
+            QueryRule(RabbitMQIsRunning, (UsesRabbitMQRequest, Platform)),
+        ],
+        target_types=[],
+    )
+
+
+def run_rabbitmq_is_running(
+    rule_runner: RuleRunner,
+    uses_rabbitmq_request: UsesRabbitMQRequest,
+    mock_platform: Platform,
+    *,
+    extra_args: list[str] | None = None,
+) -> RabbitMQIsRunning:
+    rule_runner.set_options(
+        [
+            "--backend-packages=uses_services",
+            *(extra_args or ()),
+        ],
+        env_inherit={"PATH", "PYENV_ROOT", "HOME"},
+    )
+    result = rule_runner.request(
+        RabbitMQIsRunning,
+        [uses_rabbitmq_request, mock_platform],
+    )
+    return result
+
+
+# Warning this requires that rabbitmq be running
+def test_rabbitmq_is_running(rule_runner: RuleRunner) -> None:
+    request = UsesRabbitMQRequest()
+    mock_platform = platform(os="TestMock")
+
+    # we are asserting that this does not raise an exception
+    is_running = run_rabbitmq_is_running(rule_runner, request, mock_platform)
+    assert is_running
+
+
+@pytest.mark.parametrize("mock_platform", platform_samples)
+def test_rabbitmq_not_running(rule_runner: RuleRunner, mock_platform: Platform) -> None:
+    request = UsesRabbitMQRequest(
+        mq_urls=(
+            "amqp://guest:[email protected]:10/",  # 10 = unassigned port, unlikely to be used
+        ),
+    )
+
+    with pytest.raises(ExecutionError) as exception_info:
+        run_rabbitmq_is_running(rule_runner, request, mock_platform)
+
+    execution_error = exception_info.value
+    assert len(execution_error.wrapped_exceptions) == 1
+
+    exc = execution_error.wrapped_exceptions[0]
+    assert isinstance(exc, ServiceMissingError)
+
+    assert exc.service == "rabbitmq"
+    assert "The rabbitmq service does not seem to be running" in str(exc)
+    assert exc.instructions != ""

pants-plugins/uses_services/scripts/is_rabbitmq_running.py

@@ -0,0 +1,48 @@
+# Copyright 2023 The StackStorm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import annotations
+
+import sys
+
+
+def _is_rabbitmq_running(mq_urls: list[str]) -> bool:
+    """Connect to rabbitmq with connection logic that mirrors the st2 code.
+
+    In particular, this is based on:
+      - st2common.transport.utils.get_connection()
+      - st2common.transport.bootstrap_utils.register_exchanges()
+
+    This should not import the st2 code as it should be self-contained.
+    """
+    # late import so that __file__ can be imported in the pants plugin without these imports
+    from kombu import Connection
+
+    with Connection(mq_urls) as connection:
+        try:
+            # connection is lazy. Make it connect immediately.
+            connection.connect()
+        except connection.connection_errors:
+            return False
+    return True
+
+
+if __name__ == "__main__":
+    mq_urls = list(sys.argv[1:])
+    if not mq_urls:
+        # st2.tests*.conf ends in /, but the default ends in //
+        mq_urls = ["amqp://guest:[email protected]:5672//"]
+
+    is_running = _is_rabbitmq_running(mq_urls)
+    exit_code = 0 if is_running else 1
+    sys.exit(exit_code)

st2common/tests/unit/BUILD

@@ -9,7 +9,7 @@ python_tests(
         # several files import tests.unit.base which is ambiguous. Tell pants which one to use.
         "st2common/tests/unit/base.py",
     ],
-    uses=["mongo"],
+    uses=["mongo", "rabbitmq"],
 )
 
 python_sources()

st2common/tests/unit/services/BUILD

@@ -1,4 +1,4 @@
 python_tests(
     name="tests",
-    uses=["mongo"],
+    uses=["mongo", "rabbitmq"],
 )