diff --git a/.github/.release-please-manifest.json b/.github/.release-please-manifest.json index 698886db5..4d204362b 100644 --- a/.github/.release-please-manifest.json +++ b/.github/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "3.7.1" + ".": "4.0.0" } diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 000000000..033a9c686 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,35 @@ +# Changelog + +## [4.0.0](https://github.com/testcontainers/testcontainers-python/compare/testcontainers-v3.7.1...testcontainers-v4.0.0) (2024-03-06) + + +### ⚠ BREAKING CHANGES + +* **compose:** implement compose v2 with improved typing ([#426](https://github.com/testcontainers/testcontainers-python/issues/426)) +* **core:** add support for `tc.host` and de-prioritise `docker:dind` ([#388](https://github.com/testcontainers/testcontainers-python/issues/388)) + +### Features + +* **build:** use poetry and organise modules ([#408](https://github.com/testcontainers/testcontainers-python/issues/408)) ([6c69583](https://github.com/testcontainers/testcontainers-python/commit/6c695835520bdcbf9824e8cefa00f7613d2a7cb9)) +* **compose:** allow running specific services in compose ([f61dcda](https://github.com/testcontainers/testcontainers-python/commit/f61dcda8bd7ea329cd3c836b6d6e2f0bd990335d)) +* **compose:** implement compose v2 with improved typing ([#426](https://github.com/testcontainers/testcontainers-python/issues/426)) ([5356caf](https://github.com/testcontainers/testcontainers-python/commit/5356caf2de056313a5b3f2805ed80e6a23b027a8)) +* **core:** add support for `tc.host` and de-prioritise `docker:dind` ([#388](https://github.com/testcontainers/testcontainers-python/issues/388)) ([2db8e6d](https://github.com/testcontainers/testcontainers-python/commit/2db8e6d123d42b57309408dd98ba9a06acc05c4b)) +* **redis:** support AsyncRedisContainer ([#442](https://github.com/testcontainers/testcontainers-python/issues/442)) ([cc4cb37](https://github.com/testcontainers/testcontainers-python/commit/cc4cb3762802dc75b0801727d8b1f1a1c56b7f50)) +* **release:** automate release via release-please ([#429](https://github.com/testcontainers/testcontainers-python/issues/429)) ([30f859e](https://github.com/testcontainers/testcontainers-python/commit/30f859eb1535acd6e93c331213426e1319ee9a47)) + + +### Bug Fixes + +* Added URLError to exceptions to wait for in elasticsearch ([0f9ad24](https://github.com/testcontainers/testcontainers-python/commit/0f9ad24f2c0df362ee15b81ce8d7d36b9f98e6e1)) +* **build:** add `pre-commit` as a dev dependency to simplify local dev and CI ([#438](https://github.com/testcontainers/testcontainers-python/issues/438)) ([1223583](https://github.com/testcontainers/testcontainers-python/commit/1223583d8fc3a1ab95441d82c7e1ece57f026fbf)) +* **build:** early exit strategy for modules ([#437](https://github.com/testcontainers/testcontainers-python/issues/437)) ([7358b49](https://github.com/testcontainers/testcontainers-python/commit/7358b4919c1010315a384a8f0fe2860e5a0ca6b4)) +* changed files breaks on main ([#422](https://github.com/testcontainers/testcontainers-python/issues/422)) ([3271357](https://github.com/testcontainers/testcontainers-python/commit/32713578dcf07f672a87818e00562b58874b4a52)) +* flaky garbage collection resulting in testing errors ([#423](https://github.com/testcontainers/testcontainers-python/issues/423)) ([b535ea2](https://github.com/testcontainers/testcontainers-python/commit/b535ea255bcaaa546f8cda7b2b17718c1cc7f3ca)) +* rabbitmq readiness probe ([#375](https://github.com/testcontainers/testcontainers-python/issues/375)) ([71cb75b](https://github.com/testcontainers/testcontainers-python/commit/71cb75b281df55ece4d5caf5d487059a7f38c34f)) +* **release:** prove that the release process updates the version ([#444](https://github.com/testcontainers/testcontainers-python/issues/444)) ([87b5873](https://github.com/testcontainers/testcontainers-python/commit/87b5873c1ec3a3e4e74742417d6068fa86cf1762)) +* test linting issue ([427c9b8](https://github.com/testcontainers/testcontainers-python/commit/427c9b841c2f6f516ec6cb74d5bd2839cb1939f4)) + + +### Documentation + +* Sphinx - Add title to each doc page ([#443](https://github.com/testcontainers/testcontainers-python/issues/443)) ([750e12a](https://github.com/testcontainers/testcontainers-python/commit/750e12a41172ce4aaf045c61dec33d318dc3c2f6)) diff --git a/core/testcontainers/compose/__init__.py b/core/testcontainers/compose/__init__.py new file mode 100644 index 000000000..9af994f30 --- /dev/null +++ b/core/testcontainers/compose/__init__.py @@ -0,0 +1,8 @@ +# flake8: noqa +from testcontainers.compose.compose import ( + ContainerIsNotRunning, + NoSuchPortExposed, + PublishedPort, + ComposeContainer, + DockerCompose, +) diff --git a/core/testcontainers/compose/compose.py b/core/testcontainers/compose/compose.py new file mode 100644 index 000000000..e72824bd1 --- /dev/null +++ b/core/testcontainers/compose/compose.py @@ -0,0 +1,406 @@ +import subprocess +from dataclasses import dataclass, field, fields +from functools import cached_property +from json import loads +from os import PathLike +from re import split +from typing import Callable, Literal, Optional, TypeVar, Union +from urllib.error import HTTPError, URLError +from urllib.request import urlopen + +from testcontainers.core.exceptions import ContainerIsNotRunning, NoSuchPortExposed +from testcontainers.core.waiting_utils import wait_container_is_ready + +_IPT = TypeVar("_IPT") + + +def _ignore_properties(cls: type[_IPT], dict_: any) -> _IPT: + """omits extra fields like @JsonIgnoreProperties(ignoreUnknown = true) + + https://gist.github.com/alexanderankin/2a4549ac03554a31bef6eaaf2eaf7fd5""" + if isinstance(dict_, cls): + return dict_ + class_fields = {f.name for f in fields(cls)} + filtered = {k: v for k, v in dict_.items() if k in class_fields} + return cls(**filtered) + + +@dataclass +class PublishedPort: + """ + Class that represents the response we get from compose when inquiring status + via `DockerCompose.get_running_containers()`. + """ + + URL: Optional[str] = None + TargetPort: Optional[str] = None + PublishedPort: Optional[str] = None + Protocol: Optional[str] = None + + +OT = TypeVar("OT") + + +def get_only_element_or_raise(array: list[OT], exception: Callable[[], Exception]) -> OT: + if len(array) != 1: + e = exception() + raise e + return array[0] + + +@dataclass +class ComposeContainer: + """ + A container class that represents a container managed by compose. + It is not a true testcontainers.core.container.DockerContainer, + but you can use the id with DockerClient to get that one too. + """ + + ID: Optional[str] = None + Name: Optional[str] = None + Command: Optional[str] = None + Project: Optional[str] = None + Service: Optional[str] = None + State: Optional[str] = None + Health: Optional[str] = None + ExitCode: Optional[str] = None + Publishers: list[PublishedPort] = field(default_factory=list) + + def __post_init__(self): + if self.Publishers: + self.Publishers = [_ignore_properties(PublishedPort, p) for p in self.Publishers] + + def get_publisher( + self, + by_port: Optional[int] = None, + by_host: Optional[str] = None, + prefer_ip_version: Literal["IPV4", "IPv6"] = "IPv4", + ) -> PublishedPort: + remaining_publishers = self.Publishers + + remaining_publishers = [r for r in remaining_publishers if self._matches_protocol(prefer_ip_version, r)] + + if by_port: + remaining_publishers = [item for item in remaining_publishers if by_port == item.TargetPort] + if by_host: + remaining_publishers = [item for item in remaining_publishers if by_host == item.URL] + if len(remaining_publishers) == 0: + raise NoSuchPortExposed(f"Could not find publisher for for service {self.Service}") + return get_only_element_or_raise( + remaining_publishers, + lambda: NoSuchPortExposed( + "get_publisher failed because there is " + f"not exactly 1 publisher for service {self.Service}" + f" when filtering by_port={by_port}, by_host={by_host}" + f" (but {len(remaining_publishers)})" + ), + ) + + @staticmethod + def _matches_protocol(prefer_ip_version, r): + return (":" in r.URL) is (prefer_ip_version == "IPv6") + + +@dataclass +class DockerCompose: + """ + Manage docker compose environments. + + Args: + context: + The docker context. It corresponds to the directory containing + the docker compose configuration file. + compose_file_name: + Optional. File name of the docker compose configuration file. + If specified, you need to also specify the overrides if any. + pull: + Pull images before launching environment. + build: + Run `docker compose build` before running the environment. + wait: + Wait for the services to be healthy + (as per healthcheck definitions in the docker compose configuration) + env_file: + Path to an '.env' file containing environment variables + to pass to docker compose. + services: + The list of services to use from this DockerCompose. + client_args: + arguments to pass to docker.from_env() + + Example: + + This example spins up chrome and firefox containers using docker compose. + + .. doctest:: + + >>> from testcontainers.compose import DockerCompose + + >>> compose = DockerCompose("compose/tests", compose_file_name="docker-compose-4.yml", + ... pull=True) + >>> with compose: + ... stdout, stderr = compose.get_logs() + >>> b"Hello from Docker!" in stdout + True + + .. code-block:: yaml + + services: + hello-world: + image: "hello-world" + """ + + context: Union[str, PathLike] + compose_file_name: Optional[Union[str, list[str]]] = None + pull: bool = False + build: bool = False + wait: bool = True + env_file: Optional[str] = None + services: Optional[list[str]] = None + + def __post_init__(self): + if isinstance(self.compose_file_name, str): + self.compose_file_name = [self.compose_file_name] + + def __enter__(self) -> "DockerCompose": + self.start() + return self + + def __exit__(self, exc_type, exc_val, exc_tb) -> None: + self.stop() + + def docker_compose_command(self) -> list[str]: + """ + Returns command parts used for the docker compose commands + + Returns: + cmd: Docker compose command parts. + """ + return self.compose_command_property + + @cached_property + def compose_command_property(self) -> list[str]: + docker_compose_cmd = ["docker", "compose"] + if self.compose_file_name: + for file in self.compose_file_name: + docker_compose_cmd += ["-f", file] + if self.env_file: + docker_compose_cmd += ["--env-file", self.env_file] + return docker_compose_cmd + + def start(self) -> None: + """ + Starts the docker compose environment. + """ + base_cmd = self.compose_command_property or [] + + # pull means running a separate command before starting + if self.pull: + pull_cmd = [*base_cmd, "pull"] + self._call_command(cmd=pull_cmd) + + up_cmd = [*base_cmd, "up"] + + # build means modifying the up command + if self.build: + up_cmd.append("--build") + + if self.wait: + up_cmd.append("--wait") + else: + # we run in detached mode instead of blocking + up_cmd.append("--detach") + + if self.services: + up_cmd.extend(self.services) + + self._call_command(cmd=up_cmd) + + def stop(self, down=True) -> None: + """ + Stops the docker compose environment. + """ + down_cmd = self.compose_command_property[:] + if down: + down_cmd += ["down", "--volumes"] + else: + down_cmd += ["stop"] + self._call_command(cmd=down_cmd) + + def get_logs(self, *services: str) -> tuple[str, str]: + """ + Returns all log output from stdout and stderr of a specific container. + + :param services: which services to get the logs for (or omit, for all) + + Returns: + stdout: Standard output stream. + stderr: Standard error stream. + """ + logs_cmd = [*self.compose_command_property, "logs", *services] + + result = subprocess.run( + logs_cmd, + cwd=self.context, + capture_output=True, + ) + return result.stdout.decode("utf-8"), result.stderr.decode("utf-8") + + def get_containers(self, include_all=False) -> list[ComposeContainer]: + """ + Fetch information about running containers via `docker compose ps --format json`. + Available only in V2 of compose. + + Returns: + The list of running containers. + + """ + + cmd = [*self.compose_command_property, "ps", "--format", "json"] + if include_all: + cmd = [*cmd, "-a"] + result = subprocess.run(cmd, cwd=self.context, check=True, stdout=subprocess.PIPE) + stdout = split(r"\r?\n", result.stdout.decode("utf-8")) + + containers = [] + # one line per service in docker 25, single array for docker 24.0.2 + for line in stdout: + if not line: + continue + data = loads(line) + if isinstance(data, list): + containers += [_ignore_properties(ComposeContainer, d) for d in data] + else: + containers.append(_ignore_properties(ComposeContainer, data)) + + return containers + + def get_container( + self, + service_name: Optional[str] = None, + include_all: bool = False, + ) -> ComposeContainer: + if not service_name: + containers = self.get_containers(include_all=include_all) + return get_only_element_or_raise( + containers, + lambda: ContainerIsNotRunning( + "get_container failed because no service_name given " + f"and there is not exactly 1 container (but {len(containers)})" + ), + ) + + matching_containers = [ + item for item in self.get_containers(include_all=include_all) if item.Service == service_name + ] + + if not matching_containers: + raise ContainerIsNotRunning(f"{service_name} is not running in the compose context") + + return matching_containers[0] + + def exec_in_container( + self, + command: list[str], + service_name: Optional[str] = None, + ) -> tuple[str, str, int]: + """ + Executes a command in the container of one of the services. + + Args: + service_name: Name of the docker compose service to run the command in. + command: Command to execute. + + :param service_name: specify the service name + :param command: the command to run in the container + + Returns: + stdout: Standard output stream. + stderr: Standard error stream. + exit_code: The command's exit code. + """ + if not service_name: + service_name = self.get_container().Service + exec_cmd = [*self.compose_command_property, "exec", "-T", service_name, *command] + result = subprocess.run( + exec_cmd, + cwd=self.context, + capture_output=True, + check=True, + ) + + return (result.stdout.decode("utf-8"), result.stderr.decode("utf-8"), result.returncode) + + def _call_command( + self, + cmd: Union[str, list[str]], + context: Optional[str] = None, + ) -> None: + context = context or self.context + subprocess.call(cmd, cwd=context) + + def get_service_port( + self, + service_name: Optional[str] = None, + port: Optional[int] = None, + ): + """ + Returns the mapped port for one of the services. + + Parameters + ---------- + service_name: str + Name of the docker compose service + port: int + The internal port to get the mapping for + + Returns + ------- + str: + The mapped port on the host + """ + return self.get_container(service_name).get_publisher(by_port=port).PublishedPort + + def get_service_host( + self, + service_name: Optional[str] = None, + port: Optional[int] = None, + ): + """ + Returns the host for one of the services. + + Parameters + ---------- + service_name: str + Name of the docker compose service + port: int + The internal port to get the host for + + Returns + ------- + str: + The hostname for the service + """ + return self.get_container(service_name).get_publisher(by_port=port).URL + + def get_service_host_and_port( + self, + service_name: Optional[str] = None, + port: Optional[int] = None, + ): + publisher = self.get_container(service_name).get_publisher(by_port=port) + return publisher.URL, publisher.PublishedPort + + @wait_container_is_ready(HTTPError, URLError) + def wait_for(self, url: str) -> "DockerCompose": + """ + Waits for a response from a given URL. This is typically used to block until a service in + the environment has started and is responding. Note that it does not assert any sort of + return code, only check that the connection was successful. + + Args: + url: URL from one of the services in the environment to use to wait on. + """ + with urlopen(url) as response: + response.read() + return self diff --git a/core/testcontainers/core/exceptions.py b/core/testcontainers/core/exceptions.py index 8bf027630..6694e598b 100644 --- a/core/testcontainers/core/exceptions.py +++ b/core/testcontainers/core/exceptions.py @@ -16,5 +16,9 @@ class ContainerStartException(RuntimeError): pass +class ContainerIsNotRunning(RuntimeError): + pass + + class NoSuchPortExposed(RuntimeError): pass diff --git a/core/tests/compose_fixtures/basic/docker-compose.yaml b/core/tests/compose_fixtures/basic/docker-compose.yaml new file mode 100644 index 000000000..ff3f74220 --- /dev/null +++ b/core/tests/compose_fixtures/basic/docker-compose.yaml @@ -0,0 +1,10 @@ +version: '3.0' + +services: + alpine: + image: alpine:latest + init: true + command: + - sh + - -c + - 'while true; do sleep 0.1 ; date -Ins; done' diff --git a/core/tests/compose_fixtures/port_multiple/compose.yaml b/core/tests/compose_fixtures/port_multiple/compose.yaml new file mode 100644 index 000000000..65717fc4a --- /dev/null +++ b/core/tests/compose_fixtures/port_multiple/compose.yaml @@ -0,0 +1,28 @@ +version: '3.0' + +services: + alpine: + image: nginx:alpine-slim + init: true + ports: + - '81' + - '82' + - target: 80 + host_ip: 127.0.0.1 + protocol: tcp + command: + - sh + - -c + - 'd=/etc/nginx/conf.d; echo "server { listen 81; location / { return 202; } }" > $$d/81.conf && echo "server { listen 82; location / { return 204; } }" > $$d/82.conf && nginx -g "daemon off;"' + + alpine2: + image: nginx:alpine-slim + init: true + ports: + - target: 80 + host_ip: 127.0.0.1 + protocol: tcp + command: + - sh + - -c + - 'd=/etc/nginx/conf.d; echo "server { listen 81; location / { return 202; } }" > $$d/81.conf && echo "server { listen 82; location / { return 204; } }" > $$d/82.conf && nginx -g "daemon off;"' diff --git a/core/tests/compose_fixtures/port_single/compose.yaml b/core/tests/compose_fixtures/port_single/compose.yaml new file mode 100644 index 000000000..d1bf9eb45 --- /dev/null +++ b/core/tests/compose_fixtures/port_single/compose.yaml @@ -0,0 +1,14 @@ +version: '3.0' + +services: + alpine: + image: nginx:alpine-slim + init: true + ports: + - target: 80 + host_ip: 127.0.0.1 + protocol: tcp + command: + - sh + - -c + - 'nginx -g "daemon off;"' diff --git a/core/tests/test_compose.py b/core/tests/test_compose.py new file mode 100644 index 000000000..0a244220b --- /dev/null +++ b/core/tests/test_compose.py @@ -0,0 +1,243 @@ +from pathlib import Path +from re import split +from time import sleep +from typing import Union +from urllib.request import urlopen, Request + +import pytest + +from testcontainers.compose import DockerCompose, ContainerIsNotRunning, NoSuchPortExposed + +FIXTURES = Path(__file__).parent.joinpath("compose_fixtures") + + +def test_compose_no_file_name(): + basic = DockerCompose(context=FIXTURES / "basic") + assert basic.compose_file_name is None + + +def test_compose_str_file_name(): + basic = DockerCompose(context=FIXTURES / "basic", compose_file_name="docker-compose.yaml") + assert basic.compose_file_name == ["docker-compose.yaml"] + + +def test_compose_list_file_name(): + basic = DockerCompose(context=FIXTURES / "basic", compose_file_name=["docker-compose.yaml"]) + assert basic.compose_file_name == ["docker-compose.yaml"] + + +def test_compose_stop(): + basic = DockerCompose(context=FIXTURES / "basic") + basic.stop() + + +def test_compose_start_stop(): + basic = DockerCompose(context=FIXTURES / "basic") + basic.start() + basic.stop() + + +def test_compose(): + """stream-of-consciousness e2e test""" + basic = DockerCompose(context=FIXTURES / "basic") + try: + # first it does not exist + containers = basic.get_containers(include_all=True) + assert len(containers) == 0 + + # then we create it and it exists + basic.start() + containers = basic.get_containers(include_all=True) + assert len(containers) == 1 + containers = basic.get_containers() + assert len(containers) == 1 + + # test that get_container returns the same object, value assertions, etc + from_all = containers[0] + assert from_all.State == "running" + assert from_all.Service == "alpine" + + by_name = basic.get_container("alpine") + + assert by_name.Name == from_all.Name + assert by_name.Service == from_all.Service + assert by_name.State == from_all.State + assert by_name.ID == from_all.ID + + assert by_name.ExitCode == 0 + + # what if you want to get logs after it crashes: + basic.stop(down=False) + + with pytest.raises(ContainerIsNotRunning): + assert basic.get_container("alpine") is None + + # what it looks like after it exits + stopped = basic.get_container("alpine", include_all=True) + assert stopped.State == "exited" + finally: + basic.stop() + + +def test_compose_logs(): + basic = DockerCompose(context=FIXTURES / "basic") + with basic: + sleep(1) # generate some logs every 200ms + stdout, stderr = basic.get_logs() + container = basic.get_container() + + assert not stderr + assert stdout + lines = split(r"\r?\n", stdout) + + assert len(lines) > 5 # actually 10 + for line in lines[1:]: + # either the line is blank or the first column (|-separated) contains the service name + # this is a safe way to split the string + # docker changes the prefix between versions 24 and 25 + assert not line or container.Service in next(iter(line.split("|")), None) + + +# noinspection HttpUrlsUsage +def test_compose_ports(): + # fairly straight forward - can we get the right port to request it + single = DockerCompose(context=FIXTURES / "port_single") + with single: + host, port = single.get_service_host_and_port() + endpoint = f"http://{host}:{port}" + single.wait_for(endpoint) + code, response = fetch(Request(method="GET", url=endpoint)) + assert code == 200 + assert "