diff --git a/core/testcontainers/core/config.py b/core/testcontainers/core/config.py index 110a441e..6a4ca01f 100644 --- a/core/testcontainers/core/config.py +++ b/core/testcontainers/core/config.py @@ -71,7 +71,10 @@ def read_tc_properties() -> dict[str, str]: return settings -_WARNINGS = {"DOCKER_AUTH_CONFIG": "DOCKER_AUTH_CONFIG is experimental, see testcontainers/testcontainers-python#566"} +_WARNINGS = { + "DOCKER_AUTH_CONFIG": "DOCKER_AUTH_CONFIG is experimental, see testcontainers/testcontainers-python#566", + "tc_properties_get_tc_host": "this method has moved to property 'tc_properties_tc_host'", +} @dataclass @@ -107,9 +110,19 @@ def docker_auth_config(self, value: str) -> None: self._docker_auth_config = value def tc_properties_get_tc_host(self) -> Union[str, None]: + if "tc_properties_get_tc_host" in _WARNINGS: + warning(_WARNINGS.pop("tc_properties_get_tc_host")) return self.tc_properties.get("tc.host") @property + def tc_properties_tc_host(self) -> Union[str, None]: + return self.tc_properties.get("tc.host") + + @property + def tc_properties_testcontainers_reuse_enable(self) -> bool: + enabled = self.tc_properties.get("testcontainers.reuse.enable") + return enabled == "true" + def timeout(self) -> int: return self.max_tries * self.sleep_time diff --git a/core/testcontainers/core/container.py b/core/testcontainers/core/container.py index f677182f..925334a8 100644 --- a/core/testcontainers/core/container.py +++ b/core/testcontainers/core/container.py @@ -1,4 +1,7 @@ import contextlib +import hashlib +import logging +from platform import system from os import PathLike from socket import socket from typing import TYPE_CHECKING, Optional, Union @@ -53,6 +56,7 @@ def __init__( self._name = None self._network: Optional[Network] = None self._network_aliases: Optional[list[str]] = None + self._reuse: bool = False self._kwargs = kwargs def with_env(self, key: str, value: str) -> Self: @@ -86,17 +90,24 @@ def with_kwargs(self, **kwargs) -> Self: self._kwargs = kwargs return self + def with_reuse(self, reuse=True) -> Self: + self._reuse = reuse + return self + def maybe_emulate_amd64(self) -> Self: if is_arm(): return self.with_kwargs(platform="linux/amd64") return self def start(self) -> Self: - if not c.ryuk_disabled and self.image != c.ryuk_image: + if ( + not c.ryuk_disabled + and self.image != c.ryuk_image + and not (self._reuse and c.tc_properties_testcontainers_reuse_enable) + ): logger.debug("Creating Ryuk container") Reaper.get_instance() logger.info("Pulling image %s", self.image) - docker_client = self.get_docker_client() self._configure() network_kwargs = ( @@ -110,6 +121,45 @@ def start(self) -> Self: else {} ) + if self._reuse and not c.tc_properties_testcontainers_reuse_enable: + logging.warning( + "Reuse was requested (`with_reuse`) but the environment does not " + + "support the reuse of containers. To enable container reuse, add " + + "'testcontainers.reuse.enable=true' to '~/.testcontainers.properties'." + ) + + if self._reuse and c.tc_properties_testcontainers_reuse_enable: + # NOTE: ideally the docker client would return the full container create + # request which could be used to generate the hash. + args = [ # Docker run arguments + self.image, + self._command, + self.env, + self.ports, + self._name, + self.volumes, + str(tuple(sorted(self._kwargs.values()))), + ] + hash_ = hashlib.sha256(bytes(str(args), encoding="utf-8")).hexdigest() + docker_client = self.get_docker_client() + container = docker_client.find_container_by_hash(hash_) + if container: + if container.status != "running": + container.start() + logger.info("Existing container started: %s", container.id) + self._container = container + logger.info("Container is already running: %s", container.id) + else: + self._start(network_kwargs, hash_) + else: + self._start(network_kwargs) + + if self._network: + self._network.connect(self._container.id, self._network_aliases) + return self + + def _start(self, network_kwargs, hash_=None): + docker_client = self.get_docker_client() self._container = docker_client.run( self.image, command=self._command, @@ -118,6 +168,7 @@ def start(self) -> Self: ports=self.ports, name=self._name, volumes=self.volumes, + labels={"hash": hash_} if hash is not None else {}, **network_kwargs, **self._kwargs, ) diff --git a/core/testcontainers/core/docker_client.py b/core/testcontainers/core/docker_client.py index 83127884..85d9cd37 100644 --- a/core/testcontainers/core/docker_client.py +++ b/core/testcontainers/core/docker_client.py @@ -255,9 +255,15 @@ def client_networks_create(self, name: str, param: dict): labels = create_labels("", param.get("labels")) return self.client.networks.create(name, **{**param, "labels": labels}) + def find_container_by_hash(self, hash_: str) -> Union[Container, None]: + for container in self.client.containers.list(all=True): + if container.labels.get("hash", None) == hash_: + return container + return None + def get_docker_host() -> Optional[str]: - return c.tc_properties_get_tc_host() or os.getenv("DOCKER_HOST") + return c.tc_properties_tc_host or os.getenv("DOCKER_HOST") def get_docker_auth_config() -> Optional[str]: diff --git a/core/tests/test_reusable_containers.py b/core/tests/test_reusable_containers.py new file mode 100644 index 00000000..6c956379 --- /dev/null +++ b/core/tests/test_reusable_containers.py @@ -0,0 +1,129 @@ +from time import sleep + +from docker.models.containers import Container + +from testcontainers.core.config import testcontainers_config +from testcontainers.core.container import DockerContainer +from testcontainers.core.docker_client import DockerClient +from testcontainers.core.waiting_utils import wait_for_logs +from testcontainers.core.container import Reaper + + +def test_docker_container_reuse_default(): + # Make sure Ryuk cleanup is not active from previous test runs + Reaper.delete_instance() + + container = DockerContainer("hello-world").start() + wait_for_logs(container, "Hello from Docker!") + + assert container._reuse == False + assert testcontainers_config.tc_properties_testcontainers_reuse_enable == False + assert Reaper._socket is not None + + container.stop() + containers = DockerClient().client.containers.list(all=True) + assert container._container.id not in [container.id for container in containers] + + +def test_docker_container_with_reuse_reuse_disabled(caplog): + # Make sure Ryuk cleanup is not active from previous test runs + Reaper.delete_instance() + + container = DockerContainer("hello-world").with_reuse().start() + wait_for_logs(container, "Hello from Docker!") + + assert container._reuse == True + assert testcontainers_config.tc_properties_testcontainers_reuse_enable == False + assert ( + "Reuse was requested (`with_reuse`) but the environment does not support the " + + "reuse of containers. To enable container reuse, add " + + "'testcontainers.reuse.enable=true' to '~/.testcontainers.properties'." + ) in caplog.text + assert Reaper._socket is not None + + container.stop() + containers = DockerClient().client.containers.list(all=True) + assert container._container.id not in [container.id for container in containers] + + +def test_docker_container_without_reuse_reuse_enabled(monkeypatch): + # Make sure Ryuk cleanup is not active from previous test runs + Reaper.delete_instance() + + tc_properties_mock = testcontainers_config.tc_properties | {"testcontainers.reuse.enable": "true"} + monkeypatch.setattr(testcontainers_config, "tc_properties", tc_properties_mock) + + container = DockerContainer("hello-world").start() + wait_for_logs(container, "Hello from Docker!") + + assert container._reuse == False + assert testcontainers_config.tc_properties_testcontainers_reuse_enable == True + assert Reaper._socket is not None + + container.stop() + containers = DockerClient().client.containers.list(all=True) + assert container._container.id not in [container.id for container in containers] + + +def test_docker_container_with_reuse_reuse_enabled(monkeypatch): + # Make sure Ryuk cleanup is not active from previous test runs + Reaper.delete_instance() + + tc_properties_mock = testcontainers_config.tc_properties | {"testcontainers.reuse.enable": "true"} + monkeypatch.setattr(testcontainers_config, "tc_properties", tc_properties_mock) + + container = DockerContainer("hello-world").with_reuse().start() + wait_for_logs(container, "Hello from Docker!") + + assert Reaper._socket is None + + containers = DockerClient().client.containers.list(all=True) + assert container._container.id in [container.id for container in containers] + # Cleanup after keeping container alive (with_reuse) + container.stop() + + +def test_docker_container_with_reuse_reuse_enabled_same_id(monkeypatch): + # Make sure Ryuk cleanup is not active from previous test runs + Reaper.delete_instance() + + tc_properties_mock = testcontainers_config.tc_properties | {"testcontainers.reuse.enable": "true"} + monkeypatch.setattr(testcontainers_config, "tc_properties", tc_properties_mock) + + container_1 = DockerContainer("hello-world").with_reuse().start() + id_1 = container_1._container.id + container_2 = DockerContainer("hello-world").with_reuse().start() + id_2 = container_2._container.id + assert Reaper._socket is None + assert id_1 == id_2 + # Cleanup after keeping container alive (with_reuse) + container_1.stop() + # container_2.stop() is not needed since it is the same as container_1 + + +def test_docker_container_labels_hash_default(): + # w/out reuse + with DockerContainer("hello-world") as container: + assert container._container.labels["hash"] == "" + + +def test_docker_container_labels_hash(monkeypatch): + tc_properties_mock = testcontainers_config.tc_properties | {"testcontainers.reuse.enable": "true"} + monkeypatch.setattr(testcontainers_config, "tc_properties", tc_properties_mock) + expected_hash = "1bade17a9d8236ba71ffbb676f2ece3fb419ea0e6adb5f82b5a026213c431d8e" + with DockerContainer("hello-world").with_reuse() as container: + assert container._container.labels["hash"] == expected_hash + + +def test_docker_client_find_container_by_hash_not_existing(): + with DockerContainer("hello-world"): + assert DockerClient().find_container_by_hash("foo") == None + + +def test_docker_client_find_container_by_hash_existing(monkeypatch): + tc_properties_mock = testcontainers_config.tc_properties | {"testcontainers.reuse.enable": "true"} + monkeypatch.setattr(testcontainers_config, "tc_properties", tc_properties_mock) + with DockerContainer("hello-world").with_reuse() as container: + hash_ = container._container.labels["hash"] + found_container = DockerClient().find_container_by_hash(hash_) + assert isinstance(found_container, Container) diff --git a/index.rst b/index.rst index 307f934c..4ac01afd 100644 --- a/index.rst +++ b/index.rst @@ -89,7 +89,6 @@ When trying to launch Testcontainers from within a Docker container, e.g., in co 1. The container has to provide a docker client installation. Either use an image that has docker pre-installed (e.g. the `official docker images `_) or install the client from within the `Dockerfile` specification. 2. The container has to have access to the docker daemon which can be achieved by mounting `/var/run/docker.sock` or setting the `DOCKER_HOST` environment variable as part of your `docker run` command. - Private Docker registry ----------------------- @@ -118,6 +117,36 @@ Fetching passwords from cloud providers: GCP_PASSWORD = $(gcloud auth print-access-token) AZURE_PASSWORD = $(az acr login --name --expose-token --output tsv) +Reusable Containers (Experimental) +---------------------------------- + +.. warning:: + Reusable Containers is still an experimental feature and the behavior can change. + Those containers won't stop after all tests are finished. + +Containers can be reused across consecutive test runs. To reuse a container, the container has to be started manually by calling the `start()` method. Do not call the `stop()` method directly or indirectly via a `with` statement (context manager). To reuse a container, the container configuration must be the same. + +Containers that are set up for reuse will not be automatically removed. Thus, if they are not needed anymore, those containers must be removed manually. + +Containers should not be reused in a CI environment. + +How to use? +^^^^^^^^^^^ + +1. Add :code:`testcontainers.reuse.enable=true` to :code:`~/.testcontainers.properties` +2. Disable ryuk by setting the environment variable :code:`TESTCONTAINERS_RYUK_DISABLED=true` +3. Instantiate a container using :code:`with_reuse()` and :code:`start()` + +.. doctest:: + + >>> from testcontainers.core.container import DockerContainer + + >>> container = DockerContainer("hello-world").with_reuse().start() + >>> first_id = container._container.id + >>> container = DockerContainer("hello-world").with_reuse().start() + >>> second_id == container._container.id + >>> print(first_id == second_id) + True Configuration -------------