From a933873027d529b2226ac56e1d0e59c959193c00 Mon Sep 17 00:00:00 2001 From: Matthias Schaub Date: Tue, 2 Jul 2024 14:11:50 +0200 Subject: [PATCH 01/12] feat: reusable containers adresses #109 Co-authored-by: Levi Szamek --- core/testcontainers/core/config.py | 16 ++++- core/testcontainers/core/container.py | 59 ++++++++++++++-- core/testcontainers/core/docker_client.py | 8 ++- core/tests/test_reusable_containers.py | 83 +++++++++++++++++++++++ 4 files changed, 159 insertions(+), 7 deletions(-) create mode 100644 core/tests/test_reusable_containers.py diff --git a/core/testcontainers/core/config.py b/core/testcontainers/core/config.py index 3522b91f..0f960b02 100644 --- a/core/testcontainers/core/config.py +++ b/core/testcontainers/core/config.py @@ -39,7 +39,10 @@ def read_tc_properties() -> dict[str, str]: return settings -_WARNINGS = {"DOCKER_AUTH_CONFIG": "DOCKER_AUTH_CONFIG is experimental, see testcontainers/testcontainers-python#566"} +_WARNINGS = { + "DOCKER_AUTH_CONFIG": "DOCKER_AUTH_CONFIG is experimental, see testcontainers/testcontainers-python#566", + "tc_properties_get_tc_host": "this method has moved to property 'tc_properties_tc_host'", +} @dataclass @@ -73,8 +76,19 @@ def docker_auth_config(self, value: str): self._docker_auth_config = value def tc_properties_get_tc_host(self) -> Union[str, None]: + if "tc_properties_get_tc_host" in _WARNINGS: + warning(_WARNINGS.pop("tc_properties_get_tc_host")) return self.tc_properties.get("tc.host") + @property + def tc_properties_tc_host(self) -> Union[str, None]: + return self.tc_properties.get("tc.host") + + @property + def tc_properties_testcontainers_reuse_enable(self) -> bool: + enabled = self.tc_properties.get("testcontainers.reuse.enable") + return enabled == "true" + @property def timeout(self): return self.max_tries * self.sleep_time diff --git a/core/testcontainers/core/container.py b/core/testcontainers/core/container.py index 085fc58e..caa4c61e 100644 --- a/core/testcontainers/core/container.py +++ b/core/testcontainers/core/container.py @@ -1,4 +1,6 @@ import contextlib +import hashlib +import logging from platform import system from socket import socket from typing import TYPE_CHECKING, Optional @@ -49,6 +51,7 @@ def __init__( self._name = None self._network: Optional[Network] = None self._network_aliases: Optional[list[str]] = None + self._reuse: bool = False self._kwargs = kwargs def with_env(self, key: str, value: str) -> Self: @@ -76,6 +79,10 @@ def with_kwargs(self, **kwargs) -> Self: self._kwargs = kwargs return self + def with_reuse(self, reuse=True) -> Self: + self._reuse = reuse + return self + def maybe_emulate_amd64(self) -> Self: if is_arm(): return self.with_kwargs(platform="linux/amd64") @@ -86,8 +93,49 @@ def start(self) -> Self: logger.debug("Creating Ryuk container") Reaper.get_instance() logger.info("Pulling image %s", self.image) - docker_client = self.get_docker_client() self._configure() + + # container hash consisting of run arguments + args = ( + self.image, + self._command, + self.env, + self.ports, + self._name, + self.volumes, + str(tuple(sorted(self._kwargs.items()))), + ) + hash_ = hashlib.sha256(bytes(str(args), encoding="utf-8")).hexdigest() + + # TODO: check also if ryuk is disabled + if self._reuse and not c.tc_properties_testcontainers_reuse_enable: + logging.warning( + "Reuse was requested (`with_reuse`) but the environment does not " + + "support the reuse of containers. To enable container reuse, add " + + "the property 'testcontainers.reuse.enable=true' to a file at " + + "~/.testcontainers.properties (you may need to create it)." + ) + + if self._reuse and c.tc_properties_testcontainers_reuse_enable: + docker_client = self.get_docker_client() + container = docker_client.find_container_by_hash(hash_) + if container: + if container.status != "running": + container.start() + logger.info("Existing container started: %s", container.id) + logger.info("Container is already running: %s", container.id) + self._container = container + else: + self._start(hash_) + else: + self._start(hash_) + + if self._network: + self._network.connect(self._container.id, self._network_aliases) + return self + + def _start(self, hash_): + docker_client = self.get_docker_client() self._container = docker_client.run( self.image, command=self._command, @@ -96,16 +144,17 @@ def start(self) -> Self: ports=self.ports, name=self._name, volumes=self.volumes, + labels={"hash": hash_}, **self._kwargs, ) logger.info("Container started: %s", self._container.short_id) - if self._network: - self._network.connect(self._container.id, self._network_aliases) - return self def stop(self, force=True, delete_volume=True) -> None: if self._container: - self._container.remove(force=force, v=delete_volume) + if self._reuse and c.tc_properties_testcontainers_reuse_enable: + self._container.stop() + else: + self._container.remove(force=force, v=delete_volume) self.get_docker_client().client.close() def __enter__(self) -> Self: diff --git a/core/testcontainers/core/docker_client.py b/core/testcontainers/core/docker_client.py index 286e1ef9..674b2ed6 100644 --- a/core/testcontainers/core/docker_client.py +++ b/core/testcontainers/core/docker_client.py @@ -215,9 +215,15 @@ def client_networks_create(self, name: str, param: dict): labels = create_labels("", param.get("labels")) return self.client.networks.create(name, **{**param, "labels": labels}) + def find_container_by_hash(self, hash_: str) -> Container | None: + for container in self.client.containers.list(all=True): + if container.labels.get("hash", None) == hash_: + return container + return None + def get_docker_host() -> Optional[str]: - return c.tc_properties_get_tc_host() or os.getenv("DOCKER_HOST") + return c.tc_properties_tc_host or os.getenv("DOCKER_HOST") def get_docker_auth_config() -> Optional[str]: diff --git a/core/tests/test_reusable_containers.py b/core/tests/test_reusable_containers.py new file mode 100644 index 00000000..a8ab8a9d --- /dev/null +++ b/core/tests/test_reusable_containers.py @@ -0,0 +1,83 @@ +from time import sleep + +from docker.models.containers import Container + +from testcontainers.core.config import testcontainers_config +from testcontainers.core.container import DockerContainer +from testcontainers.core.docker_client import DockerClient +from testcontainers.core.waiting_utils import wait_for_logs +from testcontainers.core.container import Reaper + + +def test_docker_container_reuse_default(): + with DockerContainer("hello-world") as container: + assert container._reuse == False + id = container._container.id + wait_for_logs(container, "Hello from Docker!") + containers = DockerClient().client.containers.list(all=True) + assert id not in [container.id for container in containers] + + +def test_docker_container_with_reuse_reuse_disabled(): + with DockerContainer("hello-world").with_reuse() as container: + assert container._reuse == True + id = container._container.id + wait_for_logs(container, "Hello from Docker!") + containers = DockerClient().client.containers.list(all=True) + assert id not in [container.id for container in containers] + + +def test_docker_container_with_reuse_reuse_enabled_ryuk_enabled(monkeypatch): + # Make sure Ryuk cleanup is not active from previous test runs + Reaper.delete_instance() + tc_properties_mock = testcontainers_config.tc_properties | {"testcontainers.reuse.enable": "true"} + monkeypatch.setattr(testcontainers_config, "tc_properties", tc_properties_mock) + monkeypatch.setattr(testcontainers_config, "ryuk_reconnection_timeout", "0.1s") + + with DockerContainer("hello-world").with_reuse() as container: + id = container._container.id + wait_for_logs(container, "Hello from Docker!") + + Reaper._socket.close() + # Sleep until Ryuk reaps all dangling containers + sleep(0.6) + + containers = DockerClient().client.containers.list(all=True) + assert id not in [container.id for container in containers] + + # Cleanup Ryuk class fields after manual Ryuk shutdown + Reaper.delete_instance() + + +def test_docker_container_with_reuse_reuse_enabled_ryuk_disabled(monkeypatch): + # Make sure Ryuk cleanup is not active from previous test runs + Reaper.delete_instance() + tc_properties_mock = testcontainers_config.tc_properties | {"testcontainers.reuse.enable": "true"} + monkeypatch.setattr(testcontainers_config, "tc_properties", tc_properties_mock) + monkeypatch.setattr(testcontainers_config, "ryuk_disabled", True) + with DockerContainer("hello-world").with_reuse() as container: + assert container._reuse == True + id = container._container.id + wait_for_logs(container, "Hello from Docker!") + containers = DockerClient().client.containers.list(all=True) + assert id in [container.id for container in containers] + # Cleanup after keeping container alive (with_reuse) + container._container.remove(force=True) + + +def test_docker_container_labels_hash(): + expected_hash = "91fde3c09244e1d3ec6f18a225b9261396b9a1cb0f6365b39b9795782817c128" + with DockerContainer("hello-world").with_reuse() as container: + assert container._container.labels["hash"] == expected_hash + + +def test_docker_client_find_container_by_hash_not_existing(): + with DockerContainer("hello-world"): + assert DockerClient().find_container_by_hash("foo") == None + + +def test_docker_client_find_container_by_hash_existing(): + with DockerContainer("hello-world").with_reuse() as container: + hash_ = container._container.labels["hash"] + found_container = DockerClient().find_container_by_hash(hash_) + assert isinstance(found_container, Container) From f0e2bc7ce92326ab430897d9c2ec2c9c86cda26d Mon Sep 17 00:00:00 2001 From: Matthias Schaub Date: Wed, 3 Jul 2024 15:17:35 +0200 Subject: [PATCH 02/12] docs: add documentation about reusable containers --- index.rst | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/index.rst b/index.rst index 70708a24..d199da39 100644 --- a/index.rst +++ b/index.rst @@ -89,7 +89,6 @@ When trying to launch Testcontainers from within a Docker container, e.g., in co 1. The container has to provide a docker client installation. Either use an image that has docker pre-installed (e.g. the `official docker images `_) or install the client from within the `Dockerfile` specification. 2. The container has to have access to the docker daemon which can be achieved by mounting `/var/run/docker.sock` or setting the `DOCKER_HOST` environment variable as part of your `docker run` command. - Private Docker registry ----------------------- @@ -118,6 +117,28 @@ Fetching passwords from cloud providers: GCP_PASSWORD = $(gcloud auth print-access-token) AZURE_PASSWORD = $(az acr login --name --expose-token --output tsv) +Reusable Containers (Experimental) +---------------------------------- + +Containers can be reused across consecutive test runs. + +How to use? +^^^^^^^^^^^ + +1. Add `testcontainers.reuse.enable=true` to `~/.testcontainers.properties` +2. Disable ryuk by setting the environment variable `TESTCONTAINERS_RYUK_DISABLED=true` +3. Instantiate a container using `with_reuse` + +.. doctest:: + + >>> from testcontainers.core.container import DockerContainer + + >>> with DockerContainer("hello-world").with_reuse() as container: + ... first_id = container._container.id + >>> with DockerContainer("hello-world").with_reuse() as container: + ... second_id == container._container.id + >>> print(first_id == second_id) + True Configuration ------------- From 08e33baace779761f35cb4f62246ee3f5a6b2304 Mon Sep 17 00:00:00 2001 From: Matthias Schaub Date: Wed, 3 Jul 2024 15:18:12 +0200 Subject: [PATCH 03/12] test: additional testcase for reusable containers --- core/tests/test_reusable_containers.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/core/tests/test_reusable_containers.py b/core/tests/test_reusable_containers.py index a8ab8a9d..c81df7c4 100644 --- a/core/tests/test_reusable_containers.py +++ b/core/tests/test_reusable_containers.py @@ -21,6 +21,7 @@ def test_docker_container_reuse_default(): def test_docker_container_with_reuse_reuse_disabled(): with DockerContainer("hello-world").with_reuse() as container: assert container._reuse == True + assert testcontainers_config.tc_properties_testcontainers_reuse_enable == False id = container._container.id wait_for_logs(container, "Hello from Docker!") containers = DockerClient().client.containers.list(all=True) @@ -30,6 +31,7 @@ def test_docker_container_with_reuse_reuse_disabled(): def test_docker_container_with_reuse_reuse_enabled_ryuk_enabled(monkeypatch): # Make sure Ryuk cleanup is not active from previous test runs Reaper.delete_instance() + tc_properties_mock = testcontainers_config.tc_properties | {"testcontainers.reuse.enable": "true"} monkeypatch.setattr(testcontainers_config, "tc_properties", tc_properties_mock) monkeypatch.setattr(testcontainers_config, "ryuk_reconnection_timeout", "0.1s") @@ -52,11 +54,12 @@ def test_docker_container_with_reuse_reuse_enabled_ryuk_enabled(monkeypatch): def test_docker_container_with_reuse_reuse_enabled_ryuk_disabled(monkeypatch): # Make sure Ryuk cleanup is not active from previous test runs Reaper.delete_instance() + tc_properties_mock = testcontainers_config.tc_properties | {"testcontainers.reuse.enable": "true"} monkeypatch.setattr(testcontainers_config, "tc_properties", tc_properties_mock) monkeypatch.setattr(testcontainers_config, "ryuk_disabled", True) + with DockerContainer("hello-world").with_reuse() as container: - assert container._reuse == True id = container._container.id wait_for_logs(container, "Hello from Docker!") containers = DockerClient().client.containers.list(all=True) @@ -65,6 +68,22 @@ def test_docker_container_with_reuse_reuse_enabled_ryuk_disabled(monkeypatch): container._container.remove(force=True) +def test_docker_container_with_reuse_reuse_enabled_ryuk_disabled_same_id(monkeypatch): + # Make sure Ryuk cleanup is not active from previous test runs + Reaper.delete_instance() + + tc_properties_mock = testcontainers_config.tc_properties | {"testcontainers.reuse.enable": "true"} + monkeypatch.setattr(testcontainers_config, "tc_properties", tc_properties_mock) + monkeypatch.setattr(testcontainers_config, "ryuk_disabled", True) + + with DockerContainer("hello-world").with_reuse() as container: + id = container._container.id + with DockerContainer("hello-world").with_reuse() as container: + assert id == container._container.id + # Cleanup after keeping container alive (with_reuse) + container._container.remove(force=True) + + def test_docker_container_labels_hash(): expected_hash = "91fde3c09244e1d3ec6f18a225b9261396b9a1cb0f6365b39b9795782817c128" with DockerContainer("hello-world").with_reuse() as container: From d2a83bcda6816b0757eff86e12d2e0a804bd6818 Mon Sep 17 00:00:00 2001 From: Matthias Schaub Date: Wed, 3 Jul 2024 15:23:31 +0200 Subject: [PATCH 04/12] test: add newlines for better readability --- core/tests/test_reusable_containers.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/core/tests/test_reusable_containers.py b/core/tests/test_reusable_containers.py index c81df7c4..a834cf23 100644 --- a/core/tests/test_reusable_containers.py +++ b/core/tests/test_reusable_containers.py @@ -62,8 +62,10 @@ def test_docker_container_with_reuse_reuse_enabled_ryuk_disabled(monkeypatch): with DockerContainer("hello-world").with_reuse() as container: id = container._container.id wait_for_logs(container, "Hello from Docker!") + containers = DockerClient().client.containers.list(all=True) assert id in [container.id for container in containers] + # Cleanup after keeping container alive (with_reuse) container._container.remove(force=True) @@ -80,6 +82,7 @@ def test_docker_container_with_reuse_reuse_enabled_ryuk_disabled_same_id(monkeyp id = container._container.id with DockerContainer("hello-world").with_reuse() as container: assert id == container._container.id + # Cleanup after keeping container alive (with_reuse) container._container.remove(force=True) From c781606fcfad7c72960f9a2e570fa25ce2183a65 Mon Sep 17 00:00:00 2001 From: Matthias Schaub Date: Wed, 3 Jul 2024 15:44:07 +0200 Subject: [PATCH 05/12] warn user if ryuk is disabled but with_reuse used --- core/testcontainers/core/container.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/core/testcontainers/core/container.py b/core/testcontainers/core/container.py index caa4c61e..13e364ce 100644 --- a/core/testcontainers/core/container.py +++ b/core/testcontainers/core/container.py @@ -108,12 +108,13 @@ def start(self) -> Self: hash_ = hashlib.sha256(bytes(str(args), encoding="utf-8")).hexdigest() # TODO: check also if ryuk is disabled - if self._reuse and not c.tc_properties_testcontainers_reuse_enable: + if self._reuse and (not c.tc_properties_testcontainers_reuse_enable or not c.ryuk_disabled): logging.warning( "Reuse was requested (`with_reuse`) but the environment does not " + "support the reuse of containers. To enable container reuse, add " - + "the property 'testcontainers.reuse.enable=true' to a file at " - + "~/.testcontainers.properties (you may need to create it)." + + "the 'testcontainers.reuse.enable=true' to " + + "'~/.testcontainers.properties' and disable ryuk by setting the " + + "environment variable 'TESTCONTAINERS_RYUK_DISABLED=true'" ) if self._reuse and c.tc_properties_testcontainers_reuse_enable: From dd429e7d66610e0ab39af4cacb76df5df7efc469 Mon Sep 17 00:00:00 2001 From: Matthias Schaub Date: Wed, 3 Jul 2024 15:45:37 +0200 Subject: [PATCH 06/12] docs: fix code highlighting --- index.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/index.rst b/index.rst index d199da39..f9a7e7dc 100644 --- a/index.rst +++ b/index.rst @@ -125,9 +125,9 @@ Containers can be reused across consecutive test runs. How to use? ^^^^^^^^^^^ -1. Add `testcontainers.reuse.enable=true` to `~/.testcontainers.properties` -2. Disable ryuk by setting the environment variable `TESTCONTAINERS_RYUK_DISABLED=true` -3. Instantiate a container using `with_reuse` +1. Add :code:`testcontainers.reuse.enable=true` to :code:`~/.testcontainers.properties` +2. Disable ryuk by setting the environment variable :code:`TESTCONTAINERS_RYUK_DISABLED=true` +3. Instantiate a container using :code:`with_reuse` .. doctest:: From e87e782fcedb6bb000355133d922b62a5409dd88 Mon Sep 17 00:00:00 2001 From: Matthias Schaub Date: Sun, 7 Jul 2024 10:21:20 +0200 Subject: [PATCH 07/12] fix: use Union instead of | for type hint --- core/testcontainers/core/docker_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/testcontainers/core/docker_client.py b/core/testcontainers/core/docker_client.py index 674b2ed6..418a842f 100644 --- a/core/testcontainers/core/docker_client.py +++ b/core/testcontainers/core/docker_client.py @@ -215,7 +215,7 @@ def client_networks_create(self, name: str, param: dict): labels = create_labels("", param.get("labels")) return self.client.networks.create(name, **{**param, "labels": labels}) - def find_container_by_hash(self, hash_: str) -> Container | None: + def find_container_by_hash(self, hash_: str) -> Union[Container, None]: for container in self.client.containers.list(all=True): if container.labels.get("hash", None) == hash_: return container From c656660f797c0cedc859ab882c051d5832185721 Mon Sep 17 00:00:00 2001 From: Matthias Schaub Date: Mon, 8 Jul 2024 09:40:02 +0200 Subject: [PATCH 08/12] refactor: remove TODO comment --- core/testcontainers/core/container.py | 1 - 1 file changed, 1 deletion(-) diff --git a/core/testcontainers/core/container.py b/core/testcontainers/core/container.py index 13e364ce..bc35b668 100644 --- a/core/testcontainers/core/container.py +++ b/core/testcontainers/core/container.py @@ -107,7 +107,6 @@ def start(self) -> Self: ) hash_ = hashlib.sha256(bytes(str(args), encoding="utf-8")).hexdigest() - # TODO: check also if ryuk is disabled if self._reuse and (not c.tc_properties_testcontainers_reuse_enable or not c.ryuk_disabled): logging.warning( "Reuse was requested (`with_reuse`) but the environment does not " From efb1265ed9435b19f2fc3f48706d2df7db5d448b Mon Sep 17 00:00:00 2001 From: Matthias Schaub Date: Mon, 8 Jul 2024 09:53:55 +0200 Subject: [PATCH 09/12] docs: update section on reusable containers --- index.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/index.rst b/index.rst index f9a7e7dc..865ccfe8 100644 --- a/index.rst +++ b/index.rst @@ -120,7 +120,11 @@ Fetching passwords from cloud providers: Reusable Containers (Experimental) ---------------------------------- -Containers can be reused across consecutive test runs. +Containers can be reused across consecutive test runs. To reuse a container, the container configuration must be the same. + +Containers that are set up for reuse will not be automatically removed. Thus, those containers need to be removed manually. + +Containers should not be reused in a CI environment. How to use? ^^^^^^^^^^^ From d4445d65e2fd2cde0e9ab88d0cdf179a470ce9a6 Mon Sep 17 00:00:00 2001 From: Matthias Schaub Date: Fri, 2 Aug 2024 11:24:12 +0200 Subject: [PATCH 10/12] feat(reuse): do not change contract of stop method --- core/testcontainers/core/container.py | 5 +---- core/tests/test_reusable_containers.py | 26 ++++++++++++++------------ index.rst | 18 +++++++++++------- 3 files changed, 26 insertions(+), 23 deletions(-) diff --git a/core/testcontainers/core/container.py b/core/testcontainers/core/container.py index bc35b668..c2e34284 100644 --- a/core/testcontainers/core/container.py +++ b/core/testcontainers/core/container.py @@ -151,10 +151,7 @@ def _start(self, hash_): def stop(self, force=True, delete_volume=True) -> None: if self._container: - if self._reuse and c.tc_properties_testcontainers_reuse_enable: - self._container.stop() - else: - self._container.remove(force=force, v=delete_volume) + self._container.remove(force=force, v=delete_volume) self.get_docker_client().client.close() def __enter__(self) -> Self: diff --git a/core/tests/test_reusable_containers.py b/core/tests/test_reusable_containers.py index a834cf23..4fbaeb2f 100644 --- a/core/tests/test_reusable_containers.py +++ b/core/tests/test_reusable_containers.py @@ -36,9 +36,9 @@ def test_docker_container_with_reuse_reuse_enabled_ryuk_enabled(monkeypatch): monkeypatch.setattr(testcontainers_config, "tc_properties", tc_properties_mock) monkeypatch.setattr(testcontainers_config, "ryuk_reconnection_timeout", "0.1s") - with DockerContainer("hello-world").with_reuse() as container: - id = container._container.id - wait_for_logs(container, "Hello from Docker!") + container = DockerContainer("hello-world").with_reuse().start() + id = container._container.id + wait_for_logs(container, "Hello from Docker!") Reaper._socket.close() # Sleep until Ryuk reaps all dangling containers @@ -59,15 +59,15 @@ def test_docker_container_with_reuse_reuse_enabled_ryuk_disabled(monkeypatch): monkeypatch.setattr(testcontainers_config, "tc_properties", tc_properties_mock) monkeypatch.setattr(testcontainers_config, "ryuk_disabled", True) - with DockerContainer("hello-world").with_reuse() as container: - id = container._container.id - wait_for_logs(container, "Hello from Docker!") + container = DockerContainer("hello-world").with_reuse().start() + id = container._container.id + wait_for_logs(container, "Hello from Docker!") containers = DockerClient().client.containers.list(all=True) assert id in [container.id for container in containers] # Cleanup after keeping container alive (with_reuse) - container._container.remove(force=True) + container.stop() def test_docker_container_with_reuse_reuse_enabled_ryuk_disabled_same_id(monkeypatch): @@ -78,13 +78,15 @@ def test_docker_container_with_reuse_reuse_enabled_ryuk_disabled_same_id(monkeyp monkeypatch.setattr(testcontainers_config, "tc_properties", tc_properties_mock) monkeypatch.setattr(testcontainers_config, "ryuk_disabled", True) - with DockerContainer("hello-world").with_reuse() as container: - id = container._container.id - with DockerContainer("hello-world").with_reuse() as container: - assert id == container._container.id + container_1 = DockerContainer("hello-world").with_reuse().start() + id_1 = container_1._container.id + container_2 = DockerContainer("hello-world").with_reuse().start() + id_2 = container_2._container.id + assert id_1 == id_2 # Cleanup after keeping container alive (with_reuse) - container._container.remove(force=True) + container_1.stop() + # container_2.stop() is not needed since it is the same as container_1 def test_docker_container_labels_hash(): diff --git a/index.rst b/index.rst index 865ccfe8..00e6dc80 100644 --- a/index.rst +++ b/index.rst @@ -120,9 +120,13 @@ Fetching passwords from cloud providers: Reusable Containers (Experimental) ---------------------------------- -Containers can be reused across consecutive test runs. To reuse a container, the container configuration must be the same. +.. warning:: + Reusable Containers is still an experimental feature and the behavior can change. + Those containers won't stop after all tests are finished. -Containers that are set up for reuse will not be automatically removed. Thus, those containers need to be removed manually. +Containers can be reused across consecutive test runs. To reuse a container, the container has to be started manually by calling the `start()` method. Do not call the `stop()` method directly or indirectly via a `with` statement (context manager). To reuse a container, the container configuration must be the same. + +Containers that are set up for reuse will not be automatically removed. Thus, if they are not needed anymore, those containers must be removed manually. Containers should not be reused in a CI environment. @@ -131,16 +135,16 @@ How to use? 1. Add :code:`testcontainers.reuse.enable=true` to :code:`~/.testcontainers.properties` 2. Disable ryuk by setting the environment variable :code:`TESTCONTAINERS_RYUK_DISABLED=true` -3. Instantiate a container using :code:`with_reuse` +3. Instantiate a container using :code:`with_reuse()` and :code:`start()` .. doctest:: >>> from testcontainers.core.container import DockerContainer - >>> with DockerContainer("hello-world").with_reuse() as container: - ... first_id = container._container.id - >>> with DockerContainer("hello-world").with_reuse() as container: - ... second_id == container._container.id + >>> container = DockerContainer("hello-world").with_reuse().start() + >>> first_id = container._container.id + >>> container = DockerContainer("hello-world").with_reuse().start() + >>> second_id == container._container.id >>> print(first_id == second_id) True From 1ea9ed16a0dd73e11d762808aed5655d44e270be Mon Sep 17 00:00:00 2001 From: Matthias Schaub Date: Fri, 2 Aug 2024 13:51:55 +0200 Subject: [PATCH 11/12] feat(reuse): do not create Ryuk cleanup instance do not create Ryuk cleanup instance if reuse enabled and container has been start with `with_reuse` --- core/testcontainers/core/container.py | 12 +++-- core/tests/test_reusable_containers.py | 71 +++++++++++++++----------- 2 files changed, 49 insertions(+), 34 deletions(-) diff --git a/core/testcontainers/core/container.py b/core/testcontainers/core/container.py index c2e34284..fb6e1691 100644 --- a/core/testcontainers/core/container.py +++ b/core/testcontainers/core/container.py @@ -89,7 +89,11 @@ def maybe_emulate_amd64(self) -> Self: return self def start(self) -> Self: - if not c.ryuk_disabled and self.image != c.ryuk_image: + if ( + not c.ryuk_disabled + and self.image != c.ryuk_image + and not (self._reuse and c.tc_properties_testcontainers_reuse_enable) + ): logger.debug("Creating Ryuk container") Reaper.get_instance() logger.info("Pulling image %s", self.image) @@ -107,13 +111,11 @@ def start(self) -> Self: ) hash_ = hashlib.sha256(bytes(str(args), encoding="utf-8")).hexdigest() - if self._reuse and (not c.tc_properties_testcontainers_reuse_enable or not c.ryuk_disabled): + if self._reuse and not c.tc_properties_testcontainers_reuse_enable: logging.warning( "Reuse was requested (`with_reuse`) but the environment does not " + "support the reuse of containers. To enable container reuse, add " - + "the 'testcontainers.reuse.enable=true' to " - + "'~/.testcontainers.properties' and disable ryuk by setting the " - + "environment variable 'TESTCONTAINERS_RYUK_DISABLED=true'" + + "'testcontainers.reuse.enable=true' to '~/.testcontainers.properties'." ) if self._reuse and c.tc_properties_testcontainers_reuse_enable: diff --git a/core/tests/test_reusable_containers.py b/core/tests/test_reusable_containers.py index 4fbaeb2f..8f2579cf 100644 --- a/core/tests/test_reusable_containers.py +++ b/core/tests/test_reusable_containers.py @@ -10,62 +10,75 @@ def test_docker_container_reuse_default(): - with DockerContainer("hello-world") as container: - assert container._reuse == False - id = container._container.id - wait_for_logs(container, "Hello from Docker!") + # Make sure Ryuk cleanup is not active from previous test runs + Reaper.delete_instance() + + container = DockerContainer("hello-world").start() + wait_for_logs(container, "Hello from Docker!") + + assert container._reuse == False + assert testcontainers_config.tc_properties_testcontainers_reuse_enable == False + assert Reaper._socket is not None + + container.stop() containers = DockerClient().client.containers.list(all=True) - assert id not in [container.id for container in containers] + assert container._container.id not in [container.id for container in containers] -def test_docker_container_with_reuse_reuse_disabled(): - with DockerContainer("hello-world").with_reuse() as container: - assert container._reuse == True - assert testcontainers_config.tc_properties_testcontainers_reuse_enable == False - id = container._container.id - wait_for_logs(container, "Hello from Docker!") +def test_docker_container_with_reuse_reuse_disabled(caplog): + # Make sure Ryuk cleanup is not active from previous test runs + Reaper.delete_instance() + + container = DockerContainer("hello-world").with_reuse().start() + wait_for_logs(container, "Hello from Docker!") + + assert container._reuse == True + assert testcontainers_config.tc_properties_testcontainers_reuse_enable == False + assert ( + "Reuse was requested (`with_reuse`) but the environment does not support the " + + "reuse of containers. To enable container reuse, add " + + "'testcontainers.reuse.enable=true' to '~/.testcontainers.properties'." + ) in caplog.text + assert Reaper._socket is not None + + container.stop() containers = DockerClient().client.containers.list(all=True) - assert id not in [container.id for container in containers] + assert container._container.id not in [container.id for container in containers] -def test_docker_container_with_reuse_reuse_enabled_ryuk_enabled(monkeypatch): +def test_docker_container_without_reuse_reuse_enabled(monkeypatch): # Make sure Ryuk cleanup is not active from previous test runs Reaper.delete_instance() tc_properties_mock = testcontainers_config.tc_properties | {"testcontainers.reuse.enable": "true"} monkeypatch.setattr(testcontainers_config, "tc_properties", tc_properties_mock) - monkeypatch.setattr(testcontainers_config, "ryuk_reconnection_timeout", "0.1s") - container = DockerContainer("hello-world").with_reuse().start() - id = container._container.id + container = DockerContainer("hello-world").start() wait_for_logs(container, "Hello from Docker!") - Reaper._socket.close() - # Sleep until Ryuk reaps all dangling containers - sleep(0.6) + assert container._reuse == False + assert testcontainers_config.tc_properties_testcontainers_reuse_enable == True + assert Reaper._socket is not None + container.stop() containers = DockerClient().client.containers.list(all=True) - assert id not in [container.id for container in containers] - - # Cleanup Ryuk class fields after manual Ryuk shutdown - Reaper.delete_instance() + assert container._container.id not in [container.id for container in containers] -def test_docker_container_with_reuse_reuse_enabled_ryuk_disabled(monkeypatch): +def test_docker_container_with_reuse_reuse_enabled(monkeypatch): # Make sure Ryuk cleanup is not active from previous test runs Reaper.delete_instance() tc_properties_mock = testcontainers_config.tc_properties | {"testcontainers.reuse.enable": "true"} monkeypatch.setattr(testcontainers_config, "tc_properties", tc_properties_mock) - monkeypatch.setattr(testcontainers_config, "ryuk_disabled", True) container = DockerContainer("hello-world").with_reuse().start() - id = container._container.id wait_for_logs(container, "Hello from Docker!") - containers = DockerClient().client.containers.list(all=True) - assert id in [container.id for container in containers] + assert Reaper._socket is None + containers = DockerClient().client.containers.list(all=True) + assert container._container.id in [container.id for container in containers] # Cleanup after keeping container alive (with_reuse) container.stop() @@ -82,8 +95,8 @@ def test_docker_container_with_reuse_reuse_enabled_ryuk_disabled_same_id(monkeyp id_1 = container_1._container.id container_2 = DockerContainer("hello-world").with_reuse().start() id_2 = container_2._container.id + assert Reaper._socket is None assert id_1 == id_2 - # Cleanup after keeping container alive (with_reuse) container_1.stop() # container_2.stop() is not needed since it is the same as container_1 From ea6fec7cad3355ee80547fd9ec40b383681411f9 Mon Sep 17 00:00:00 2001 From: Matthias Schaub Date: Sat, 3 Aug 2024 10:09:25 +0200 Subject: [PATCH 12/12] refactor: move hash generation into if clause --- core/testcontainers/core/container.py | 32 +++++++++++++------------- core/tests/test_reusable_containers.py | 19 +++++++++++---- 2 files changed, 30 insertions(+), 21 deletions(-) diff --git a/core/testcontainers/core/container.py b/core/testcontainers/core/container.py index fb6e1691..8af3754d 100644 --- a/core/testcontainers/core/container.py +++ b/core/testcontainers/core/container.py @@ -99,18 +99,6 @@ def start(self) -> Self: logger.info("Pulling image %s", self.image) self._configure() - # container hash consisting of run arguments - args = ( - self.image, - self._command, - self.env, - self.ports, - self._name, - self.volumes, - str(tuple(sorted(self._kwargs.items()))), - ) - hash_ = hashlib.sha256(bytes(str(args), encoding="utf-8")).hexdigest() - if self._reuse and not c.tc_properties_testcontainers_reuse_enable: logging.warning( "Reuse was requested (`with_reuse`) but the environment does not " @@ -119,24 +107,36 @@ def start(self) -> Self: ) if self._reuse and c.tc_properties_testcontainers_reuse_enable: + # NOTE: ideally the docker client would return the full container create + # request which could be used to generate the hash. + args = [ # Docker run arguments + self.image, + self._command, + self.env, + self.ports, + self._name, + self.volumes, + str(tuple(sorted(self._kwargs.values()))), + ] + hash_ = hashlib.sha256(bytes(str(args), encoding="utf-8")).hexdigest() docker_client = self.get_docker_client() container = docker_client.find_container_by_hash(hash_) if container: if container.status != "running": container.start() logger.info("Existing container started: %s", container.id) - logger.info("Container is already running: %s", container.id) self._container = container + logger.info("Container is already running: %s", container.id) else: self._start(hash_) else: - self._start(hash_) + self._start() if self._network: self._network.connect(self._container.id, self._network_aliases) return self - def _start(self, hash_): + def _start(self, hash_=None): docker_client = self.get_docker_client() self._container = docker_client.run( self.image, @@ -146,7 +146,7 @@ def _start(self, hash_): ports=self.ports, name=self._name, volumes=self.volumes, - labels={"hash": hash_}, + labels={"hash": hash_} if hash is not None else {}, **self._kwargs, ) logger.info("Container started: %s", self._container.short_id) diff --git a/core/tests/test_reusable_containers.py b/core/tests/test_reusable_containers.py index 8f2579cf..6c956379 100644 --- a/core/tests/test_reusable_containers.py +++ b/core/tests/test_reusable_containers.py @@ -83,13 +83,12 @@ def test_docker_container_with_reuse_reuse_enabled(monkeypatch): container.stop() -def test_docker_container_with_reuse_reuse_enabled_ryuk_disabled_same_id(monkeypatch): +def test_docker_container_with_reuse_reuse_enabled_same_id(monkeypatch): # Make sure Ryuk cleanup is not active from previous test runs Reaper.delete_instance() tc_properties_mock = testcontainers_config.tc_properties | {"testcontainers.reuse.enable": "true"} monkeypatch.setattr(testcontainers_config, "tc_properties", tc_properties_mock) - monkeypatch.setattr(testcontainers_config, "ryuk_disabled", True) container_1 = DockerContainer("hello-world").with_reuse().start() id_1 = container_1._container.id @@ -102,8 +101,16 @@ def test_docker_container_with_reuse_reuse_enabled_ryuk_disabled_same_id(monkeyp # container_2.stop() is not needed since it is the same as container_1 -def test_docker_container_labels_hash(): - expected_hash = "91fde3c09244e1d3ec6f18a225b9261396b9a1cb0f6365b39b9795782817c128" +def test_docker_container_labels_hash_default(): + # w/out reuse + with DockerContainer("hello-world") as container: + assert container._container.labels["hash"] == "" + + +def test_docker_container_labels_hash(monkeypatch): + tc_properties_mock = testcontainers_config.tc_properties | {"testcontainers.reuse.enable": "true"} + monkeypatch.setattr(testcontainers_config, "tc_properties", tc_properties_mock) + expected_hash = "1bade17a9d8236ba71ffbb676f2ece3fb419ea0e6adb5f82b5a026213c431d8e" with DockerContainer("hello-world").with_reuse() as container: assert container._container.labels["hash"] == expected_hash @@ -113,7 +120,9 @@ def test_docker_client_find_container_by_hash_not_existing(): assert DockerClient().find_container_by_hash("foo") == None -def test_docker_client_find_container_by_hash_existing(): +def test_docker_client_find_container_by_hash_existing(monkeypatch): + tc_properties_mock = testcontainers_config.tc_properties | {"testcontainers.reuse.enable": "true"} + monkeypatch.setattr(testcontainers_config, "tc_properties", tc_properties_mock) with DockerContainer("hello-world").with_reuse() as container: hash_ = container._container.labels["hash"] found_container = DockerClient().find_container_by_hash(hash_)