diff --git a/.github/.release-please-manifest.json b/.github/.release-please-manifest.json index 98ecd87e..0cd2cc7e 100644 --- a/.github/.release-please-manifest.json +++ b/.github/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "4.8.2" + ".": "4.9.0" } diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 12c8d19f..1b1f6b92 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -39,7 +39,7 @@ You need to have the following tools available to you: ## Adding new containers -We have an [issue template](.github/ISSUE_TEMPLATE/new-container.md) for adding new containers, please refer to that for more information. +We have an [issue template](./ISSUE_TEMPLATE/new-container.md) for adding new containers, please refer to that for more information. Once you've talked to the maintainers (we do our best to reply!) then you can proceed with contributing the new container. > [!WARNING] diff --git a/CHANGELOG.md b/CHANGELOG.md index 9372aeeb..c6cb6856 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,28 @@ # Changelog +## [4.9.0](https://github.com/testcontainers/testcontainers-python/compare/testcontainers-v4.8.2...testcontainers-v4.9.0) (2024-11-26) + + +### Features + +* **compose:** support for setting profiles ([#738](https://github.com/testcontainers/testcontainers-python/issues/738)) ([3e00e71](https://github.com/testcontainers/testcontainers-python/commit/3e00e71da4d2b5e7fd30315468d4e54c86ba6150)) +* **core:** Support working with env files ([#737](https://github.com/testcontainers/testcontainers-python/issues/737)) ([932ee30](https://github.com/testcontainers/testcontainers-python/commit/932ee307955e3591a63f194aee8e2f6d8e2f6bf9)) + + +### Bug Fixes + +* allow running all tests ([#721](https://github.com/testcontainers/testcontainers-python/issues/721)) ([f958cf9](https://github.com/testcontainers/testcontainers-python/commit/f958cf9fe62a5f3ee2dc255713ec8b16de6a767d)) +* **core:** Avoid hanging upon bad docker host connection ([#742](https://github.com/testcontainers/testcontainers-python/issues/742)) ([4ced198](https://github.com/testcontainers/testcontainers-python/commit/4ced1983162914fe511a6e714f136b670e1dbdfb)) +* **core:** running testcontainer inside container ([#714](https://github.com/testcontainers/testcontainers-python/issues/714)) ([85a6666](https://github.com/testcontainers/testcontainers-python/commit/85a66667c23d76e87aecc6761bbb01429adb3dee)) +* **generic:** Also catch URLError waiting for ServerContainer ([#743](https://github.com/testcontainers/testcontainers-python/issues/743)) ([24e354f](https://github.com/testcontainers/testcontainers-python/commit/24e354f3bfa5912eaf7877da9442a885d7872f1a)) +* update wait_for_logs to not throw on 'created', and an optimization ([#719](https://github.com/testcontainers/testcontainers-python/issues/719)) ([271ca9a](https://github.com/testcontainers/testcontainers-python/commit/271ca9a0fef2e5f2b216457bfee44318e93990bf)) +* Vault health check ([#734](https://github.com/testcontainers/testcontainers-python/issues/734)) ([79434d6](https://github.com/testcontainers/testcontainers-python/commit/79434d6744b2918493884cf8fbf27aeadf78ecfd)) + + +### Documentation + +* Documentation fix for ServerContainer ([#671](https://github.com/testcontainers/testcontainers-python/issues/671)) ([0303d47](https://github.com/testcontainers/testcontainers-python/commit/0303d47d7173e1c4ec1a4f565efee9b2fe694928)) + ## [4.8.2](https://github.com/testcontainers/testcontainers-python/compare/testcontainers-v4.8.1...testcontainers-v4.8.2) (2024-09-27) diff --git a/core/testcontainers/compose/compose.py b/core/testcontainers/compose/compose.py index 564eda8f..e8ce3745 100644 --- a/core/testcontainers/compose/compose.py +++ b/core/testcontainers/compose/compose.py @@ -171,6 +171,7 @@ class DockerCompose: env_file: Optional[str] = None services: Optional[list[str]] = None docker_command_path: Optional[str] = None + profiles: Optional[list[str]] = None def __post_init__(self): if isinstance(self.compose_file_name, str): @@ -198,6 +199,8 @@ def compose_command_property(self) -> list[str]: if self.compose_file_name: for file in self.compose_file_name: docker_compose_cmd += ["-f", file] + if self.profiles: + docker_compose_cmd += [item for profile in self.profiles for item in ["--profile", profile]] if self.env_file: docker_compose_cmd += ["--env-file", self.env_file] return docker_compose_cmd diff --git a/core/testcontainers/core/container.py b/core/testcontainers/core/container.py index c05e4ceb..fabd9a31 100644 --- a/core/testcontainers/core/container.py +++ b/core/testcontainers/core/container.py @@ -1,16 +1,18 @@ import contextlib +from os import PathLike from socket import socket from typing import TYPE_CHECKING, Optional, Union import docker.errors from docker import version from docker.types import EndpointConfig +from dotenv import dotenv_values from typing_extensions import Self, assert_never from testcontainers.core.config import ConnectionMode from testcontainers.core.config import testcontainers_config as c from testcontainers.core.docker_client import DockerClient -from testcontainers.core.exceptions import ContainerStartException +from testcontainers.core.exceptions import ContainerConnectException, ContainerStartException from testcontainers.core.labels import LABEL_SESSION_ID, SESSION_ID from testcontainers.core.network import Network from testcontainers.core.utils import is_arm, setup_logger @@ -56,6 +58,12 @@ def __init__( def with_env(self, key: str, value: str) -> Self: self.env[key] = value return self + + def with_env_file(self, env_file: Union[str, PathLike]) -> Self: + env_values = dotenv_values(env_file) + for key, value in env_values.items(): + self.with_env(key, value) + return self def with_bind_ports(self, container: Union[str, int], host: Optional[Union[str, int]] = None) -> Self: """ @@ -72,7 +80,6 @@ def with_bind_ports(self, container: Union[str, int], host: Optional[Union[str, >>> container = container.with_bind_ports("8081/tcp", 8081) """ - self.ports[container] = host return self @@ -248,15 +255,21 @@ def _create_instance(cls) -> "Reaper": .with_env("RYUK_RECONNECTION_TIMEOUT", c.ryuk_reconnection_timeout) .start() ) - wait_for_logs(Reaper._container, r".* Started!") + wait_for_logs(Reaper._container, r".* Started!", timeout=20, raise_on_exit=True) container_host = Reaper._container.get_container_host_ip() container_port = int(Reaper._container.get_exposed_port(8080)) + if not container_host or not container_port: + raise ContainerConnectException( + f"Could not obtain network details for {Reaper._container._container.id}. Host: {container_host} Port: {container_port}" + ) + last_connection_exception: Optional[Exception] = None for _ in range(50): try: Reaper._socket = socket() + Reaper._socket.settimeout(1) Reaper._socket.connect((container_host, container_port)) last_connection_exception = None break diff --git a/core/testcontainers/core/exceptions.py b/core/testcontainers/core/exceptions.py index 6694e598..99ea9881 100644 --- a/core/testcontainers/core/exceptions.py +++ b/core/testcontainers/core/exceptions.py @@ -16,6 +16,10 @@ class ContainerStartException(RuntimeError): pass +class ContainerConnectException(RuntimeError): + pass + + class ContainerIsNotRunning(RuntimeError): pass diff --git a/core/tests/compose_fixtures/profile_support/compose.yaml b/core/tests/compose_fixtures/profile_support/compose.yaml new file mode 100644 index 00000000..c7bec7cc --- /dev/null +++ b/core/tests/compose_fixtures/profile_support/compose.yaml @@ -0,0 +1,16 @@ +services: + runs-always: &simple-service + image: alpine:latest + init: true + command: + - sh + - -c + - 'while true; do sleep 0.1 ; date -Ins; done' + runs-profile-a: + <<: *simple-service + profiles: + - profile-a + runs-profile-b: + <<: *simple-service + profiles: + - profile-b diff --git a/core/tests/test_compose.py b/core/tests/test_compose.py index b43da28c..9279ce3f 100644 --- a/core/tests/test_compose.py +++ b/core/tests/test_compose.py @@ -2,7 +2,7 @@ from pathlib import Path from re import split from time import sleep -from typing import Union +from typing import Union, Optional from urllib.request import urlopen, Request import pytest @@ -352,3 +352,27 @@ def fetch(req: Union[Request, str]): if 200 < res.getcode() >= 400: raise Exception(f"HTTP Error: {res.getcode()} - {res.reason}: {body}") return res.getcode(), body + + +@pytest.mark.parametrize( + argnames=["profiles", "running", "not_running"], + argvalues=[ + pytest.param(None, ["runs-always"], ["runs-profile-a", "runs-profile-b"], id="default"), + pytest.param( + ["profile-a"], ["runs-always", "runs-profile-a"], ["runs-profile-b"], id="one-additional-profile-via-str" + ), + pytest.param( + ["profile-a", "profile-b"], + ["runs-always", "runs-profile-a", "runs-profile-b"], + [], + id="all-profiles-explicitly", + ), + ], +) +def test_compose_profile_support(profiles: Optional[list[str]], running: list[str], not_running: list[str]): + with DockerCompose(context=FIXTURES / "profile_support", profiles=profiles) as compose: + for service in running: + assert compose.get_container(service) is not None + for service in not_running: + with pytest.raises(ContainerIsNotRunning): + compose.get_container(service) diff --git a/core/tests/test_core.py b/core/tests/test_core.py index 42321e28..9312b0bc 100644 --- a/core/tests/test_core.py +++ b/core/tests/test_core.py @@ -1,3 +1,6 @@ +import tempfile +from pathlib import Path + from testcontainers.core.container import DockerContainer @@ -17,3 +20,29 @@ def test_get_logs(): assert isinstance(stdout, bytes) assert isinstance(stderr, bytes) assert "Hello from Docker".encode() in stdout, "There should be something on stdout" + + +def test_docker_container_with_env_file(): + """Test that environment variables can be loaded from a file""" + with tempfile.TemporaryDirectory() as temp_directory: + env_file_path = Path(temp_directory) / "env_file" + with open(env_file_path, "w") as f: + f.write( + """ + TEST_ENV_VAR=hello + NUMBER=123 + DOMAIN=example.org + ADMIN_EMAIL=admin@${DOMAIN} + ROOT_URL=${DOMAIN}/app + """ + ) + container = DockerContainer("alpine").with_command("tail -f /dev/null") # Keep the container running + container.with_env_file(env_file_path) # Load the environment variables from the file + with container: + output = container.exec("env").output.decode("utf-8").strip() + assert "TEST_ENV_VAR=hello" in output + assert "NUMBER=123" in output + assert "DOMAIN=example.org" in output + assert "ADMIN_EMAIL=admin@example.org" in output + assert "ROOT_URL=example.org/app" in output + print(output) diff --git a/modules/generic/testcontainers/generic/server.py b/modules/generic/testcontainers/generic/server.py index 03a54677..fe990f17 100644 --- a/modules/generic/testcontainers/generic/server.py +++ b/modules/generic/testcontainers/generic/server.py @@ -1,5 +1,5 @@ from typing import Union -from urllib.error import HTTPError +from urllib.error import HTTPError, URLError from urllib.request import urlopen import httpx @@ -31,8 +31,8 @@ class ServerContainer(DockerContainer): ... delay = wait_for_logs(srv, "GET / HTTP/1.1") - :param path: Path to the Dockerfile to build the image - :param tag: Tag for the image to be built (default: None) + :param port: Port to be exposed on the container. + :param image: Docker image to be used for the container. """ def __init__(self, port: int, image: Union[str, DockerImage]) -> None: @@ -40,7 +40,7 @@ def __init__(self, port: int, image: Union[str, DockerImage]) -> None: self.internal_port = port self.with_exposed_ports(self.internal_port) - @wait_container_is_ready(HTTPError) + @wait_container_is_ready(HTTPError, URLError) def _connect(self) -> None: # noinspection HttpUrlsUsage url = self._create_connection_url() diff --git a/modules/mysql/testcontainers/mysql/__init__.py b/modules/mysql/testcontainers/mysql/__init__.py index c4449593..4c381ff9 100644 --- a/modules/mysql/testcontainers/mysql/__init__.py +++ b/modules/mysql/testcontainers/mysql/__init__.py @@ -31,14 +31,14 @@ class MySqlContainer(DbContainer): The example will spin up a MySql database to which you can connect with the credentials passed in the constructor. Alternatively, you may use the :code:`get_connection_url()` method which returns a sqlalchemy-compatible url in format - :code:`dialect+driver://username:password@host:port/database`. + :code:`mysql+dialect://username:password@host:port/database`. .. doctest:: >>> import sqlalchemy >>> from testcontainers.mysql import MySqlContainer - >>> with MySqlContainer('mysql:5.7.17') as mysql: + >>> with MySqlContainer("mysql:5.7.17", dialect="pymysql") as mysql: ... engine = sqlalchemy.create_engine(mysql.get_connection_url()) ... with engine.begin() as connection: ... result = connection.execute(sqlalchemy.text("select version()")) @@ -64,6 +64,7 @@ class MySqlContainer(DbContainer): def __init__( self, image: str = "mysql:latest", + dialect: Optional[str] = None, username: Optional[str] = None, root_password: Optional[str] = None, password: Optional[str] = None, @@ -72,6 +73,10 @@ def __init__( seed: Optional[str] = None, **kwargs, ) -> None: + if dialect is not None and dialect.startswith("mysql+"): + msg = "Please remove 'mysql+' prefix from dialect parameter" + raise ValueError(msg) + raise_for_deprecated_parameter(kwargs, "MYSQL_USER", "username") raise_for_deprecated_parameter(kwargs, "MYSQL_ROOT_PASSWORD", "root_password") raise_for_deprecated_parameter(kwargs, "MYSQL_PASSWORD", "password") @@ -85,6 +90,9 @@ def __init__( self.password = password or environ.get("MYSQL_PASSWORD", "test") self.dbname = dbname or environ.get("MYSQL_DATABASE", "test") + self.dialect = dialect or environ.get("MYSQL_DIALECT", None) + self._db_url_dialect_part = "mysql" if self.dialect is None else f"mysql+{self.dialect}" + if self.username == "root": self.root_password = self.password self.seed = seed @@ -105,7 +113,11 @@ def _connect(self) -> None: def get_connection_url(self) -> str: return super()._create_connection_url( - dialect="mysql+pymysql", username=self.username, password=self.password, dbname=self.dbname, port=self.port + dialect=self._db_url_dialect_part, + username=self.username, + password=self.password, + dbname=self.dbname, + port=self.port, ) def _transfer_seed(self) -> None: diff --git a/modules/mysql/tests/test_mysql.py b/modules/mysql/tests/test_mysql.py index af0b4491..a2d2c2ec 100644 --- a/modules/mysql/tests/test_mysql.py +++ b/modules/mysql/tests/test_mysql.py @@ -11,9 +11,14 @@ @pytest.mark.inside_docker_check def test_docker_run_mysql(): - config = MySqlContainer("mysql:8.3.0") + config = MySqlContainer("mysql:8.3.0", dialect="pymysql") with config as mysql: - engine = sqlalchemy.create_engine(mysql.get_connection_url()) + connection_url = mysql.get_connection_url() + + assert mysql.dialect == "pymysql" + assert connection_url.startswith("mysql+pymysql://") + + engine = sqlalchemy.create_engine(connection_url) with engine.begin() as connection: result = connection.execute(sqlalchemy.text("select version()")) for row in result: @@ -22,7 +27,7 @@ def test_docker_run_mysql(): @pytest.mark.skipif(is_arm(), reason="mysql container not available for ARM") def test_docker_run_legacy_mysql(): - config = MySqlContainer("mysql:5.7.44") + config = MySqlContainer("mysql:5.7.44", dialect="pymysql") with config as mysql: engine = sqlalchemy.create_engine(mysql.get_connection_url()) with engine.begin() as connection: @@ -35,7 +40,7 @@ def test_docker_run_legacy_mysql(): def test_docker_run_mysql_8_seed(): # Avoid pytest CWD path issues SEEDS_PATH = (Path(__file__).parent / "seeds").absolute() - config = MySqlContainer("mysql:8", seed=SEEDS_PATH) + config = MySqlContainer("mysql:8", dialect="pymysql", seed=str(SEEDS_PATH)) with config as mysql: engine = sqlalchemy.create_engine(mysql.get_connection_url()) with engine.begin() as connection: @@ -45,7 +50,7 @@ def test_docker_run_mysql_8_seed(): @pytest.mark.parametrize("version", ["11.3.2", "10.11.7"]) def test_docker_run_mariadb(version: str): - with MySqlContainer(f"mariadb:{version}") as mariadb: + with MySqlContainer(f"mariadb:{version}", dialect="pymysql") as mariadb: engine = sqlalchemy.create_engine(mariadb.get_connection_url()) with engine.begin() as connection: result = connection.execute(sqlalchemy.text("select version()")) @@ -55,7 +60,7 @@ def test_docker_run_mariadb(version: str): def test_docker_env_variables(): with ( - mock.patch.dict("os.environ", MYSQL_USER="demo", MYSQL_DATABASE="custom_db"), + mock.patch.dict("os.environ", MYSQL_DIALECT="pymysql", MYSQL_USER="demo", MYSQL_DATABASE="custom_db"), MySqlContainer("mariadb:10.6.5").with_bind_ports(3306, 32785) as container, ): url = container.get_connection_url() @@ -63,6 +68,21 @@ def test_docker_env_variables(): assert re.match(pattern, url) +@pytest.mark.parametrize( + "dialect", + [ + "mysql+pymysql", + "mysql+mariadb", + "mysql+mysqldb", + ], +) +def test_mysql_dialect_expecting_error_on_mysql_prefix(dialect: str): + match = f"Please remove *.* prefix from dialect parameter" + + with pytest.raises(ValueError, match=match): + _ = MySqlContainer("mariadb:10.6.5", dialect=dialect) + + # This is a feature in the generic DbContainer class # but it can't be tested on its own # so is tested in various database modules: @@ -75,18 +95,18 @@ def test_quoted_password(): user = "root" password = "p@$%25+0&%rd :/!=?" quoted_password = "p%40%24%2525+0%26%25rd %3A%2F%21%3D%3F" - driver = "pymysql" - with MySqlContainer("mariadb:10.6.5", username=user, password=password) as container: + dialect = "pymysql" + with MySqlContainer("mariadb:10.6.5", dialect=dialect, username=user, password=password) as container: host = container.get_container_host_ip() port = container.get_exposed_port(3306) - expected_url = f"mysql+{driver}://{user}:{quoted_password}@{host}:{port}/test" + expected_url = f"mysql+{dialect}://{user}:{quoted_password}@{host}:{port}/test" url = container.get_connection_url() assert url == expected_url with sqlalchemy.create_engine(expected_url).begin() as connection: connection.execute(sqlalchemy.text("select version()")) - raw_pass_url = f"mysql+{driver}://{user}:{password}@{host}:{port}/test" + raw_pass_url = f"mysql+{dialect}://{user}:{password}@{host}:{port}/test" with pytest.raises(Exception): with sqlalchemy.create_engine(raw_pass_url).begin() as connection: connection.execute(sqlalchemy.text("select version()")) diff --git a/modules/vault/testcontainers/vault/__init__.py b/modules/vault/testcontainers/vault/__init__.py index 5f50cdd4..c89b27ea 100644 --- a/modules/vault/testcontainers/vault/__init__.py +++ b/modules/vault/testcontainers/vault/__init__.py @@ -12,6 +12,7 @@ # under the License. from http.client import HTTPException +from urllib.error import URLError from urllib.request import urlopen from testcontainers.core.container import DockerContainer @@ -61,7 +62,7 @@ def get_connection_url(self) -> str: exposed_port = self.get_exposed_port(self.port) return f"http://{host_ip}:{exposed_port}" - @wait_container_is_ready(HTTPException) + @wait_container_is_ready(HTTPException, URLError) def _healthcheck(self) -> None: url = f"{self.get_connection_url()}/v1/sys/health" with urlopen(url) as res: diff --git a/poetry.lock b/poetry.lock index b83de36f..bd54659e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -4678,4 +4678,4 @@ weaviate = ["weaviate-client"] [metadata] lock-version = "2.0" python-versions = ">=3.9,<4.0" -content-hash = "7ffcf39257e1ac79d951b7ccb2cdb972beaaef35d0cd1f722d2964c5bdced674" +content-hash = "5c400cc87dc9708588ee8d7d50646de789235732d868b74ebc43f1cf2a403c88" diff --git a/pyproject.toml b/pyproject.toml index bd4e7ed0..8bbf82ea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "testcontainers" -version = "4.8.2" # auto-incremented by release-please +version = "4.9.0" # auto-incremented by release-please description = "Python library for throwaway instances of anything that can run in a Docker container" authors = ["Sergey Pirogov "] maintainers = [ @@ -83,6 +83,7 @@ docker = "*" # ">=4.0" urllib3 = "*" # "<2.0" wrapt = "*" # "^1.16.0" typing-extensions = "*" +python-dotenv = "*" # community modules python-arango = { version = "^7.8", optional = true }