Merge branch 'main' into typed_docker_client

.github/.release-please-manifest.json

@@ -1,3 +1,3 @@
 {
-    ".": "4.8.2"
+    ".": "4.9.0"
 }

CHANGELOG.md

@@ -1,5 +1,28 @@
 # Changelog
 
+## [4.9.0](https://github.com/testcontainers/testcontainers-python/compare/testcontainers-v4.8.2...testcontainers-v4.9.0) (2024-11-26)
+
+
+### Features
+
+* **compose:** support for setting profiles ([#738](https://github.com/testcontainers/testcontainers-python/issues/738)) ([3e00e71](https://github.com/testcontainers/testcontainers-python/commit/3e00e71da4d2b5e7fd30315468d4e54c86ba6150))
+* **core:** Support working with env files ([#737](https://github.com/testcontainers/testcontainers-python/issues/737)) ([932ee30](https://github.com/testcontainers/testcontainers-python/commit/932ee307955e3591a63f194aee8e2f6d8e2f6bf9))
+
+
+### Bug Fixes
+
+* allow running all tests ([#721](https://github.com/testcontainers/testcontainers-python/issues/721)) ([f958cf9](https://github.com/testcontainers/testcontainers-python/commit/f958cf9fe62a5f3ee2dc255713ec8b16de6a767d))
+* **core:** Avoid hanging upon bad docker host connection ([#742](https://github.com/testcontainers/testcontainers-python/issues/742)) ([4ced198](https://github.com/testcontainers/testcontainers-python/commit/4ced1983162914fe511a6e714f136b670e1dbdfb))
+* **core:** running testcontainer inside container ([#714](https://github.com/testcontainers/testcontainers-python/issues/714)) ([85a6666](https://github.com/testcontainers/testcontainers-python/commit/85a66667c23d76e87aecc6761bbb01429adb3dee))
+* **generic:** Also catch URLError waiting for ServerContainer ([#743](https://github.com/testcontainers/testcontainers-python/issues/743)) ([24e354f](https://github.com/testcontainers/testcontainers-python/commit/24e354f3bfa5912eaf7877da9442a885d7872f1a))
+* update wait_for_logs to not throw on 'created', and an optimization ([#719](https://github.com/testcontainers/testcontainers-python/issues/719)) ([271ca9a](https://github.com/testcontainers/testcontainers-python/commit/271ca9a0fef2e5f2b216457bfee44318e93990bf))
+* Vault health check ([#734](https://github.com/testcontainers/testcontainers-python/issues/734)) ([79434d6](https://github.com/testcontainers/testcontainers-python/commit/79434d6744b2918493884cf8fbf27aeadf78ecfd))
+
+
+### Documentation
+
+* Documentation fix for ServerContainer ([#671](https://github.com/testcontainers/testcontainers-python/issues/671)) ([0303d47](https://github.com/testcontainers/testcontainers-python/commit/0303d47d7173e1c4ec1a4f565efee9b2fe694928))
+
 ## [4.8.2](https://github.com/testcontainers/testcontainers-python/compare/testcontainers-v4.8.1...testcontainers-v4.8.2) (2024-09-27)
 
 

core/testcontainers/compose/compose.py

@@ -171,6 +171,7 @@ class DockerCompose:
     env_file: Optional[str] = None
     services: Optional[list[str]] = None
     docker_command_path: Optional[str] = None
+    profiles: Optional[list[str]] = None
 
     def __post_init__(self):
         if isinstance(self.compose_file_name, str):
@@ -198,6 +199,8 @@ def compose_command_property(self) -> list[str]:
         if self.compose_file_name:
             for file in self.compose_file_name:
                 docker_compose_cmd += ["-f", file]
+        if self.profiles:
+            docker_compose_cmd += [item for profile in self.profiles for item in ["--profile", profile]]
         if self.env_file:
             docker_compose_cmd += ["--env-file", self.env_file]
         return docker_compose_cmd

core/testcontainers/core/container.py

@@ -1,16 +1,18 @@
 import contextlib
+from os import PathLike
 from socket import socket
 from typing import TYPE_CHECKING, Optional, Union
 
 import docker.errors
 from docker import version
 from docker.types import EndpointConfig
+from dotenv import dotenv_values
 from typing_extensions import Self, assert_never
 
 from testcontainers.core.config import ConnectionMode
 from testcontainers.core.config import testcontainers_config as c
 from testcontainers.core.docker_client import DockerClient
-from testcontainers.core.exceptions import ContainerStartException
+from testcontainers.core.exceptions import ContainerConnectException, ContainerStartException
 from testcontainers.core.labels import LABEL_SESSION_ID, SESSION_ID
 from testcontainers.core.network import Network
 from testcontainers.core.utils import is_arm, setup_logger
@@ -57,6 +59,12 @@ def with_env(self, key: str, value: str) -> Self:
         self.env[key] = value
         return self
 
+    def with_env_file(self, env_file: Union[str, PathLike]) -> Self:
+        env_values = dotenv_values(env_file)
+        for key, value in env_values.items():
+            self.with_env(key, value)
+        return self
+
     def with_bind_ports(self, container: int, host: Optional[int] = None) -> Self:
         self.ports[container] = host
         return self
@@ -220,15 +228,21 @@ def _create_instance(cls) -> "Reaper":
             .with_env("RYUK_RECONNECTION_TIMEOUT", c.ryuk_reconnection_timeout)
             .start()
         )
-        wait_for_logs(Reaper._container, r".* Started!")
+        wait_for_logs(Reaper._container, r".* Started!", timeout=20, raise_on_exit=True)
 
         container_host = Reaper._container.get_container_host_ip()
         container_port = int(Reaper._container.get_exposed_port(8080))
 
+        if not container_host or not container_port:
+            raise ContainerConnectException(
+                f"Could not obtain network details for {Reaper._container._container.id}. Host: {container_host} Port: {container_port}"
+            )
+
         last_connection_exception: Optional[Exception] = None
         for _ in range(50):
             try:
                 Reaper._socket = socket()
+                Reaper._socket.settimeout(1)
                 Reaper._socket.connect((container_host, container_port))
                 last_connection_exception = None
                 break

core/testcontainers/core/exceptions.py

@@ -16,6 +16,10 @@ class ContainerStartException(RuntimeError):
     pass
 
 
+class ContainerConnectException(RuntimeError):
+    pass
+
+
 class ContainerIsNotRunning(RuntimeError):
     pass
 

core/tests/compose_fixtures/profile_support/compose.yaml

@@ -0,0 +1,16 @@
+services:
+  runs-always: &simple-service
+    image: alpine:latest
+    init: true
+    command:
+      - sh
+      - -c
+      - 'while true; do sleep 0.1 ; date -Ins; done'
+  runs-profile-a:
+    <<: *simple-service
+    profiles:
+      - profile-a
+  runs-profile-b:
+    <<: *simple-service
+    profiles:
+      - profile-b

core/tests/test_compose.py

@@ -2,7 +2,7 @@
 from pathlib import Path
 from re import split
 from time import sleep
-from typing import Union
+from typing import Union, Optional
 from urllib.request import urlopen, Request
 
 import pytest
@@ -352,3 +352,27 @@ def fetch(req: Union[Request, str]):
         if 200 < res.getcode() >= 400:
             raise Exception(f"HTTP Error: {res.getcode()} - {res.reason}: {body}")
         return res.getcode(), body
+
+
+@pytest.mark.parametrize(
+    argnames=["profiles", "running", "not_running"],
+    argvalues=[
+        pytest.param(None, ["runs-always"], ["runs-profile-a", "runs-profile-b"], id="default"),
+        pytest.param(
+            ["profile-a"], ["runs-always", "runs-profile-a"], ["runs-profile-b"], id="one-additional-profile-via-str"
+        ),
+        pytest.param(
+            ["profile-a", "profile-b"],
+            ["runs-always", "runs-profile-a", "runs-profile-b"],
+            [],
+            id="all-profiles-explicitly",
+        ),
+    ],
+)
+def test_compose_profile_support(profiles: Optional[list[str]], running: list[str], not_running: list[str]):
+    with DockerCompose(context=FIXTURES / "profile_support", profiles=profiles) as compose:
+        for service in running:
+            assert compose.get_container(service) is not None
+        for service in not_running:
+            with pytest.raises(ContainerIsNotRunning):
+                compose.get_container(service)

core/tests/test_core.py

@@ -1,3 +1,6 @@
+import tempfile
+from pathlib import Path
+
 from testcontainers.core.container import DockerContainer
 
 
@@ -17,3 +20,29 @@ def test_get_logs():
         assert isinstance(stdout, bytes)
         assert isinstance(stderr, bytes)
         assert "Hello from Docker".encode() in stdout, "There should be something on stdout"
+
+
+def test_docker_container_with_env_file():
+    """Test that environment variables can be loaded from a file"""
+    with tempfile.TemporaryDirectory() as temp_directory:
+        env_file_path = Path(temp_directory) / "env_file"
+        with open(env_file_path, "w") as f:
+            f.write(
+                """
+                TEST_ENV_VAR=hello
+                NUMBER=123
+                DOMAIN=example.org
+                ADMIN_EMAIL=admin@${DOMAIN}
+                ROOT_URL=${DOMAIN}/app
+                """
+            )
+        container = DockerContainer("alpine").with_command("tail -f /dev/null")  # Keep the container running
+        container.with_env_file(env_file_path)  # Load the environment variables from the file
+        with container:
+            output = container.exec("env").output.decode("utf-8").strip()
+            assert "TEST_ENV_VAR=hello" in output
+            assert "NUMBER=123" in output
+            assert "DOMAIN=example.org" in output
+            assert "[email protected]" in output
+            assert "ROOT_URL=example.org/app" in output
+            print(output)

modules/generic/testcontainers/generic/server.py

@@ -1,5 +1,5 @@
 from typing import Union
-from urllib.error import HTTPError
+from urllib.error import HTTPError, URLError
 from urllib.request import urlopen
 
 import httpx
@@ -40,7 +40,7 @@ def __init__(self, port: int, image: Union[str, DockerImage]) -> None:
         self.internal_port = port
         self.with_exposed_ports(self.internal_port)
 
-    @wait_container_is_ready(HTTPError)
+    @wait_container_is_ready(HTTPError, URLError)
     def _connect(self) -> None:
         # noinspection HttpUrlsUsage
         url = self._create_connection_url()

modules/mysql/testcontainers/mysql/__init__.py

@@ -31,14 +31,14 @@ class MySqlContainer(DbContainer):
         The example will spin up a MySql database to which you can connect with the credentials
         passed in the constructor. Alternatively, you may use the :code:`get_connection_url()`
         method which returns a sqlalchemy-compatible url in format
-        :code:`dialect+driver://username:password@host:port/database`.
+        :code:`mysql+dialect://username:password@host:port/database`.
 
         .. doctest::
 
             >>> import sqlalchemy
             >>> from testcontainers.mysql import MySqlContainer
 
-            >>> with MySqlContainer('mysql:5.7.17') as mysql:
+            >>> with MySqlContainer("mysql:5.7.17", dialect="pymysql") as mysql:
             ...     engine = sqlalchemy.create_engine(mysql.get_connection_url())
             ...     with engine.begin() as connection:
             ...         result = connection.execute(sqlalchemy.text("select version()"))
@@ -64,6 +64,7 @@ class MySqlContainer(DbContainer):
     def __init__(
         self,
         image: str = "mysql:latest",
+        dialect: Optional[str] = None,
         username: Optional[str] = None,
         root_password: Optional[str] = None,
         password: Optional[str] = None,
@@ -72,6 +73,10 @@ def __init__(
         seed: Optional[str] = None,
         **kwargs,
     ) -> None:
+        if dialect is not None and dialect.startswith("mysql+"):
+            msg = "Please remove 'mysql+' prefix from dialect parameter"
+            raise ValueError(msg)
+
         raise_for_deprecated_parameter(kwargs, "MYSQL_USER", "username")
         raise_for_deprecated_parameter(kwargs, "MYSQL_ROOT_PASSWORD", "root_password")
         raise_for_deprecated_parameter(kwargs, "MYSQL_PASSWORD", "password")
@@ -85,6 +90,9 @@ def __init__(
         self.password = password or environ.get("MYSQL_PASSWORD", "test")
         self.dbname = dbname or environ.get("MYSQL_DATABASE", "test")
 
+        self.dialect = dialect or environ.get("MYSQL_DIALECT", None)
+        self._db_url_dialect_part = "mysql" if self.dialect is None else f"mysql+{self.dialect}"
+
         if self.username == "root":
             self.root_password = self.password
         self.seed = seed
@@ -105,7 +113,11 @@ def _connect(self) -> None:
 
     def get_connection_url(self) -> str:
         return super()._create_connection_url(
-            dialect="mysql+pymysql", username=self.username, password=self.password, dbname=self.dbname, port=self.port
+            dialect=self._db_url_dialect_part,
+            username=self.username,
+            password=self.password,
+            dbname=self.dbname,
+            port=self.port,
         )
 
     def _transfer_seed(self) -> None:

modules/mysql/tests/test_mysql.py

@@ -11,9 +11,14 @@
 
 @pytest.mark.inside_docker_check
 def test_docker_run_mysql():
-    config = MySqlContainer("mysql:8.3.0")
+    config = MySqlContainer("mysql:8.3.0", dialect="pymysql")
     with config as mysql:
-        engine = sqlalchemy.create_engine(mysql.get_connection_url())
+        connection_url = mysql.get_connection_url()
+
+        assert mysql.dialect == "pymysql"
+        assert connection_url.startswith("mysql+pymysql://")
+
+        engine = sqlalchemy.create_engine(connection_url)
         with engine.begin() as connection:
             result = connection.execute(sqlalchemy.text("select version()"))
             for row in result:
@@ -22,7 +27,7 @@ def test_docker_run_mysql():
 
 @pytest.mark.skipif(is_arm(), reason="mysql container not available for ARM")
 def test_docker_run_legacy_mysql():
-    config = MySqlContainer("mysql:5.7.44")
+    config = MySqlContainer("mysql:5.7.44", dialect="pymysql")
     with config as mysql:
         engine = sqlalchemy.create_engine(mysql.get_connection_url())
         with engine.begin() as connection:
@@ -35,7 +40,7 @@ def test_docker_run_legacy_mysql():
 def test_docker_run_mysql_8_seed():
     # Avoid pytest CWD path issues
     SEEDS_PATH = (Path(__file__).parent / "seeds").absolute()
-    config = MySqlContainer("mysql:8", seed=SEEDS_PATH)
+    config = MySqlContainer("mysql:8", dialect="pymysql", seed=str(SEEDS_PATH))
     with config as mysql:
         engine = sqlalchemy.create_engine(mysql.get_connection_url())
         with engine.begin() as connection:
@@ -45,7 +50,7 @@ def test_docker_run_mysql_8_seed():
 
 @pytest.mark.parametrize("version", ["11.3.2", "10.11.7"])
 def test_docker_run_mariadb(version: str):
-    with MySqlContainer(f"mariadb:{version}") as mariadb:
+    with MySqlContainer(f"mariadb:{version}", dialect="pymysql") as mariadb:
         engine = sqlalchemy.create_engine(mariadb.get_connection_url())
         with engine.begin() as connection:
             result = connection.execute(sqlalchemy.text("select version()"))
@@ -55,14 +60,29 @@ def test_docker_run_mariadb(version: str):
 
 def test_docker_env_variables():
     with (
-        mock.patch.dict("os.environ", MYSQL_USER="demo", MYSQL_DATABASE="custom_db"),
+        mock.patch.dict("os.environ", MYSQL_DIALECT="pymysql", MYSQL_USER="demo", MYSQL_DATABASE="custom_db"),
         MySqlContainer("mariadb:10.6.5").with_bind_ports(3306, 32785) as container,
     ):
         url = container.get_connection_url()
         pattern = r"mysql\+pymysql:\/\/demo:test@[\w,.]+:(3306|32785)\/custom_db"
         assert re.match(pattern, url)
 
 
+@pytest.mark.parametrize(
+    "dialect",
+    [
+        "mysql+pymysql",
+        "mysql+mariadb",
+        "mysql+mysqldb",
+    ],
+)
+def test_mysql_dialect_expecting_error_on_mysql_prefix(dialect: str):
+    match = f"Please remove *.* prefix from dialect parameter"
+
+    with pytest.raises(ValueError, match=match):
+        _ = MySqlContainer("mariadb:10.6.5", dialect=dialect)
+
+
 # This is a feature in the generic DbContainer class
 # but it can't be tested on its own
 # so is tested in various database modules:
@@ -75,18 +95,18 @@ def test_quoted_password():
     user = "root"
     password = "p@$%25+0&%rd :/!=?"
     quoted_password = "p%40%24%2525+0%26%25rd %3A%2F%21%3D%3F"
-    driver = "pymysql"
-    with MySqlContainer("mariadb:10.6.5", username=user, password=password) as container:
+    dialect = "pymysql"
+    with MySqlContainer("mariadb:10.6.5", dialect=dialect, username=user, password=password) as container:
         host = container.get_container_host_ip()
         port = container.get_exposed_port(3306)
-        expected_url = f"mysql+{driver}://{user}:{quoted_password}@{host}:{port}/test"
+        expected_url = f"mysql+{dialect}://{user}:{quoted_password}@{host}:{port}/test"
         url = container.get_connection_url()
         assert url == expected_url
 
         with sqlalchemy.create_engine(expected_url).begin() as connection:
             connection.execute(sqlalchemy.text("select version()"))
 
-        raw_pass_url = f"mysql+{driver}://{user}:{password}@{host}:{port}/test"
+        raw_pass_url = f"mysql+{dialect}://{user}:{password}@{host}:{port}/test"
         with pytest.raises(Exception):
             with sqlalchemy.create_engine(raw_pass_url).begin() as connection:
                 connection.execute(sqlalchemy.text("select version()"))