From 932ee307955e3591a63f194aee8e2f6d8e2f6bf9 Mon Sep 17 00:00:00 2001 From: Roy Moore Date: Mon, 18 Nov 2024 14:35:01 +0200 Subject: [PATCH 1/6] feat(core): Support working with env files (#737) Fix: https://github.com/testcontainers/testcontainers-python/issues/687 Users should not be required to load each env var manually if they have an env file. Added support for loading a dot-env file. Usage: ```python with DockerContainer("nginx:latest").with_env_file("my_env_file"): # We now have a container with the relevant env vars from the file ``` This is an implementation of ```docker run --env-file ...``` --- core/testcontainers/core/container.py | 8 ++++++++ core/tests/test_core.py | 29 +++++++++++++++++++++++++++ poetry.lock | 2 +- pyproject.toml | 1 + 4 files changed, 39 insertions(+), 1 deletion(-) diff --git a/core/testcontainers/core/container.py b/core/testcontainers/core/container.py index 90967e95..6bf2f5b4 100644 --- a/core/testcontainers/core/container.py +++ b/core/testcontainers/core/container.py @@ -1,10 +1,12 @@ import contextlib +from os import PathLike from socket import socket from typing import TYPE_CHECKING, Optional, Union import docker.errors from docker import version from docker.types import EndpointConfig +from dotenv import dotenv_values from typing_extensions import Self, assert_never from testcontainers.core.config import ConnectionMode @@ -57,6 +59,12 @@ def with_env(self, key: str, value: str) -> Self: self.env[key] = value return self + def with_env_file(self, env_file: Union[str, PathLike]) -> Self: + env_values = dotenv_values(env_file) + for key, value in env_values.items(): + self.with_env(key, value) + return self + def with_bind_ports(self, container: int, host: Optional[int] = None) -> Self: self.ports[container] = host return self diff --git a/core/tests/test_core.py b/core/tests/test_core.py index 42321e28..9312b0bc 100644 --- a/core/tests/test_core.py +++ b/core/tests/test_core.py @@ -1,3 +1,6 @@ +import tempfile +from pathlib import Path + from testcontainers.core.container import DockerContainer @@ -17,3 +20,29 @@ def test_get_logs(): assert isinstance(stdout, bytes) assert isinstance(stderr, bytes) assert "Hello from Docker".encode() in stdout, "There should be something on stdout" + + +def test_docker_container_with_env_file(): + """Test that environment variables can be loaded from a file""" + with tempfile.TemporaryDirectory() as temp_directory: + env_file_path = Path(temp_directory) / "env_file" + with open(env_file_path, "w") as f: + f.write( + """ + TEST_ENV_VAR=hello + NUMBER=123 + DOMAIN=example.org + ADMIN_EMAIL=admin@${DOMAIN} + ROOT_URL=${DOMAIN}/app + """ + ) + container = DockerContainer("alpine").with_command("tail -f /dev/null") # Keep the container running + container.with_env_file(env_file_path) # Load the environment variables from the file + with container: + output = container.exec("env").output.decode("utf-8").strip() + assert "TEST_ENV_VAR=hello" in output + assert "NUMBER=123" in output + assert "DOMAIN=example.org" in output + assert "ADMIN_EMAIL=admin@example.org" in output + assert "ROOT_URL=example.org/app" in output + print(output) diff --git a/poetry.lock b/poetry.lock index b83de36f..bd54659e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -4678,4 +4678,4 @@ weaviate = ["weaviate-client"] [metadata] lock-version = "2.0" python-versions = ">=3.9,<4.0" -content-hash = "7ffcf39257e1ac79d951b7ccb2cdb972beaaef35d0cd1f722d2964c5bdced674" +content-hash = "5c400cc87dc9708588ee8d7d50646de789235732d868b74ebc43f1cf2a403c88" diff --git a/pyproject.toml b/pyproject.toml index bd4e7ed0..a44cc3a3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -83,6 +83,7 @@ docker = "*" # ">=4.0" urllib3 = "*" # "<2.0" wrapt = "*" # "^1.16.0" typing-extensions = "*" +python-dotenv = "*" # community modules python-arango = { version = "^7.8", optional = true } From 4ced1983162914fe511a6e714f136b670e1dbdfb Mon Sep 17 00:00:00 2001 From: Miles Mason Winther <42948872+mmwinther@users.noreply.github.com> Date: Mon, 25 Nov 2024 14:20:44 +0100 Subject: [PATCH 2/6] fix(core): Avoid hanging upon bad docker host connection (#742) - Shorten the socket connection timeout - Add a timeout on waiting for logs - Add check for host and port values - The previous implementation _technically_ would have timed out, but the default socket timeout * 50 iterations is a very long time to wait. Resolves #741 --------- Co-authored-by: David Ankin --- core/testcontainers/core/container.py | 10 ++++++++-- core/testcontainers/core/exceptions.py | 4 ++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/core/testcontainers/core/container.py b/core/testcontainers/core/container.py index 6bf2f5b4..f677182f 100644 --- a/core/testcontainers/core/container.py +++ b/core/testcontainers/core/container.py @@ -12,7 +12,7 @@ from testcontainers.core.config import ConnectionMode from testcontainers.core.config import testcontainers_config as c from testcontainers.core.docker_client import DockerClient -from testcontainers.core.exceptions import ContainerStartException +from testcontainers.core.exceptions import ContainerConnectException, ContainerStartException from testcontainers.core.labels import LABEL_SESSION_ID, SESSION_ID from testcontainers.core.network import Network from testcontainers.core.utils import is_arm, setup_logger @@ -228,15 +228,21 @@ def _create_instance(cls) -> "Reaper": .with_env("RYUK_RECONNECTION_TIMEOUT", c.ryuk_reconnection_timeout) .start() ) - wait_for_logs(Reaper._container, r".* Started!") + wait_for_logs(Reaper._container, r".* Started!", timeout=20, raise_on_exit=True) container_host = Reaper._container.get_container_host_ip() container_port = int(Reaper._container.get_exposed_port(8080)) + if not container_host or not container_port: + raise ContainerConnectException( + f"Could not obtain network details for {Reaper._container._container.id}. Host: {container_host} Port: {container_port}" + ) + last_connection_exception: Optional[Exception] = None for _ in range(50): try: Reaper._socket = socket() + Reaper._socket.settimeout(1) Reaper._socket.connect((container_host, container_port)) last_connection_exception = None break diff --git a/core/testcontainers/core/exceptions.py b/core/testcontainers/core/exceptions.py index 6694e598..99ea9881 100644 --- a/core/testcontainers/core/exceptions.py +++ b/core/testcontainers/core/exceptions.py @@ -16,6 +16,10 @@ class ContainerStartException(RuntimeError): pass +class ContainerConnectException(RuntimeError): + pass + + class ContainerIsNotRunning(RuntimeError): pass From 24e354f3bfa5912eaf7877da9442a885d7872f1a Mon Sep 17 00:00:00 2001 From: Miles Mason Winther <42948872+mmwinther@users.noreply.github.com> Date: Tue, 26 Nov 2024 18:55:49 +0100 Subject: [PATCH 3/6] fix(generic): Also catch URLError waiting for ServerContainer (#743) ## Problem summary There was a race condition here because the server hasn't always started before we send a request to it. This was causing a lot of unnecessary test failures. This change makes use of `ServerContainer` much more reliable. ## Example Traceback ``` ../../../Library/Caches/pypoetry/virtualenvs//lib/python3.12/site-packages/testcontainers/generic/server.py:70: in start self._connect() ../../../Library/Caches/pypoetry/virtualenvs//lib/python3.12/site-packages/testcontainers/core/waiting_utils.py:59: in wrapper return wrapped(*args, **kwargs) ../../../Library/Caches/pypoetry/virtualenvs//lib/python3.12/site-packages/testcontainers/generic/server.py:48: in _connect with urlopen(url) as r: /usr/local/Cellar/python@3.12/3.12.7/Frameworks/Python.framework/Versions/3.12/lib/python3.12/urllib/request.py:215: in urlopen return opener.open(url, data, timeout) /usr/local/Cellar/python@3.12/3.12.7/Frameworks/Python.framework/Versions/3.12/lib/python3.12/urllib/request.py:515: in open response = self._open(req, data) /usr/local/Cellar/python@3.12/3.12.7/Frameworks/Python.framework/Versions/3.12/lib/python3.12/urllib/request.py:532: in _open result = self._call_chain(self.handle_open, protocol, protocol + /usr/local/Cellar/python@3.12/3.12.7/Frameworks/Python.framework/Versions/3.12/lib/python3.12/urllib/request.py:492: in _call_chain result = func(*args) /usr/local/Cellar/python@3.12/3.12.7/Frameworks/Python.framework/Versions/3.12/lib/python3.12/urllib/request.py:1373: in http_open return self.do_open(http.client.HTTPConnection, req) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = http_class = req = , http_conn_args = {} host = '192.168.106.2:32876' h = headers = {'Connection': 'close', 'Host': '192.168.106.2:32876', 'User-Agent': 'Python-urllib/3.12'} def do_open(self, http_class, req, **http_conn_args): """Return an HTTPResponse object for the request, using http_class. http_class must implement the HTTPConnection API from http.client. """ host = req.host if not host: raise URLError('no host given') # will parse host:port h = http_class(host, timeout=req.timeout, **http_conn_args) h.set_debuglevel(self._debuglevel) headers = dict(req.unredirected_hdrs) headers.update({k: v for k, v in req.headers.items() if k not in headers}) # TODO(jhylton): Should this be redesigned to handle # persistent connections? # We want to make an HTTP/1.1 request, but the addinfourl # class isn't prepared to deal with a persistent connection. # It will try to read all remaining data from the socket, # which will block while the server waits for the next request. # So make sure the connection gets closed after the (only) # request. headers["Connection"] = "close" headers = {name.title(): val for name, val in headers.items()} if req._tunnel_host: tunnel_headers = {} proxy_auth_hdr = "Proxy-Authorization" if proxy_auth_hdr in headers: tunnel_headers[proxy_auth_hdr] = headers[proxy_auth_hdr] # Proxy-Authorization should not be sent to origin # server. del headers[proxy_auth_hdr] h.set_tunnel(req._tunnel_host, headers=tunnel_headers) try: try: h.request(req.get_method(), req.selector, req.data, headers, encode_chunked=req.has_header('Transfer-encoding')) except OSError as err: # timeout error > raise URLError(err) E urllib.error.URLError: /usr/local/Cellar/python@3.12/3.12.7/Frameworks/Python.framework/Versions/3.12/lib/python3.12/urllib/request.py:1347: URLError ``` --- modules/generic/testcontainers/generic/server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/generic/testcontainers/generic/server.py b/modules/generic/testcontainers/generic/server.py index ce61e9b7..fe990f17 100644 --- a/modules/generic/testcontainers/generic/server.py +++ b/modules/generic/testcontainers/generic/server.py @@ -1,5 +1,5 @@ from typing import Union -from urllib.error import HTTPError +from urllib.error import HTTPError, URLError from urllib.request import urlopen import httpx @@ -40,7 +40,7 @@ def __init__(self, port: int, image: Union[str, DockerImage]) -> None: self.internal_port = port self.with_exposed_ports(self.internal_port) - @wait_container_is_ready(HTTPError) + @wait_container_is_ready(HTTPError, URLError) def _connect(self) -> None: # noinspection HttpUrlsUsage url = self._create_connection_url() From 3e00e71da4d2b5e7fd30315468d4e54c86ba6150 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A1lint=20Bartha?= <39852431+totallyzen@users.noreply.github.com> Date: Tue, 26 Nov 2024 20:21:09 +0100 Subject: [PATCH 4/6] feat(compose): support for setting profiles (#738) # Change Adds `profiles` support for docker compose. # Context I've had to use a workaround in a company project for compose profiles leading to multiple compose profile files. This should help by supporting profiles in compose which is a very handy feature when your service has some variety of ways to run (contextual, environmental, etc) Without this, it's still possible to run profiles buy setting the `COMPOSE_PROFILES` env variable, but this is certainly cleaner and easier on test writing. # Docker Docs https://docs.docker.com/compose/how-tos/profiles/#assigning-profiles-to-services --- core/testcontainers/compose/compose.py | 3 +++ .../profile_support/compose.yaml | 16 ++++++++++++ core/tests/test_compose.py | 26 ++++++++++++++++++- 3 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 core/tests/compose_fixtures/profile_support/compose.yaml diff --git a/core/testcontainers/compose/compose.py b/core/testcontainers/compose/compose.py index 564eda8f..e8ce3745 100644 --- a/core/testcontainers/compose/compose.py +++ b/core/testcontainers/compose/compose.py @@ -171,6 +171,7 @@ class DockerCompose: env_file: Optional[str] = None services: Optional[list[str]] = None docker_command_path: Optional[str] = None + profiles: Optional[list[str]] = None def __post_init__(self): if isinstance(self.compose_file_name, str): @@ -198,6 +199,8 @@ def compose_command_property(self) -> list[str]: if self.compose_file_name: for file in self.compose_file_name: docker_compose_cmd += ["-f", file] + if self.profiles: + docker_compose_cmd += [item for profile in self.profiles for item in ["--profile", profile]] if self.env_file: docker_compose_cmd += ["--env-file", self.env_file] return docker_compose_cmd diff --git a/core/tests/compose_fixtures/profile_support/compose.yaml b/core/tests/compose_fixtures/profile_support/compose.yaml new file mode 100644 index 00000000..c7bec7cc --- /dev/null +++ b/core/tests/compose_fixtures/profile_support/compose.yaml @@ -0,0 +1,16 @@ +services: + runs-always: &simple-service + image: alpine:latest + init: true + command: + - sh + - -c + - 'while true; do sleep 0.1 ; date -Ins; done' + runs-profile-a: + <<: *simple-service + profiles: + - profile-a + runs-profile-b: + <<: *simple-service + profiles: + - profile-b diff --git a/core/tests/test_compose.py b/core/tests/test_compose.py index b43da28c..9279ce3f 100644 --- a/core/tests/test_compose.py +++ b/core/tests/test_compose.py @@ -2,7 +2,7 @@ from pathlib import Path from re import split from time import sleep -from typing import Union +from typing import Union, Optional from urllib.request import urlopen, Request import pytest @@ -352,3 +352,27 @@ def fetch(req: Union[Request, str]): if 200 < res.getcode() >= 400: raise Exception(f"HTTP Error: {res.getcode()} - {res.reason}: {body}") return res.getcode(), body + + +@pytest.mark.parametrize( + argnames=["profiles", "running", "not_running"], + argvalues=[ + pytest.param(None, ["runs-always"], ["runs-profile-a", "runs-profile-b"], id="default"), + pytest.param( + ["profile-a"], ["runs-always", "runs-profile-a"], ["runs-profile-b"], id="one-additional-profile-via-str" + ), + pytest.param( + ["profile-a", "profile-b"], + ["runs-always", "runs-profile-a", "runs-profile-b"], + [], + id="all-profiles-explicitly", + ), + ], +) +def test_compose_profile_support(profiles: Optional[list[str]], running: list[str], not_running: list[str]): + with DockerCompose(context=FIXTURES / "profile_support", profiles=profiles) as compose: + for service in running: + assert compose.get_container(service) is not None + for service in not_running: + with pytest.raises(ContainerIsNotRunning): + compose.get_container(service) From 3436cbfd7530b2168400b941635c8baeafa5cc97 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 10 Dec 2024 12:24:43 -0500 Subject: [PATCH 5/6] chore(main): release testcontainers 4.9.0 (#722) :robot: I have created a release *beep* *boop* --- ## [4.9.0](https://github.com/testcontainers/testcontainers-python/compare/testcontainers-v4.8.2...testcontainers-v4.9.0) (2024-11-26) ### Features * **compose:** support for setting profiles ([#738](https://github.com/testcontainers/testcontainers-python/issues/738)) ([3e00e71](https://github.com/testcontainers/testcontainers-python/commit/3e00e71da4d2b5e7fd30315468d4e54c86ba6150)) * **core:** Support working with env files ([#737](https://github.com/testcontainers/testcontainers-python/issues/737)) ([932ee30](https://github.com/testcontainers/testcontainers-python/commit/932ee307955e3591a63f194aee8e2f6d8e2f6bf9)) ### Bug Fixes * allow running all tests ([#721](https://github.com/testcontainers/testcontainers-python/issues/721)) ([f958cf9](https://github.com/testcontainers/testcontainers-python/commit/f958cf9fe62a5f3ee2dc255713ec8b16de6a767d)) * **core:** Avoid hanging upon bad docker host connection ([#742](https://github.com/testcontainers/testcontainers-python/issues/742)) ([4ced198](https://github.com/testcontainers/testcontainers-python/commit/4ced1983162914fe511a6e714f136b670e1dbdfb)) * **core:** running testcontainer inside container ([#714](https://github.com/testcontainers/testcontainers-python/issues/714)) ([85a6666](https://github.com/testcontainers/testcontainers-python/commit/85a66667c23d76e87aecc6761bbb01429adb3dee)) * **generic:** Also catch URLError waiting for ServerContainer ([#743](https://github.com/testcontainers/testcontainers-python/issues/743)) ([24e354f](https://github.com/testcontainers/testcontainers-python/commit/24e354f3bfa5912eaf7877da9442a885d7872f1a)) * update wait_for_logs to not throw on 'created', and an optimization ([#719](https://github.com/testcontainers/testcontainers-python/issues/719)) ([271ca9a](https://github.com/testcontainers/testcontainers-python/commit/271ca9a0fef2e5f2b216457bfee44318e93990bf)) * Vault health check ([#734](https://github.com/testcontainers/testcontainers-python/issues/734)) ([79434d6](https://github.com/testcontainers/testcontainers-python/commit/79434d6744b2918493884cf8fbf27aeadf78ecfd)) ### Documentation * Documentation fix for ServerContainer ([#671](https://github.com/testcontainers/testcontainers-python/issues/671)) ([0303d47](https://github.com/testcontainers/testcontainers-python/commit/0303d47d7173e1c4ec1a4f565efee9b2fe694928)) --- This PR was generated with [Release Please](https://github.com/googleapis/release-please). See [documentation](https://github.com/googleapis/release-please#release-please). Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- .github/.release-please-manifest.json | 2 +- CHANGELOG.md | 23 +++++++++++++++++++++++ pyproject.toml | 2 +- 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/.github/.release-please-manifest.json b/.github/.release-please-manifest.json index 98ecd87e..0cd2cc7e 100644 --- a/.github/.release-please-manifest.json +++ b/.github/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "4.8.2" + ".": "4.9.0" } diff --git a/CHANGELOG.md b/CHANGELOG.md index 9372aeeb..c6cb6856 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,28 @@ # Changelog +## [4.9.0](https://github.com/testcontainers/testcontainers-python/compare/testcontainers-v4.8.2...testcontainers-v4.9.0) (2024-11-26) + + +### Features + +* **compose:** support for setting profiles ([#738](https://github.com/testcontainers/testcontainers-python/issues/738)) ([3e00e71](https://github.com/testcontainers/testcontainers-python/commit/3e00e71da4d2b5e7fd30315468d4e54c86ba6150)) +* **core:** Support working with env files ([#737](https://github.com/testcontainers/testcontainers-python/issues/737)) ([932ee30](https://github.com/testcontainers/testcontainers-python/commit/932ee307955e3591a63f194aee8e2f6d8e2f6bf9)) + + +### Bug Fixes + +* allow running all tests ([#721](https://github.com/testcontainers/testcontainers-python/issues/721)) ([f958cf9](https://github.com/testcontainers/testcontainers-python/commit/f958cf9fe62a5f3ee2dc255713ec8b16de6a767d)) +* **core:** Avoid hanging upon bad docker host connection ([#742](https://github.com/testcontainers/testcontainers-python/issues/742)) ([4ced198](https://github.com/testcontainers/testcontainers-python/commit/4ced1983162914fe511a6e714f136b670e1dbdfb)) +* **core:** running testcontainer inside container ([#714](https://github.com/testcontainers/testcontainers-python/issues/714)) ([85a6666](https://github.com/testcontainers/testcontainers-python/commit/85a66667c23d76e87aecc6761bbb01429adb3dee)) +* **generic:** Also catch URLError waiting for ServerContainer ([#743](https://github.com/testcontainers/testcontainers-python/issues/743)) ([24e354f](https://github.com/testcontainers/testcontainers-python/commit/24e354f3bfa5912eaf7877da9442a885d7872f1a)) +* update wait_for_logs to not throw on 'created', and an optimization ([#719](https://github.com/testcontainers/testcontainers-python/issues/719)) ([271ca9a](https://github.com/testcontainers/testcontainers-python/commit/271ca9a0fef2e5f2b216457bfee44318e93990bf)) +* Vault health check ([#734](https://github.com/testcontainers/testcontainers-python/issues/734)) ([79434d6](https://github.com/testcontainers/testcontainers-python/commit/79434d6744b2918493884cf8fbf27aeadf78ecfd)) + + +### Documentation + +* Documentation fix for ServerContainer ([#671](https://github.com/testcontainers/testcontainers-python/issues/671)) ([0303d47](https://github.com/testcontainers/testcontainers-python/commit/0303d47d7173e1c4ec1a4f565efee9b2fe694928)) + ## [4.8.2](https://github.com/testcontainers/testcontainers-python/compare/testcontainers-v4.8.1...testcontainers-v4.8.2) (2024-09-27) diff --git a/pyproject.toml b/pyproject.toml index a44cc3a3..8bbf82ea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "testcontainers" -version = "4.8.2" # auto-incremented by release-please +version = "4.9.0" # auto-incremented by release-please description = "Python library for throwaway instances of anything that can run in a Docker container" authors = ["Sergey Pirogov "] maintainers = [ From 8d77bd3541e1c5e73c7ed5d5bd3c0d7bb617f5c0 Mon Sep 17 00:00:00 2001 From: Ivan Belyaev Date: Thu, 12 Dec 2024 19:39:44 +0300 Subject: [PATCH 6/6] fix(mysql): add dialect parameter instead of hardcoded mysql dialect (#739) closes https://github.com/testcontainers/testcontainers-python/issues/727 * add parameter `dialect`; * tests fixing and add some assertions --- .../mysql/testcontainers/mysql/__init__.py | 18 +++++++-- modules/mysql/tests/test_mysql.py | 40 ++++++++++++++----- 2 files changed, 45 insertions(+), 13 deletions(-) diff --git a/modules/mysql/testcontainers/mysql/__init__.py b/modules/mysql/testcontainers/mysql/__init__.py index c4449593..4c381ff9 100644 --- a/modules/mysql/testcontainers/mysql/__init__.py +++ b/modules/mysql/testcontainers/mysql/__init__.py @@ -31,14 +31,14 @@ class MySqlContainer(DbContainer): The example will spin up a MySql database to which you can connect with the credentials passed in the constructor. Alternatively, you may use the :code:`get_connection_url()` method which returns a sqlalchemy-compatible url in format - :code:`dialect+driver://username:password@host:port/database`. + :code:`mysql+dialect://username:password@host:port/database`. .. doctest:: >>> import sqlalchemy >>> from testcontainers.mysql import MySqlContainer - >>> with MySqlContainer('mysql:5.7.17') as mysql: + >>> with MySqlContainer("mysql:5.7.17", dialect="pymysql") as mysql: ... engine = sqlalchemy.create_engine(mysql.get_connection_url()) ... with engine.begin() as connection: ... result = connection.execute(sqlalchemy.text("select version()")) @@ -64,6 +64,7 @@ class MySqlContainer(DbContainer): def __init__( self, image: str = "mysql:latest", + dialect: Optional[str] = None, username: Optional[str] = None, root_password: Optional[str] = None, password: Optional[str] = None, @@ -72,6 +73,10 @@ def __init__( seed: Optional[str] = None, **kwargs, ) -> None: + if dialect is not None and dialect.startswith("mysql+"): + msg = "Please remove 'mysql+' prefix from dialect parameter" + raise ValueError(msg) + raise_for_deprecated_parameter(kwargs, "MYSQL_USER", "username") raise_for_deprecated_parameter(kwargs, "MYSQL_ROOT_PASSWORD", "root_password") raise_for_deprecated_parameter(kwargs, "MYSQL_PASSWORD", "password") @@ -85,6 +90,9 @@ def __init__( self.password = password or environ.get("MYSQL_PASSWORD", "test") self.dbname = dbname or environ.get("MYSQL_DATABASE", "test") + self.dialect = dialect or environ.get("MYSQL_DIALECT", None) + self._db_url_dialect_part = "mysql" if self.dialect is None else f"mysql+{self.dialect}" + if self.username == "root": self.root_password = self.password self.seed = seed @@ -105,7 +113,11 @@ def _connect(self) -> None: def get_connection_url(self) -> str: return super()._create_connection_url( - dialect="mysql+pymysql", username=self.username, password=self.password, dbname=self.dbname, port=self.port + dialect=self._db_url_dialect_part, + username=self.username, + password=self.password, + dbname=self.dbname, + port=self.port, ) def _transfer_seed(self) -> None: diff --git a/modules/mysql/tests/test_mysql.py b/modules/mysql/tests/test_mysql.py index af0b4491..a2d2c2ec 100644 --- a/modules/mysql/tests/test_mysql.py +++ b/modules/mysql/tests/test_mysql.py @@ -11,9 +11,14 @@ @pytest.mark.inside_docker_check def test_docker_run_mysql(): - config = MySqlContainer("mysql:8.3.0") + config = MySqlContainer("mysql:8.3.0", dialect="pymysql") with config as mysql: - engine = sqlalchemy.create_engine(mysql.get_connection_url()) + connection_url = mysql.get_connection_url() + + assert mysql.dialect == "pymysql" + assert connection_url.startswith("mysql+pymysql://") + + engine = sqlalchemy.create_engine(connection_url) with engine.begin() as connection: result = connection.execute(sqlalchemy.text("select version()")) for row in result: @@ -22,7 +27,7 @@ def test_docker_run_mysql(): @pytest.mark.skipif(is_arm(), reason="mysql container not available for ARM") def test_docker_run_legacy_mysql(): - config = MySqlContainer("mysql:5.7.44") + config = MySqlContainer("mysql:5.7.44", dialect="pymysql") with config as mysql: engine = sqlalchemy.create_engine(mysql.get_connection_url()) with engine.begin() as connection: @@ -35,7 +40,7 @@ def test_docker_run_legacy_mysql(): def test_docker_run_mysql_8_seed(): # Avoid pytest CWD path issues SEEDS_PATH = (Path(__file__).parent / "seeds").absolute() - config = MySqlContainer("mysql:8", seed=SEEDS_PATH) + config = MySqlContainer("mysql:8", dialect="pymysql", seed=str(SEEDS_PATH)) with config as mysql: engine = sqlalchemy.create_engine(mysql.get_connection_url()) with engine.begin() as connection: @@ -45,7 +50,7 @@ def test_docker_run_mysql_8_seed(): @pytest.mark.parametrize("version", ["11.3.2", "10.11.7"]) def test_docker_run_mariadb(version: str): - with MySqlContainer(f"mariadb:{version}") as mariadb: + with MySqlContainer(f"mariadb:{version}", dialect="pymysql") as mariadb: engine = sqlalchemy.create_engine(mariadb.get_connection_url()) with engine.begin() as connection: result = connection.execute(sqlalchemy.text("select version()")) @@ -55,7 +60,7 @@ def test_docker_run_mariadb(version: str): def test_docker_env_variables(): with ( - mock.patch.dict("os.environ", MYSQL_USER="demo", MYSQL_DATABASE="custom_db"), + mock.patch.dict("os.environ", MYSQL_DIALECT="pymysql", MYSQL_USER="demo", MYSQL_DATABASE="custom_db"), MySqlContainer("mariadb:10.6.5").with_bind_ports(3306, 32785) as container, ): url = container.get_connection_url() @@ -63,6 +68,21 @@ def test_docker_env_variables(): assert re.match(pattern, url) +@pytest.mark.parametrize( + "dialect", + [ + "mysql+pymysql", + "mysql+mariadb", + "mysql+mysqldb", + ], +) +def test_mysql_dialect_expecting_error_on_mysql_prefix(dialect: str): + match = f"Please remove *.* prefix from dialect parameter" + + with pytest.raises(ValueError, match=match): + _ = MySqlContainer("mariadb:10.6.5", dialect=dialect) + + # This is a feature in the generic DbContainer class # but it can't be tested on its own # so is tested in various database modules: @@ -75,18 +95,18 @@ def test_quoted_password(): user = "root" password = "p@$%25+0&%rd :/!=?" quoted_password = "p%40%24%2525+0%26%25rd %3A%2F%21%3D%3F" - driver = "pymysql" - with MySqlContainer("mariadb:10.6.5", username=user, password=password) as container: + dialect = "pymysql" + with MySqlContainer("mariadb:10.6.5", dialect=dialect, username=user, password=password) as container: host = container.get_container_host_ip() port = container.get_exposed_port(3306) - expected_url = f"mysql+{driver}://{user}:{quoted_password}@{host}:{port}/test" + expected_url = f"mysql+{dialect}://{user}:{quoted_password}@{host}:{port}/test" url = container.get_connection_url() assert url == expected_url with sqlalchemy.create_engine(expected_url).begin() as connection: connection.execute(sqlalchemy.text("select version()")) - raw_pass_url = f"mysql+{driver}://{user}:{password}@{host}:{port}/test" + raw_pass_url = f"mysql+{dialect}://{user}:{password}@{host}:{port}/test" with pytest.raises(Exception): with sqlalchemy.create_engine(raw_pass_url).begin() as connection: connection.execute(sqlalchemy.text("select version()"))