From 68de4ba41df1124f1fd824b83aa6975980db9525 Mon Sep 17 00:00:00 2001 From: Vemund Santi Date: Wed, 3 Apr 2024 13:54:37 +0200 Subject: [PATCH 01/13] Use log waiting strategy for mysql module --- modules/mysql/testcontainers/mysql/__init__.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/modules/mysql/testcontainers/mysql/__init__.py b/modules/mysql/testcontainers/mysql/__init__.py index 65b317b0..2f2ff9f9 100644 --- a/modules/mysql/testcontainers/mysql/__init__.py +++ b/modules/mysql/testcontainers/mysql/__init__.py @@ -10,11 +10,13 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. +import re from os import environ from typing import Optional from testcontainers.core.generic import DbContainer from testcontainers.core.utils import raise_for_deprecated_parameter +from testcontainers.core.waiting_utils import wait_for_logs class MySqlContainer(DbContainer): @@ -74,6 +76,12 @@ def _configure(self) -> None: self.with_env("MYSQL_USER", self.username) self.with_env("MYSQL_PASSWORD", self.password) + def _connect(self) -> None: + wait_for_logs( + self, + re.compile(".*: ready for connections.*: ready for connections.*", flags=re.DOTALL | re.MULTILINE).search, + ) + def get_connection_url(self) -> str: return super()._create_connection_url( dialect="mysql+pymysql", username=self.username, password=self.password, dbname=self.dbname, port=self.port From f63d3c4df4a59d815608c0378d3661739beaaf63 Mon Sep 17 00:00:00 2001 From: Vemund Santi Date: Wed, 3 Apr 2024 13:56:43 +0200 Subject: [PATCH 02/13] Update tests with modern tags for mysql. Support more ARM --- modules/mysql/tests/test_mysql.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/modules/mysql/tests/test_mysql.py b/modules/mysql/tests/test_mysql.py index a84df4d1..30c4bdce 100644 --- a/modules/mysql/tests/test_mysql.py +++ b/modules/mysql/tests/test_mysql.py @@ -8,41 +8,41 @@ from testcontainers.mysql import MySqlContainer -@pytest.mark.skipif(is_arm(), reason="mysql container not available for ARM") def test_docker_run_mysql(): - config = MySqlContainer("mysql:5.7.17") + config = MySqlContainer("mysql:8.3.0") with config as mysql: engine = sqlalchemy.create_engine(mysql.get_connection_url()) with engine.begin() as connection: result = connection.execute(sqlalchemy.text("select version()")) for row in result: - assert row[0].startswith("5.7.17") + assert row[0].startswith("8.3.0") @pytest.mark.skipif(is_arm(), reason="mysql container not available for ARM") -def test_docker_run_mysql_8(): - config = MySqlContainer("mysql:8") +def test_docker_run_legacy_mysql(): + config = MySqlContainer("mysql:5.7.44") with config as mysql: engine = sqlalchemy.create_engine(mysql.get_connection_url()) with engine.begin() as connection: result = connection.execute(sqlalchemy.text("select version()")) for row in result: - assert row[0].startswith("8") + assert row[0].startswith("5.7.17") -def test_docker_run_mariadb(): - with MySqlContainer("mariadb:10.6.5").maybe_emulate_amd64() as mariadb: +@pytest.mark.parametrize("version", ["11.3.2", "10.11.7"]) +def test_docker_run_mariadb(version: str): + with MySqlContainer(f"mariadb:{version}") as mariadb: engine = sqlalchemy.create_engine(mariadb.get_connection_url()) with engine.begin() as connection: result = connection.execute(sqlalchemy.text("select version()")) for row in result: - assert row[0].startswith("10.6.5") + assert row[0].startswith(version) def test_docker_env_variables(): with mock.patch.dict("os.environ", MYSQL_USER="demo", MYSQL_DATABASE="custom_db"), MySqlContainer( "mariadb:10.6.5" - ).with_bind_ports(3306, 32785).maybe_emulate_amd64() as container: + ).with_bind_ports(3306, 32785) as container: url = container.get_connection_url() pattern = r"mysql\+pymysql:\/\/demo:test@[\w,.]+:(3306|32785)\/custom_db" assert re.match(pattern, url) From 5132c5717ee5a195ff4e000f4c4a86820fa9e00c Mon Sep 17 00:00:00 2001 From: Vemund Santi Date: Wed, 3 Apr 2024 14:43:37 +0200 Subject: [PATCH 03/13] Fix legacy mysql version assert --- modules/mysql/tests/test_mysql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/mysql/tests/test_mysql.py b/modules/mysql/tests/test_mysql.py index 30c4bdce..2e99711c 100644 --- a/modules/mysql/tests/test_mysql.py +++ b/modules/mysql/tests/test_mysql.py @@ -26,7 +26,7 @@ def test_docker_run_legacy_mysql(): with engine.begin() as connection: result = connection.execute(sqlalchemy.text("select version()")) for row in result: - assert row[0].startswith("5.7.17") + assert row[0].startswith("5.7.44") @pytest.mark.parametrize("version", ["11.3.2", "10.11.7"]) From 01e193b3c220c8fd5047e53134ed2b1e56546f1b Mon Sep 17 00:00:00 2001 From: Vemund Santi Date: Wed, 3 Apr 2024 15:54:02 +0200 Subject: [PATCH 04/13] Use log waiting strategy for oracle-free instead of client test --- modules/oracle-free/testcontainers/oracle/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/modules/oracle-free/testcontainers/oracle/__init__.py b/modules/oracle-free/testcontainers/oracle/__init__.py index 2b903ac5..781be428 100644 --- a/modules/oracle-free/testcontainers/oracle/__init__.py +++ b/modules/oracle-free/testcontainers/oracle/__init__.py @@ -3,6 +3,7 @@ from typing import Optional from testcontainers.core.generic import DbContainer +from testcontainers.core.waiting_utils import wait_for_logs class OracleDbContainer(DbContainer): @@ -36,7 +37,7 @@ def __init__( password: Optional[str] = None, port: int = 1521, dbname: Optional[str] = None, - **kwargs + **kwargs, ) -> None: super().__init__(image=image, **kwargs) @@ -57,6 +58,9 @@ def get_connection_url(self) -> str: ) + "/?service_name={}".format(self.dbname or "FREEPDB1") # Default DB is "FREEPDB1" + def _connect(self) -> None: + wait_for_logs(self, "DATABASE IS READY TO USE!") + def _configure(self) -> None: # if self.oracle_password is not None: # self.with_env("ORACLE_PASSWORD", self.oracle_password) From 50f3f03869b72d3e440fd1de2c73604c2c960184 Mon Sep 17 00:00:00 2001 From: Vemund Santi Date: Wed, 3 Apr 2024 16:02:48 +0200 Subject: [PATCH 05/13] Set default timeout on wait_for_logs --- core/testcontainers/core/waiting_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/testcontainers/core/waiting_utils.py b/core/testcontainers/core/waiting_utils.py index ea52683d..35fee81e 100644 --- a/core/testcontainers/core/waiting_utils.py +++ b/core/testcontainers/core/waiting_utils.py @@ -78,7 +78,7 @@ def wait_for(condition: Callable[..., bool]) -> bool: def wait_for_logs( - container: "DockerContainer", predicate: Union[Callable, str], timeout: Optional[float] = None, interval: float = 1 + container: "DockerContainer", predicate: Union[Callable, str], timeout: float = config.TIMEOUT, interval: float = 1 ) -> float: """ Wait for the container to emit logs satisfying the predicate. @@ -103,6 +103,6 @@ def wait_for_logs( stderr = container.get_logs()[1].decode() if predicate(stdout) or predicate(stderr): return duration - if timeout and duration > timeout: + if duration > timeout: raise TimeoutError(f"Container did not emit logs satisfying predicate in {timeout:.3f} " "seconds") time.sleep(interval) From 7c591189021d2f7b46e424dd5d46065d64cf54d9 Mon Sep 17 00:00:00 2001 From: Vemund Santi Date: Wed, 3 Apr 2024 16:06:28 +0200 Subject: [PATCH 06/13] Make linter happy --- core/testcontainers/core/waiting_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/testcontainers/core/waiting_utils.py b/core/testcontainers/core/waiting_utils.py index 35fee81e..6f8c4a22 100644 --- a/core/testcontainers/core/waiting_utils.py +++ b/core/testcontainers/core/waiting_utils.py @@ -15,7 +15,7 @@ import re import time import traceback -from typing import TYPE_CHECKING, Any, Callable, Optional, Union +from typing import TYPE_CHECKING, Any, Callable, Union import wrapt From 1649efdc2e4124222559293932a2e72623d31239 Mon Sep 17 00:00:00 2001 From: Vemund Santi Date: Wed, 3 Apr 2024 22:03:26 +0200 Subject: [PATCH 07/13] Replace client wait strategy with sqlcmd exec in mssql container --- modules/mssql/testcontainers/mssql/__init__.py | 12 +++++++++++- modules/mssql/tests/test_mssql.py | 14 +++++++++----- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/modules/mssql/testcontainers/mssql/__init__.py b/modules/mssql/testcontainers/mssql/__init__.py index 98b66826..ea818987 100644 --- a/modules/mssql/testcontainers/mssql/__init__.py +++ b/modules/mssql/testcontainers/mssql/__init__.py @@ -1,6 +1,8 @@ +import time from os import environ from typing import Optional +from testcontainers.core.config import MAX_TRIES, SLEEP_TIME from testcontainers.core.generic import DbContainer from testcontainers.core.utils import raise_for_deprecated_parameter @@ -30,7 +32,7 @@ def __init__( port: int = 1433, dbname: str = "tempdb", dialect: str = "mssql+pymssql", - **kwargs + **kwargs, ) -> None: raise_for_deprecated_parameter(kwargs, "user", "username") super().__init__(image, **kwargs) @@ -49,6 +51,14 @@ def _configure(self) -> None: self.with_env("SQLSERVER_DBNAME", self.dbname) self.with_env("ACCEPT_EULA", "Y") + def _connect(self) -> None: + for _ in range(MAX_TRIES): + status, _ = self.exec(f"/opt/mssql-tools/bin/sqlcmd -U {self.username} -P {self.password} -Q SELECT 1") + if status == 0: + return + else: + time.sleep(SLEEP_TIME) + def get_connection_url(self) -> str: return super()._create_connection_url( dialect=self.dialect, username=self.username, password=self.password, dbname=self.dbname, port=self.port diff --git a/modules/mssql/tests/test_mssql.py b/modules/mssql/tests/test_mssql.py index 6f48f0a1..812327dd 100644 --- a/modules/mssql/tests/test_mssql.py +++ b/modules/mssql/tests/test_mssql.py @@ -1,19 +1,23 @@ +import pytest import sqlalchemy +from testcontainers.core.utils import is_arm from testcontainers.mssql import SqlServerContainer -def test_docker_run_mssql(): - image = "mcr.microsoft.com/azure-sql-edge" - dialect = "mssql+pymssql" - with SqlServerContainer(image, dialect=dialect) as mssql: +@pytest.mark.skipif(is_arm(), reason="mysql container not available for ARM") +@pytest.mark.parametrize("version", ["2022-CU12-ubuntu-22.04", "2019-CU25-ubuntu-20.04"]) +def test_docker_run_mssql(version: str): + with SqlServerContainer(f"mcr.microsoft.com/mssql/server:{version}", password="1Secure*Password2") as mssql: engine = sqlalchemy.create_engine(mssql.get_connection_url()) with engine.begin() as connection: result = connection.execute(sqlalchemy.text("select @@servicename")) for row in result: assert row[0] == "MSSQLSERVER" - with SqlServerContainer(image, password="1Secure*Password2", dialect=dialect) as mssql: + +def test_docker_run_azure_sql_edge(): + with SqlServerContainer("mcr.microsoft.com/azure-sql-edge:1.0.7") as mssql: engine = sqlalchemy.create_engine(mssql.get_connection_url()) with engine.begin() as connection: result = connection.execute(sqlalchemy.text("select @@servicename")) From 0dd0ee18113277ce419954faf1722273d4bc83ed Mon Sep 17 00:00:00 2001 From: Vemund Santi Date: Wed, 3 Apr 2024 22:17:46 +0200 Subject: [PATCH 08/13] Pin MSSQL image in doctest --- modules/mssql/testcontainers/mssql/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/mssql/testcontainers/mssql/__init__.py b/modules/mssql/testcontainers/mssql/__init__.py index ea818987..de6464e3 100644 --- a/modules/mssql/testcontainers/mssql/__init__.py +++ b/modules/mssql/testcontainers/mssql/__init__.py @@ -18,7 +18,7 @@ class SqlServerContainer(DbContainer): >>> import sqlalchemy >>> from testcontainers.mssql import SqlServerContainer - >>> with SqlServerContainer() as mssql: + >>> with SqlServerContainer("mcr.microsoft.com/mssql/server:2022-CU12-ubuntu-22.04") as mssql: ... engine = sqlalchemy.create_engine(mssql.get_connection_url()) ... with engine.begin() as connection: ... result = connection.execute(sqlalchemy.text("select @@VERSION")) From 4bec99bfc513d0f120823ca5242d9abfbf44c7bf Mon Sep 17 00:00:00 2001 From: Vemund Santi Date: Wed, 3 Apr 2024 22:18:01 +0200 Subject: [PATCH 09/13] Fix typo in skip message --- modules/mssql/tests/test_mssql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/mssql/tests/test_mssql.py b/modules/mssql/tests/test_mssql.py index 812327dd..e7273042 100644 --- a/modules/mssql/tests/test_mssql.py +++ b/modules/mssql/tests/test_mssql.py @@ -5,7 +5,7 @@ from testcontainers.mssql import SqlServerContainer -@pytest.mark.skipif(is_arm(), reason="mysql container not available for ARM") +@pytest.mark.skipif(is_arm(), reason="mssql container not available for ARM") @pytest.mark.parametrize("version", ["2022-CU12-ubuntu-22.04", "2019-CU25-ubuntu-20.04"]) def test_docker_run_mssql(version: str): with SqlServerContainer(f"mcr.microsoft.com/mssql/server:{version}", password="1Secure*Password2") as mssql: From 7b863b3a9b2bb121fc33fa201ec125393a8c4a3c Mon Sep 17 00:00:00 2001 From: David Ankin Date: Sun, 14 Apr 2024 01:25:16 -0400 Subject: [PATCH 10/13] refer to config dataclass not constants --- core/testcontainers/core/waiting_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/testcontainers/core/waiting_utils.py b/core/testcontainers/core/waiting_utils.py index fb3d708c..cc3351d1 100644 --- a/core/testcontainers/core/waiting_utils.py +++ b/core/testcontainers/core/waiting_utils.py @@ -78,7 +78,7 @@ def wait_for(condition: Callable[..., bool]) -> bool: def wait_for_logs( - container: "DockerContainer", predicate: Union[Callable, str], timeout: float = config.TIMEOUT, interval: float = 1 + container: "DockerContainer", predicate: Union[Callable, str], timeout: float = config.timeout, interval: float = 1 ) -> float: """ Wait for the container to emit logs satisfying the predicate. From 050ac0ab6d4c06d59318dc6c64e0d268df4dc8a5 Mon Sep 17 00:00:00 2001 From: David Ankin Date: Sun, 14 Apr 2024 01:26:46 -0400 Subject: [PATCH 11/13] turns out this contribution was from ruff --- modules/mysql/tests/test_mysql.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/modules/mysql/tests/test_mysql.py b/modules/mysql/tests/test_mysql.py index 2e99711c..40eb536b 100644 --- a/modules/mysql/tests/test_mysql.py +++ b/modules/mysql/tests/test_mysql.py @@ -40,9 +40,10 @@ def test_docker_run_mariadb(version: str): def test_docker_env_variables(): - with mock.patch.dict("os.environ", MYSQL_USER="demo", MYSQL_DATABASE="custom_db"), MySqlContainer( - "mariadb:10.6.5" - ).with_bind_ports(3306, 32785) as container: + with ( + mock.patch.dict("os.environ", MYSQL_USER="demo", MYSQL_DATABASE="custom_db"), + MySqlContainer("mariadb:10.6.5").with_bind_ports(3306, 32785) as container, + ): url = container.get_connection_url() pattern = r"mysql\+pymysql:\/\/demo:test@[\w,.]+:(3306|32785)\/custom_db" assert re.match(pattern, url) From 9989ab874bf4de2c45a095d243a59fde9199af63 Mon Sep 17 00:00:00 2001 From: David Ankin Date: Sun, 14 Apr 2024 02:05:50 -0400 Subject: [PATCH 12/13] fix config reference --- modules/mssql/testcontainers/mssql/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/mssql/testcontainers/mssql/__init__.py b/modules/mssql/testcontainers/mssql/__init__.py index de6464e3..83462f7b 100644 --- a/modules/mssql/testcontainers/mssql/__init__.py +++ b/modules/mssql/testcontainers/mssql/__init__.py @@ -2,7 +2,7 @@ from os import environ from typing import Optional -from testcontainers.core.config import MAX_TRIES, SLEEP_TIME +from testcontainers.core.config import testcontainers_config as c from testcontainers.core.generic import DbContainer from testcontainers.core.utils import raise_for_deprecated_parameter @@ -52,12 +52,12 @@ def _configure(self) -> None: self.with_env("ACCEPT_EULA", "Y") def _connect(self) -> None: - for _ in range(MAX_TRIES): + for _ in range(c.max_tries): status, _ = self.exec(f"/opt/mssql-tools/bin/sqlcmd -U {self.username} -P {self.password} -Q SELECT 1") if status == 0: return else: - time.sleep(SLEEP_TIME) + time.sleep(c.sleep_time) def get_connection_url(self) -> str: return super()._create_connection_url( From 97d41efd1c60a09b55572c608a8918ba19d2fb73 Mon Sep 17 00:00:00 2001 From: David Ankin Date: Sun, 14 Apr 2024 02:25:35 -0400 Subject: [PATCH 13/13] fix waiting on mssql/sql server - it was taking 2min b/c maxing out --- modules/mssql/testcontainers/mssql/__init__.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/modules/mssql/testcontainers/mssql/__init__.py b/modules/mssql/testcontainers/mssql/__init__.py index 83462f7b..6cee3681 100644 --- a/modules/mssql/testcontainers/mssql/__init__.py +++ b/modules/mssql/testcontainers/mssql/__init__.py @@ -1,10 +1,9 @@ -import time from os import environ from typing import Optional -from testcontainers.core.config import testcontainers_config as c from testcontainers.core.generic import DbContainer from testcontainers.core.utils import raise_for_deprecated_parameter +from testcontainers.core.waiting_utils import wait_container_is_ready class SqlServerContainer(DbContainer): @@ -51,13 +50,10 @@ def _configure(self) -> None: self.with_env("SQLSERVER_DBNAME", self.dbname) self.with_env("ACCEPT_EULA", "Y") + @wait_container_is_ready(AssertionError) def _connect(self) -> None: - for _ in range(c.max_tries): - status, _ = self.exec(f"/opt/mssql-tools/bin/sqlcmd -U {self.username} -P {self.password} -Q SELECT 1") - if status == 0: - return - else: - time.sleep(c.sleep_time) + status, _ = self.exec(f"/opt/mssql-tools/bin/sqlcmd -U {self.username} -P {self.password} -Q 'SELECT 1'") + assert status == 0, "Cannot run 'SELECT 1': container is not ready" def get_connection_url(self) -> str: return super()._create_connection_url(