From 176679b128a6f8de52a95b9a6114c07d5787e82d Mon Sep 17 00:00:00 2001 From: Jb DOYON Date: Tue, 21 May 2024 00:53:48 +0100 Subject: [PATCH 1/6] Fix mysql "seed" feature's docstring --- modules/mysql/testcontainers/mysql/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/mysql/testcontainers/mysql/__init__.py b/modules/mysql/testcontainers/mysql/__init__.py index 46efbcfb..257c7359 100644 --- a/modules/mysql/testcontainers/mysql/__init__.py +++ b/modules/mysql/testcontainers/mysql/__init__.py @@ -50,8 +50,10 @@ class MySqlContainer(DbContainer): automatically. .. doctest:: + >>> import sqlalchemy >>> from testcontainers.mysql import MySqlContainer + >>> with MySqlContainer(seed="../../tests/seeds/") as mysql: ... engine = sqlalchemy.create_engine(mysql.get_connection_url()) ... with engine.begin() as connection: From 77647821efb0282805b1840d3b9a96c3cd25dfa9 Mon Sep 17 00:00:00 2001 From: Jb DOYON Date: Tue, 21 May 2024 00:54:21 +0100 Subject: [PATCH 2/6] Add seed feature to postgres --- .../testcontainers/postgres/__init__.py | 34 +++++++++++++++++++ modules/postgres/tests/seeds/01-schema.sql | 5 +++ modules/postgres/tests/seeds/02-seeds.sql | 4 +++ modules/postgres/tests/test_postgres.py | 11 ++++++ 4 files changed, 54 insertions(+) create mode 100644 modules/postgres/tests/seeds/01-schema.sql create mode 100644 modules/postgres/tests/seeds/02-seeds.sql diff --git a/modules/postgres/testcontainers/postgres/__init__.py b/modules/postgres/testcontainers/postgres/__init__.py index 9b347aa6..b23d2ba8 100644 --- a/modules/postgres/testcontainers/postgres/__init__.py +++ b/modules/postgres/testcontainers/postgres/__init__.py @@ -11,6 +11,9 @@ # License for the specific language governing permissions and limitations # under the License. import os +import tarfile +from io import BytesIO +from pathlib import Path from time import sleep from typing import Optional @@ -45,6 +48,24 @@ class PostgresContainer(DbContainer): ... version, = result.fetchone() >>> version 'PostgreSQL 16...' + + The optional :code:`seed` parameter enables arbitrary SQL files to be loaded. + This is perfect for schema and sample data. This works by mounting the seed to + `/docker-entrypoint-initdb./d`, which containerized Postgres are set up to load + automatically. + + .. doctest:: + + >>> from testcontainers.postgres import PostgresContainer + >>> import sqlalchemy + >>> + >>> with PostgresContainer(seed="../../tests/seeds/") as postgres: + ... engine = sqlalchemy.create_engine(postgres.get_connection_url()) + ... with engine.begin() as connection: + ... query = "select * from stuff" # Can now rely on schema/data + ... result = connection.execute(sqlalchemy.text(query)) + ... first_stuff, = result.fetchone() + """ def __init__( @@ -55,6 +76,7 @@ def __init__( password: Optional[str] = None, dbname: Optional[str] = None, driver: Optional[str] = "psycopg2", + seed: Optional[str] = None, **kwargs, ) -> None: raise_for_deprecated_parameter(kwargs, "user", "username") @@ -64,6 +86,7 @@ def __init__( self.dbname: str = dbname or os.environ.get("POSTGRES_DB", "test") self.port = port self.driver = f"+{driver}" if driver else "" + self.seed = seed self.with_exposed_ports(self.port) @@ -103,3 +126,14 @@ def _connect(self) -> None: count += 1 raise RuntimeError("Postgres could not get into a ready state") + + def _transfer_seed(self) -> None: + if self.seed is None: + return + src_path = Path(self.seed) + dest_path = "/docker-entrypoint-initdb.d/" + with BytesIO() as archive, tarfile.TarFile(fileobj=archive, mode="w") as tar: + for filename in src_path.iterdir(): + tar.add(filename.absolute(), arcname=filename.relative_to(src_path)) + archive.seek(0) + self.get_wrapped_container().put_archive(dest_path, archive) diff --git a/modules/postgres/tests/seeds/01-schema.sql b/modules/postgres/tests/seeds/01-schema.sql new file mode 100644 index 00000000..91e3d756 --- /dev/null +++ b/modules/postgres/tests/seeds/01-schema.sql @@ -0,0 +1,5 @@ +-- Sample SQL schema, no data +CREATE TABLE stuff ( + id integer primary key generated always as identity, + name text NOT NULL +); diff --git a/modules/postgres/tests/seeds/02-seeds.sql b/modules/postgres/tests/seeds/02-seeds.sql new file mode 100644 index 00000000..30fa938d --- /dev/null +++ b/modules/postgres/tests/seeds/02-seeds.sql @@ -0,0 +1,4 @@ +-- Sample data, to be loaded after the schema +INSERT INTO stuff (name) +VALUES ('foo'), ('bar'), ('qux'), ('frob') +RETURNING id; diff --git a/modules/postgres/tests/test_postgres.py b/modules/postgres/tests/test_postgres.py index 52840361..90d427a0 100644 --- a/modules/postgres/tests/test_postgres.py +++ b/modules/postgres/tests/test_postgres.py @@ -38,6 +38,17 @@ def test_docker_run_postgres_with_sqlalchemy(): assert row[0].lower().startswith("postgresql 9.5") +def test_docker_run_postgres_seeds_with_sqlalchemy(): + # Avoid pytest CWD path issues + SEEDS_PATH = (Path(__file__).parent / "seeds").absolute() + postgres_container = PostgresContainer("postgres:latest", seed=SEEDS_PATH) + with postgres_container as postgres: + engine = sqlalchemy.create_engine(postgres.get_connection_url()) + with engine.begin() as connection: + result = connection.execute(sqlalchemy.text("select * from stuff")) + assert len(list(result)) == 4, "Should have gotten all the stuff" + + def test_docker_run_postgres_with_driver_pg8000(): postgres_container = PostgresContainer("postgres:9.5", driver="pg8000") with postgres_container as postgres: From 6a681f0f6f3edb373e97c188ea970b543a43fa6c Mon Sep 17 00:00:00 2001 From: Jb DOYON Date: Sun, 2 Jun 2024 01:31:05 +0100 Subject: [PATCH 3/6] Replace mysql command init to wait for files --- core/testcontainers/core/generic.py | 45 ++++++++++++++++++- .../mysql/testcontainers/mysql/__init__.py | 22 +++------ 2 files changed, 50 insertions(+), 17 deletions(-) diff --git a/core/testcontainers/core/generic.py b/core/testcontainers/core/generic.py index 6dd635e6..66e8c3d6 100644 --- a/core/testcontainers/core/generic.py +++ b/core/testcontainers/core/generic.py @@ -10,6 +10,9 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. +import tarfile +from io import BytesIO +from pathlib import Path from typing import Optional from urllib.parse import quote @@ -26,6 +29,13 @@ except ImportError: pass +SEEDS_CONTAINER_PATH = "/docker-entrypoint-initdb.d/" +TRANSFER_COMPLETE_SENTINEL_PATH = "/sentinel" +TRANSFER_COMPLETE_SENTINEL_FILEPATH = "/sentinel/completed" +TRANSFER_COMPLETE_SENTINEL_CONTENTS = BytesIO( + bytes(f"Testcontainers seeds folder transferred to {SEEDS_CONTAINER_PATH}", encoding="utf-8") +) + class DbContainer(DockerContainer): """ @@ -78,4 +88,37 @@ def _configure(self) -> None: raise NotImplementedError def _transfer_seed(self) -> None: - pass + if self.seed is None: + return + src_path = Path(self.seed) + container = self.get_wrapped_container() + transfer_folder(container, src_path, SEEDS_CONTAINER_PATH) + transfer_file_contents(container, TRANSFER_COMPLETE_SENTINEL_CONTENTS, TRANSFER_COMPLETE_SENTINEL_PATH) + + def override_command_for_seed(self): + """Replace the image's command for seed purposes""" + image_info = self._docker.client.api.inspect_image(self.image) + cmd_list: list[str] = image_info["Config"]["Cmd"] + self.original_cmd = " ".join(cmd_list) + sentinel = TRANSFER_COMPLETE_SENTINEL_FILEPATH + command = f'sh -c "set -x;mkdir {TRANSFER_COMPLETE_SENTINEL_PATH}; while [ ! -f {sentinel} ]; do sleep 0.1; done; echo SENTINELOK; chmod u+x {sentinel}; ls -hal {sentinel};source /usr/local/bin/docker-entrypoint.sh; _main {self.original_cmd}"' + self.with_command(command) + + +def transfer_folder(container, local_path, remote_path): + """Transfer local_path to remote_path on the given container, using put_archive""" + with BytesIO() as archive, tarfile.TarFile(fileobj=archive, mode="w") as tar: + for filename in local_path.iterdir(): + tar.add(filename.absolute(), arcname=filename.relative_to(local_path)) + archive.seek(0) + container.put_archive(remote_path, archive) + + +def transfer_file_contents(container, content, remote_path): + """Create a file from raw content to remote_path on container, using put_archive""" + with BytesIO() as archive, tarfile.TarFile(fileobj=archive, mode="w") as tar: + tarinfo = tarfile.TarInfo(name="completed") + tarinfo.size = len(content.getvalue()) + tar.addfile(tarinfo, fileobj=content) + archive.seek(0) + container.put_archive(remote_path, archive) diff --git a/modules/mysql/testcontainers/mysql/__init__.py b/modules/mysql/testcontainers/mysql/__init__.py index 257c7359..a69c4e35 100644 --- a/modules/mysql/testcontainers/mysql/__init__.py +++ b/modules/mysql/testcontainers/mysql/__init__.py @@ -11,16 +11,15 @@ # License for the specific language governing permissions and limitations # under the License. import re -import tarfile -from io import BytesIO from os import environ -from pathlib import Path from typing import Optional from testcontainers.core.generic import DbContainer -from testcontainers.core.utils import raise_for_deprecated_parameter +from testcontainers.core.utils import raise_for_deprecated_parameter, setup_logger from testcontainers.core.waiting_utils import wait_for_logs +LOGGER = setup_logger(__name__) + class MySqlContainer(DbContainer): """ @@ -70,8 +69,8 @@ def __init__( root_password: Optional[str] = None, password: Optional[str] = None, dbname: Optional[str] = None, - port: int = 3306, seed: Optional[str] = None, + port: int = 3306, **kwargs, ) -> None: raise_for_deprecated_parameter(kwargs, "MYSQL_USER", "username") @@ -90,6 +89,8 @@ def __init__( if self.username == "root": self.root_password = self.password self.seed = seed + if self.seed is not None: + super().override_command_for_seed() def _configure(self) -> None: self.with_env("MYSQL_ROOT_PASSWORD", self.root_password) @@ -109,14 +110,3 @@ def get_connection_url(self) -> str: return super()._create_connection_url( dialect="mysql+pymysql", username=self.username, password=self.password, dbname=self.dbname, port=self.port ) - - def _transfer_seed(self) -> None: - if self.seed is None: - return - src_path = Path(self.seed) - dest_path = "/docker-entrypoint-initdb.d/" - with BytesIO() as archive, tarfile.TarFile(fileobj=archive, mode="w") as tar: - for filename in src_path.iterdir(): - tar.add(filename.absolute(), arcname=filename.relative_to(src_path)) - archive.seek(0) - self.get_wrapped_container().put_archive(dest_path, archive) From 9dfc358f933b8f88482515f48d6fd47eef8b8fab Mon Sep 17 00:00:00 2001 From: Jb DOYON Date: Sun, 2 Jun 2024 02:01:45 +0100 Subject: [PATCH 4/6] Refactor mysql mountpoint to generic --- core/testcontainers/core/generic.py | 43 +++++++++++-------- .../mysql/testcontainers/mysql/__init__.py | 5 ++- 2 files changed, 30 insertions(+), 18 deletions(-) diff --git a/core/testcontainers/core/generic.py b/core/testcontainers/core/generic.py index 66e8c3d6..ff2deccc 100644 --- a/core/testcontainers/core/generic.py +++ b/core/testcontainers/core/generic.py @@ -29,12 +29,9 @@ except ImportError: pass -SEEDS_CONTAINER_PATH = "/docker-entrypoint-initdb.d/" -TRANSFER_COMPLETE_SENTINEL_PATH = "/sentinel" -TRANSFER_COMPLETE_SENTINEL_FILEPATH = "/sentinel/completed" -TRANSFER_COMPLETE_SENTINEL_CONTENTS = BytesIO( - bytes(f"Testcontainers seeds folder transferred to {SEEDS_CONTAINER_PATH}", encoding="utf-8") -) +SENTINEL_FOLDER = "/sentinel" +SENTINEL_FILENAME = "completed" +SENTINEL_FULLPATH = f"{SENTINEL_FOLDER}/{SENTINEL_FILENAME}" class DbContainer(DockerContainer): @@ -92,19 +89,30 @@ def _transfer_seed(self) -> None: return src_path = Path(self.seed) container = self.get_wrapped_container() - transfer_folder(container, src_path, SEEDS_CONTAINER_PATH) - transfer_file_contents(container, TRANSFER_COMPLETE_SENTINEL_CONTENTS, TRANSFER_COMPLETE_SENTINEL_PATH) + transfer_folder(container, src_path, self.seed_mountpoint) + transfer_file_contents(container, "Sentinel completed", SENTINEL_FOLDER) - def override_command_for_seed(self): + def override_command_for_seed(self, startup_command): """Replace the image's command for seed purposes""" - image_info = self._docker.client.api.inspect_image(self.image) - cmd_list: list[str] = image_info["Config"]["Cmd"] - self.original_cmd = " ".join(cmd_list) - sentinel = TRANSFER_COMPLETE_SENTINEL_FILEPATH - command = f'sh -c "set -x;mkdir {TRANSFER_COMPLETE_SENTINEL_PATH}; while [ ! -f {sentinel} ]; do sleep 0.1; done; echo SENTINELOK; chmod u+x {sentinel}; ls -hal {sentinel};source /usr/local/bin/docker-entrypoint.sh; _main {self.original_cmd}"' + image_cmd = get_image_cmd(self._docker.client, self.image) + cmd_full = " ".join([startup_command, image_cmd]) + command = f"""sh -c " + mkdir {SENTINEL_FOLDER}; + while [ ! -f {SENTINEL_FULLPATH} ]; + do + sleep 0.1; + done; + {cmd_full}" + """ self.with_command(command) +def get_image_cmd(client, image): + image_info = client.api.inspect_image(image) + cmd_list: list[str] = image_info["Config"]["Cmd"] + return " ".join(cmd_list) + + def transfer_folder(container, local_path, remote_path): """Transfer local_path to remote_path on the given container, using put_archive""" with BytesIO() as archive, tarfile.TarFile(fileobj=archive, mode="w") as tar: @@ -114,10 +122,11 @@ def transfer_folder(container, local_path, remote_path): container.put_archive(remote_path, archive) -def transfer_file_contents(container, content, remote_path): - """Create a file from raw content to remote_path on container, using put_archive""" +def transfer_file_contents(container, content_str, remote_path): + """Create a file from raw content_str to remote_path on container, via put_archive""" with BytesIO() as archive, tarfile.TarFile(fileobj=archive, mode="w") as tar: - tarinfo = tarfile.TarInfo(name="completed") + tarinfo = tarfile.TarInfo(name=SENTINEL_FILENAME) + content = BytesIO(bytes(content_str, encoding="utf-8")) tarinfo.size = len(content.getvalue()) tar.addfile(tarinfo, fileobj=content) archive.seek(0) diff --git a/modules/mysql/testcontainers/mysql/__init__.py b/modules/mysql/testcontainers/mysql/__init__.py index a69c4e35..bfe7153e 100644 --- a/modules/mysql/testcontainers/mysql/__init__.py +++ b/modules/mysql/testcontainers/mysql/__init__.py @@ -62,6 +62,9 @@ class MySqlContainer(DbContainer): """ + seed_mountpoint: str = "/docker-entrypoint-initdb.d/" + startup_command: str = "source /usr/local/bin/docker-entrypoint.sh; _main " + def __init__( self, image: str = "mysql:latest", @@ -90,7 +93,7 @@ def __init__( self.root_password = self.password self.seed = seed if self.seed is not None: - super().override_command_for_seed() + super().override_command_for_seed(self.startup_command) def _configure(self) -> None: self.with_env("MYSQL_ROOT_PASSWORD", self.root_password) From b9a45ad9d2e66a411e921ef39fe66e336acca235 Mon Sep 17 00:00:00 2001 From: Jb DOYON Date: Mon, 3 Jun 2024 00:21:49 +0100 Subject: [PATCH 5/6] Replace postgres seed with DbContainer solution Issue currently is the tests for postgres fail: The container exits on postgres (not mysql, for identical both entrypoint and command...) with error: sh: 8: source: not found sh: 8: _main: not found I'm still tracking it down, because it's weird that /bin/sh says "source" does not exist (but not with mysql) but the _main should be available from the sourced entrypoint script. --- .../testcontainers/postgres/__init__.py | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/modules/postgres/testcontainers/postgres/__init__.py b/modules/postgres/testcontainers/postgres/__init__.py index b23d2ba8..624aea3c 100644 --- a/modules/postgres/testcontainers/postgres/__init__.py +++ b/modules/postgres/testcontainers/postgres/__init__.py @@ -11,9 +11,6 @@ # License for the specific language governing permissions and limitations # under the License. import os -import tarfile -from io import BytesIO -from pathlib import Path from time import sleep from typing import Optional @@ -68,6 +65,9 @@ class PostgresContainer(DbContainer): """ + seed_mountpoint: str = "/docker-entrypoint-initdb.d/" + startup_command: str = "source /usr/local/bin/docker-entrypoint.sh; _main " + def __init__( self, image: str = "postgres:latest", @@ -87,6 +87,8 @@ def __init__( self.port = port self.driver = f"+{driver}" if driver else "" self.seed = seed + if self.seed is not None: + super().override_command_for_seed(self.startup_command) self.with_exposed_ports(self.port) @@ -126,14 +128,3 @@ def _connect(self) -> None: count += 1 raise RuntimeError("Postgres could not get into a ready state") - - def _transfer_seed(self) -> None: - if self.seed is None: - return - src_path = Path(self.seed) - dest_path = "/docker-entrypoint-initdb.d/" - with BytesIO() as archive, tarfile.TarFile(fileobj=archive, mode="w") as tar: - for filename in src_path.iterdir(): - tar.add(filename.absolute(), arcname=filename.relative_to(src_path)) - archive.seek(0) - self.get_wrapped_container().put_archive(dest_path, archive) From 0eec7ae767f9cf960d3d6bdb472fe7b8fd8b633c Mon Sep 17 00:00:00 2001 From: Jb DOYON Date: Sun, 7 Jul 2024 16:11:21 +0100 Subject: [PATCH 6/6] Wrap command in bash -c --- core/testcontainers/core/generic.py | 2 +- modules/postgres/tests/test_postgres.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/testcontainers/core/generic.py b/core/testcontainers/core/generic.py index ff2deccc..96c2425d 100644 --- a/core/testcontainers/core/generic.py +++ b/core/testcontainers/core/generic.py @@ -102,7 +102,7 @@ def override_command_for_seed(self, startup_command): do sleep 0.1; done; - {cmd_full}" + bash -c '{cmd_full}'" """ self.with_command(command) diff --git a/modules/postgres/tests/test_postgres.py b/modules/postgres/tests/test_postgres.py index 90d427a0..5ec77e9b 100644 --- a/modules/postgres/tests/test_postgres.py +++ b/modules/postgres/tests/test_postgres.py @@ -41,7 +41,7 @@ def test_docker_run_postgres_with_sqlalchemy(): def test_docker_run_postgres_seeds_with_sqlalchemy(): # Avoid pytest CWD path issues SEEDS_PATH = (Path(__file__).parent / "seeds").absolute() - postgres_container = PostgresContainer("postgres:latest", seed=SEEDS_PATH) + postgres_container = PostgresContainer("postgres", seed=SEEDS_PATH) with postgres_container as postgres: engine = sqlalchemy.create_engine(postgres.get_connection_url()) with engine.begin() as connection: