From b10d916848cccc016fc457333f7b382b18a7b3ef Mon Sep 17 00:00:00 2001 From: Dee Moore <117185602+deeninetyone@users.noreply.github.com> Date: Wed, 20 Mar 2024 12:50:03 +0200 Subject: [PATCH] fix(core): DinD issues #141, #329 (#368) Fix #141 - find IP from custom network if the container is not using the default network Close #329 - This seems fixed in the underlying docker libraries. Improve support for Docker in Docker running on a custom network, by attempting to find the right custom network and use it for new containers. This adds support for using testcontainers-python running the GitHub Actions Runner Controller to run self-hosted actions runners on prem, when you run your workflows in containers. --------- Co-authored-by: Dee Moore Co-authored-by: David Ankin Co-authored-by: Balint Bartha <39852431+totallyzen@users.noreply.github.com> --- core/testcontainers/core/docker_client.py | 54 +++++++++++++++++-- core/tests/test_docker_in_docker.py | 63 ++++++++++++++++++++--- 2 files changed, 107 insertions(+), 10 deletions(-) diff --git a/core/testcontainers/core/docker_client.py b/core/testcontainers/core/docker_client.py index 9c1ea485..04fdca59 100644 --- a/core/testcontainers/core/docker_client.py +++ b/core/testcontainers/core/docker_client.py @@ -11,8 +11,10 @@ # License for the specific language governing permissions and limitations # under the License. import functools as ft +import ipaddress import os import urllib +import urllib.parse from os.path import exists from pathlib import Path from typing import Optional, Union @@ -34,7 +36,7 @@ class DockerClient: """ def __init__(self, **kwargs) -> None: - docker_host = read_tc_properties().get("tc.host") + docker_host = get_docker_host() if docker_host: LOGGER.info(f"using host {docker_host}") @@ -57,6 +59,12 @@ def run( remove: bool = False, **kwargs, ) -> Container: + # If the user has specified a network, we'll assume the user knows best + if "network" not in kwargs and not get_docker_host(): + # Otherwise we'll try to find the docker host for dind usage. + host_network = self.find_host_network() + if host_network: + kwargs["network"] = host_network container = self.client.containers.run( image, command=command, @@ -71,6 +79,30 @@ def run( ) return container + def find_host_network(self) -> Optional[str]: + """ + Try to find the docker host network. + + :return: The network name if found, None if not set. + """ + # If we're docker in docker running on a custom network, we need to inherit the + # network settings, so we can access the resulting container. + try: + docker_host = ipaddress.IPv4Address(self.host()) + # See if we can find the host on our networks + for network in self.client.networks.list(filters={"type": "custom"}): + if "IPAM" in network.attrs: + for config in network.attrs["IPAM"]["Config"]: + try: + subnet = ipaddress.IPv4Network(config["Subnet"]) + except ipaddress.AddressValueError: + continue + if docker_host in subnet: + return network.name + except ipaddress.AddressValueError: + pass + return None + def port(self, container_id: str, port: int) -> int: """ Lookup the public-facing port that is NAT-ed to :code:`port`. @@ -94,14 +126,26 @@ def bridge_ip(self, container_id: str) -> str: Get the bridge ip address for a container. """ container = self.get_container(container_id) - return container["NetworkSettings"]["Networks"]["bridge"]["IPAddress"] + network_name = self.network_name(container_id) + return container["NetworkSettings"]["Networks"][network_name]["IPAddress"] + + def network_name(self, container_id: str) -> str: + """ + Get the name of the network this container runs on + """ + container = self.get_container(container_id) + name = container["HostConfig"]["NetworkMode"] + if name == "default": + return "bridge" + return name def gateway_ip(self, container_id: str) -> str: """ Get the gateway ip address for a container. """ container = self.get_container(container_id) - return container["NetworkSettings"]["Networks"]["bridge"]["Gateway"] + network_name = self.network_name(container_id) + return container["NetworkSettings"]["Networks"][network_name]["Gateway"] def host(self) -> str: """ @@ -145,3 +189,7 @@ def read_tc_properties() -> dict[str, str]: tuples = [line.split("=") for line in contents.readlines() if "=" in line] settings = {**settings, **{item[0].strip(): item[1].strip() for item in tuples}} return settings + + +def get_docker_host() -> Optional[str]: + return read_tc_properties().get("tc.host") or os.getenv("DOCKER_HOST") diff --git a/core/tests/test_docker_in_docker.py b/core/tests/test_docker_in_docker.py index 95392408..6a424884 100644 --- a/core/tests/test_docker_in_docker.py +++ b/core/tests/test_docker_in_docker.py @@ -1,11 +1,28 @@ -import pytest - +import time +import socket from testcontainers.core.container import DockerContainer from testcontainers.core.docker_client import DockerClient from testcontainers.core.waiting_utils import wait_for_logs -@pytest.mark.xfail(reason="https://github.com/docker/docker-py/issues/2717") +def _wait_for_dind_return_ip(client, dind): + # get ip address for DOCKER_HOST + # avoiding DockerContainer class here to prevent code changes affecting the test + docker_host_ip = client.bridge_ip(dind.id) + # Wait for startup + timeout = 10 + start_wait = time.perf_counter() + while True: + try: + with socket.create_connection((docker_host_ip, 2375), timeout=timeout): + break + except ConnectionRefusedError: + if time.perf_counter() - start_wait > timeout: + raise RuntimeError("Docker in docker took longer than 10 seconds to start") + time.sleep(0.01) + return docker_host_ip + + def test_wait_for_logs_docker_in_docker(): # real dind isn't possible (AFAIK) in CI # forwarding the socket to a container port is at least somewhat the same @@ -18,11 +35,38 @@ def test_wait_for_logs_docker_in_docker(): ) not_really_dind.start() + docker_host_ip = _wait_for_dind_return_ip(client, not_really_dind) + docker_host = f"tcp://{docker_host_ip}:2375" - # get ip address for DOCKER_HOST - # avoiding DockerContainer class here to prevent code changes affecting the test - specs = client.get_container(not_really_dind.id) - docker_host_ip = specs["NetworkSettings"]["Networks"]["bridge"]["IPAddress"] + with DockerContainer( + image="hello-world", + docker_client_kw={"environment": {"DOCKER_HOST": docker_host, "DOCKER_CERT_PATH": "", "DOCKER_TLS_VERIFY": ""}}, + ) as container: + assert container.get_container_host_ip() == docker_host_ip + wait_for_logs(container, "Hello from Docker!") + stdout, stderr = container.get_logs() + assert stdout, "There should be something on stdout" + + not_really_dind.stop() + not_really_dind.remove() + + +def test_dind_inherits_network(): + client = DockerClient() + try: + custom_network = client.client.networks.create("custom_network", driver="bridge", check_duplicate=True) + except Exception: + custom_network = client.client.networks.list(names=["custom_network"])[0] + not_really_dind = client.run( + image="alpine/socat", + command="tcp-listen:2375,fork,reuseaddr unix-connect:/var/run/docker.sock", + volumes={"/var/run/docker.sock": {"bind": "/var/run/docker.sock"}}, + detach=True, + ) + + not_really_dind.start() + + docker_host_ip = _wait_for_dind_return_ip(client, not_really_dind) docker_host = f"tcp://{docker_host_ip}:2375" with DockerContainer( @@ -30,9 +74,14 @@ def test_wait_for_logs_docker_in_docker(): docker_client_kw={"environment": {"DOCKER_HOST": docker_host, "DOCKER_CERT_PATH": "", "DOCKER_TLS_VERIFY": ""}}, ) as container: assert container.get_container_host_ip() == docker_host_ip + # Check the gateways are the same, so they can talk to each other + assert container.get_docker_client().gateway_ip(container.get_wrapped_container().id) == client.gateway_ip( + not_really_dind.id + ) wait_for_logs(container, "Hello from Docker!") stdout, stderr = container.get_logs() assert stdout, "There should be something on stdout" not_really_dind.stop() not_really_dind.remove() + custom_network.remove()