feat: support for custom registry prefixes at the configuration level (…

…#1928)

* feat: support for setting the reaper image at the properties level

* chore: simplify reaper creation using the config

* docs: document Ryuk properties/env vars

* chore: propose a Hub prefix instead

* feat: always apply the prefix substitution to all images

* chore: exclude certain conditions for the Hub prefix substitution

* docs: include docs about image substitutors and hub prefix

* chore: do not break users of the constant

* docs: update link

* docs: remove blanks

container.go

@@ -125,7 +125,7 @@ type ContainerRequest struct {
 	User                    string                                     // for specifying uid:gid
 	SkipReaper              bool                                       // Deprecated: The reaper is globally controlled by the .testcontainers.properties file or the TESTCONTAINERS_RYUK_DISABLED environment variable
 	ReaperImage             string                                     // Deprecated: use WithImageName ContainerOption instead. Alternative reaper image
-	ReaperOptions           []ContainerOption                          // options for the reaper
+	ReaperOptions           []ContainerOption                          // Deprecated: the reaper is configured at the properties level, for an entire test session
 	AutoRemove              bool                                       // Deprecated: Use HostConfigModifier instead. If set to true, the container will be removed from the host when stopped
 	AlwaysPullImage         bool                                       // Always pull image
 	ImagePlatform           string                                     // ImagePlatform describes the platform which the image runs on.
@@ -145,17 +145,19 @@ type containerOptions struct {
 	RegistryCredentials string // Deprecated: Testcontainers will detect registry credentials automatically
 }
 
+// Deprecated: it will be removed in the next major release
 // functional option for setting the reaper image
 type ContainerOption func(*containerOptions)
 
+// Deprecated: it will be removed in the next major release
 // WithImageName sets the reaper image name
 func WithImageName(imageName string) ContainerOption {
 	return func(o *containerOptions) {
 		o.ImageName = imageName
 	}
 }
 
-// Deprecated: Testcontainers will detect registry credentials automatically
+// Deprecated: Testcontainers will detect registry credentials automatically, and it will be removed in the next major release
 // WithRegistryCredentials sets the reaper registry credentials
 func WithRegistryCredentials(registryCredentials string) ContainerOption {
 	return func(o *containerOptions) {

container_test.go

@@ -306,11 +306,13 @@ func Test_BuildImageWithContexts(t *testing.T) {
 
 func Test_GetLogsFromFailedContainer(t *testing.T) {
 	ctx := context.Background()
+	// directDockerHubReference {
 	req := ContainerRequest{
 		Image:      "docker.io/alpine",
 		Cmd:        []string{"echo", "-n", "I was not expecting this"},
 		WaitingFor: wait.ForLog("I was expecting this").WithStartupTimeout(5 * time.Second),
 	}
+	// }
 
 	c, err := GenericContainer(ctx, GenericContainerRequest{
 		ContainerRequest: req,
@@ -340,6 +342,7 @@ func Test_GetLogsFromFailedContainer(t *testing.T) {
 	}
 }
 
+// dockerImageSubstitutor {
 type dockerImageSubstitutor struct{}
 
 func (s dockerImageSubstitutor) Description() string {
@@ -350,6 +353,8 @@ func (s dockerImageSubstitutor) Substitute(image string) (string, error) {
 	return "docker.io/" + image, nil
 }
 
+// }
+
 // noopImageSubstitutor {
 type NoopImageSubstitutor struct{}
 
@@ -506,3 +511,35 @@ func TestParseDockerIgnore(t *testing.T) {
 		assert.Equal(t, testCase.expectedExcluded, excluded)
 	}
 }
+
+func ExampleGenericContainer_withSubstitutors() {
+	ctx := context.Background()
+
+	// applyImageSubstitutors {
+	container, err := GenericContainer(ctx, GenericContainerRequest{
+		ContainerRequest: ContainerRequest{
+			Image:             "alpine:latest",
+			ImageSubstitutors: []ImageSubstitutor{dockerImageSubstitutor{}},
+		},
+		Started: true,
+	})
+	// }
+	if err != nil {
+		panic(err)
+	}
+
+	defer func() {
+		err := container.Terminate(ctx)
+		if err != nil {
+			panic(err)
+		}
+	}()
+
+	// enforce the concrete type, as GenericContainer returns an interface,
+	// which will be changed in future implementations of the library
+	dockerContainer := container.(*DockerContainer)
+
+	fmt.Println(dockerContainer.Image)
+
+	// Output: docker.io/alpine:latest
+}

docker.go

@@ -31,6 +31,7 @@ import (
 	specs "github.com/opencontainers/image-spec/specs-go/v1"
 
 	tcexec "github.com/testcontainers/testcontainers-go/exec"
+	"github.com/testcontainers/testcontainers-go/internal/config"
 	"github.com/testcontainers/testcontainers-go/internal/testcontainersdocker"
 	"github.com/testcontainers/testcontainers-go/internal/testcontainerssession"
 	"github.com/testcontainers/testcontainers-go/wait"
@@ -882,20 +883,13 @@ func (p *DockerProvider) CreateContainer(ctx context.Context, req ContainerReque
 		req.Labels = make(map[string]string)
 	}
 
-	reaperOpts := containerOptions{
-		ImageName: req.ReaperImage,
-	}
-	for _, opt := range req.ReaperOptions {
-		opt(&reaperOpts)
-	}
-
 	tcConfig := p.Config().Config
 
 	var termSignal chan bool
 	// the reaper does not need to start a reaper for itself
-	isReaperContainer := strings.EqualFold(imageName, reaperImage(reaperOpts.ImageName))
+	isReaperContainer := strings.HasSuffix(imageName, config.ReaperDefaultImage)
 	if !tcConfig.RyukDisabled && !isReaperContainer {
-		r, err := reuseOrCreateReaper(context.WithValue(ctx, testcontainersdocker.DockerHostContextKey, p.host), testcontainerssession.SessionID(), p, req.ReaperOptions...)
+		r, err := reuseOrCreateReaper(context.WithValue(ctx, testcontainersdocker.DockerHostContextKey, p.host), testcontainerssession.SessionID(), p)
 		if err != nil {
 			return nil, fmt.Errorf("%w: creating reaper failed", err)
 		}
@@ -916,14 +910,19 @@ func (p *DockerProvider) CreateContainer(ctx context.Context, req ContainerReque
 		return nil, err
 	}
 
+	// always append the hub substitutor after the user-defined ones
+	req.ImageSubstitutors = append(req.ImageSubstitutors, newPrependHubRegistry())
+
 	for _, is := range req.ImageSubstitutors {
 		modifiedTag, err := is.Substitute(imageName)
 		if err != nil {
 			return nil, fmt.Errorf("failed to substitute image %s with %s: %w", imageName, is.Description(), err)
 		}
 
-		p.Logger.Printf("✍🏼 Replacing image with %s. From: %s to %s\n", is.Description(), imageName, modifiedTag)
-		imageName = modifiedTag
+		if modifiedTag != imageName {
+			p.Logger.Printf("✍🏼 Replacing image with %s. From: %s to %s\n", is.Description(), imageName, modifiedTag)
+			imageName = modifiedTag
+		}
 	}
 
 	var platform *specs.Platform
@@ -1146,7 +1145,7 @@ func (p *DockerProvider) ReuseOrCreateContainer(ctx context.Context, req Contain
 
 	var termSignal chan bool
 	if !tcConfig.RyukDisabled {
-		r, err := reuseOrCreateReaper(context.WithValue(ctx, testcontainersdocker.DockerHostContextKey, p.host), sessionID, p, req.ReaperOptions...)
+		r, err := reuseOrCreateReaper(context.WithValue(ctx, testcontainersdocker.DockerHostContextKey, p.host), sessionID, p)
 		if err != nil {
 			return nil, fmt.Errorf("%w: creating reaper failed", err)
 		}
@@ -1314,7 +1313,7 @@ func (p *DockerProvider) CreateNetwork(ctx context.Context, req NetworkRequest)
 
 	var termSignal chan bool
 	if !tcConfig.RyukDisabled {
-		r, err := reuseOrCreateReaper(context.WithValue(ctx, testcontainersdocker.DockerHostContextKey, p.host), sessionID, p, req.ReaperOptions...)
+		r, err := reuseOrCreateReaper(context.WithValue(ctx, testcontainersdocker.DockerHostContextKey, p.host), sessionID, p)
 		if err != nil {
 			return nil, fmt.Errorf("%w: creating network reaper failed", err)
 		}

docs/features/common_functional_options.md

@@ -2,7 +2,16 @@
 
 - Since testcontainers-go <a href="https://github.com/testcontainers/testcontainers-go/releases/tag/v0.26.0"><span class="tc-version">:material-tag: v0.26.0</span></a>
 
-{% include "./image_name_substitution.md" %}
+In more locked down / secured environments, it can be problematic to pull images from Docker Hub and run them without additional precautions.
+
+An image name substitutor converts a Docker image name, as may be specified in code, to an alternative name. This is intended to provide a way to override image names, for example to enforce pulling of images from a private registry.
+
+_Testcontainers for Go_ exposes an interface to perform this operations: `ImageSubstitutor`, and a No-operation implementation to be used as reference for custom implementations:
+
+<!--codeinclude-->
+[Image Substitutor Interface](../../options.go) inside_block:imageSubstitutor
+[Noop Image Substitutor](../../container_test.go) inside_block:noopImageSubstitutor
+<!--/codeinclude-->
 
 Using the `WithImageSubstitutors` options, you could define your own substitutions to the container images. E.g. adding a prefix to the images so that they can be pulled from a Docker registry other than Docker Hub. This is the usual mechanism for using Docker image proxies, caches, etc.
 

docs/features/configuration.md

@@ -40,11 +40,19 @@ docker.tls.verify=1                         # Equivalent to the DOCKER_TLS_VERIF
 docker.cert.path=/some/path                 # Equivalent to the DOCKER_CERT_PATH environment variable
 ```
 
-### Disabling Ryuk
-Ryuk must be started as a privileged container.  
-If your environment already implements automatic cleanup of containers after the execution,
+## Customizing images
+
+Please read more about customizing images in the [Image name substitution](image_name_substitution.md) section.
+
+## Customizing Ryuk, the resource reaper
+
+1. Ryuk must be started as a privileged container. For that, you can set the `TESTCONTAINERS_RYUK_CONTAINER_PRIVILEGED` **environment variable**, or the  `ryuk.container.privileged` **property** to `true`.
+1. If your environment already implements automatic cleanup of containers after the execution,
 but does not allow starting privileged containers, you can turn off the Ryuk container by setting
 `TESTCONTAINERS_RYUK_DISABLED` **environment variable** to `true`.
+1. You can specify the connection timeout for Ryuk by setting the `ryuk.connection.timeout` **property**. The default value is 1 minute.
+1. You can specify the reconnection timeout for Ryuk by setting the `ryuk.reconnection.timeout` **property**. The default value is 10 seconds.
+
 
 !!!info
     For more information about Ryuk, see [Garbage Collector](garbage_collector.md).

docs/features/image_name_substitution.md

@@ -1,10 +1,95 @@
-In more locked down / secured environments, it can be problematic to pull images from Docker Hub and run them without additional precautions.
+# Image name substitution
 
-An image name substitutor converts a Docker image name, as may be specified in code, to an alternative name. This is intended to provide a way to override image names, for example to enforce pulling of images from a private registry.
+_Testcontainers for Go_ supports automatic substitution of Docker image names.
 
-_Testcontainers for Go_ exposes an interface to perform this operations: `ImageSubstitutor`, and a No-operation implementation to be used as reference for custom implementations:
+This allows the replacement of an image name specified in test code with an alternative name - for example, to replace the 
+name of a Docker Hub image dependency with an alternative hosted on a private image registry.
+
+This is advisable to avoid Docker Hub rate limiting, and some companies will prefer this for policy reasons.
+
+!!!info
+    As of November 2020 Docker Hub pulls are rate limited. As Testcontainers uses Docker Hub for standard images, some users may hit these rate limits and should mitigate accordingly. Suggested mitigations are noted in [this issue in Testcontainers for Java](https://github.com/testcontainers/testcontainers-java/issues/3099) at present.
+
+This page describes two approaches for image name substitution:
+
+* [Automatically modifying Docker Hub image names](#automatically-modifying-docker-hub-image-names), prefixing them with a private registry URL.
+* [Using an Image Name Substitutor](#developing-a-custom-function-for-transforming-image-names-on-the-fly), developing a custom function for transforming image names on the fly.
+
+!!!warning
+    It is assumed that you have already set up a private registry hosting [all the Docker images your build requires](../supported_docker_environment/image_registry_rate_limiting.md#which-images-are-used-by-testcontainers).
+
+## Automatically modifying Docker Hub image names
+
+_Testcontainers for Go_ can be configured to modify Docker Hub image names on the fly to apply a prefix string.
+
+Consider this if:
+
+* Developers and CI machines need to use different image names. For example, developers are able to pull images from Docker Hub, but CI machines need to pull from a private registry.
+* Your private registry has copies of images from Docker Hub where the names are predictable, and just adding a prefix is enough. 
+  For example, `registry.mycompany.com/mirror/mysql:8.0.24` can be derived from the original Docker Hub image name (`mysql:8.0.24`) with a consistent prefix string: `registry.mycompany.com/mirror`
+
+In this case, image name references in code are **unchanged**.
+i.e. you would leave as-is:
+
+<!--codeinclude-->
+[Unchanged direct Docker Hub image name](../../container_test.go) inside_block:directDockerHubReference
+<!--/codeinclude-->
+
+You can then configure _Testcontainers for Go_ to apply a given prefix (e.g. `registry.mycompany.com/mirror`) to every image that it tries to pull from Docker Hub. Important to notice that **the prefix should not include a trailing slash**. This can be done in one of two ways:
+
+* Setting the `TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX=registry.mycompany.com/mirror` environment variable.
+* Via config file, setting `hub.image.name.prefix` in the `~/.testcontainers.properties` file in your user home directory.
+
+_Testcontainers for Go_ will automatically apply the prefix to every image that it pulls from Docker Hub - please verify that all [the required images](#images-used-by-testcontainers) exist in your registry.
+
+_Testcontainers for Go_ will not apply the prefix to:
+
+* non-Hub image names (e.g. where another registry is set)
+* Docker Hub image names where the hub registry is explicitly part of the name (i.e. anything with a `docker.io` or `registry.hub.docker.com` host part)
+
+## Developing a custom function for transforming image names on the fly
+
+Consider this if:
+
+* You have complex rules about which private registry images should be used as substitutes, e.g.:
+    * non-deterministic mapping of names meaning that a [name prefix](#automatically-modifying-docker-hub-image-names) cannot be used, or
+    * rules depending upon developer identity or location, or
+* you wish to add audit logging of images used in the build, or
+* you wish to prevent accidental usage of images that are not on an approved list.
+
+In this case, image name references in code are **unchanged**. i.e. you would leave as-is:
+
+<!--codeinclude-->
+[Unchanged direct Docker Hub image name](../../container_test.go) inside_block:directDockerHubReference
+<!--/codeinclude-->
+
+You can implement a custom image name substitutor by:
+
+* implementing the `ImageNameSubstitutor` interface, exposed by the `testcontainers` package.
+* configuring _Testcontainers for Go_ to use your custom implementation, defined at the `ContainerRequest` level.
+
+The following is an example image substitutor implementation prepending the `docker.io/` prefix, used in the tests:
 
 <!--codeinclude-->
 [Image Substitutor Interface](../../options.go) inside_block:imageSubstitutor
-[Noop Image Substitutor](../../container_test.go) inside_block:noopImageSubstitutor
+[Docker prefix Image Substitutor](../../container_test.go) inside_block:dockerImageSubstitutor
+[Applying the substitutor](../../container_test.go) inside_block:applyImageSubstitutors
 <!--/codeinclude-->
+
+## Images used by Testcontainers
+
+As of the current version of Testcontainers ({{latest_version}}):
+
+* every image directly used by your tests
+* images pulled by Testcontainers itself to support functionality:
+    * [`testcontainers/ryuk`](https://hub.docker.com/r/testcontainers/ryuk) - performs fail-safe cleanup of containers, and always required (unless [Ryuk is disabled](./configuration.md#customizing-ryuk-the-resource-reaper)).
+    * [`alpine`](https://hub.docker.com/r/_/alpine).
+    * [`Docker in Docker`](https://hub.docker.com/_/docker).
+    * [`nginx`](https://hub.docker.com/r/_/nginx).
+    * [`delayed nginx`](https://hub.docker.com/r/menedev/delayed-nginx).
+    * [`localstack`](https://hub.docker.com/r/localstack/localstack).
+    * [`mysql`](https://hub.docker.com/r/_/mysql).
+    * [`postgres`](https://hub.docker.com/r/_/postgres).
+    * [`postgis`](https://hub.docker.com/r/postgis/postgis).
+    * [`redis`](https://hub.docker.com/r/_/redis).
+    * [`registry`](https://hub.docker.com/r/_/registry).

docs/system_requirements/ci/bitbucket_pipelines.md

@@ -2,7 +2,7 @@
 
 To enable access to Docker in Bitbucket Pipelines, you need to add `docker` as a service on the step.
 
-Furthermore, Ryuk needs to be turned off since Bitbucket Pipelines does not allow starting privileged containers (see [Disabling Ryuk](../../features/configuration.md#disabling-ryuk)). This can either be done by setting a repository variable in Bitbucket's project settings or by explicitly exporting the variable on a step.
+Furthermore, Ryuk needs to be turned off since Bitbucket Pipelines does not allow starting privileged containers (see [Disabling Ryuk](../../features/configuration.md#customizing-ryuk-the-resource-reaper)). This can either be done by setting a repository variable in Bitbucket's project settings or by explicitly exporting the variable on a step.
 
 In some cases the memory available to Docker needs to be increased.