diff --git a/docs/features/images.md b/docs/features/images.md index 5ce5a6882..b738d0755 100644 --- a/docs/features/images.md +++ b/docs/features/images.md @@ -90,3 +90,27 @@ const container = await GenericContainer .withCache(false) .build(); ``` + +### Dynamic build context + +If you would like to send a build context that you created in code (maybe you have a dynamic Dockerfile), you can send +the build context as a `NodeJS.ReadableStream` since the Docker Daemon accepts it as a _tar_ file. You can use the +[tar-fs](https://www.npmjs.com/package/tar-fs) (or [tar-stream](https://www.npmjs.com/package/tar-stream)) package to +create a custom dynamic context. + +```javascript +const tar = require('tar-stream'); + +const tarStream = tar.pack(); +tarStream.entry({ name: 'alpine.Dockerfile' }, + ` + FROM alpine:latest + CMD ["sleep", "infinity"] + ` +); +tarStream.finalize(); + +const container = await GenericContainer + .fromContextArchive(tarStream, 'alpine.Dockerfile') + .build(); +``` diff --git a/package-lock.json b/package-lock.json index 021320997..852fe0ec7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2240,9 +2240,10 @@ } }, "node_modules/@types/tar-stream": { - "version": "2.2.2", + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/@types/tar-stream/-/tar-stream-3.1.3.tgz", + "integrity": "sha512-Zbnx4wpkWBMBSu5CytMbrT5ZpMiF55qgM+EpHzR4yIDu7mv52cej8hTkOc6K+LzpkOAbxwn/m7j3iO+/l42YkQ==", "dev": true, - "license": "MIT", "dependencies": { "@types/node": "*" } @@ -12083,7 +12084,8 @@ }, "node_modules/tar-stream": { "version": "3.1.6", - "license": "MIT", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-3.1.6.tgz", + "integrity": "sha512-B/UyjYwPpMBv+PaFSWAmtYjwdrlEaZQEhMIBFNC5oEG8lpiW8XjcSdmEaClj28ArfKScKHs2nshz3k2le6crsg==", "dependencies": { "b4a": "^1.6.4", "fast-fifo": "^1.2.0", @@ -13173,7 +13175,9 @@ "@types/proper-lockfile": "^4.1.3", "@types/properties-reader": "^2.1.2", "@types/tar-fs": "^2.0.3", - "@types/tmp": "^0.2.5" + "@types/tar-stream": "^3.1.3", + "@types/tmp": "^0.2.5", + "tar-stream": "^3.1.6" } } } diff --git a/packages/testcontainers/package.json b/packages/testcontainers/package.json index 6f16ac098..83864694f 100644 --- a/packages/testcontainers/package.json +++ b/packages/testcontainers/package.json @@ -55,6 +55,8 @@ "@types/proper-lockfile": "^4.1.3", "@types/properties-reader": "^2.1.2", "@types/tar-fs": "^2.0.3", - "@types/tmp": "^0.2.5" + "@types/tar-stream": "^3.1.3", + "@types/tmp": "^0.2.5", + "tar-stream": "^3.1.6" } } diff --git a/packages/testcontainers/src/container-runtime/clients/image/docker-image-client.ts b/packages/testcontainers/src/container-runtime/clients/image/docker-image-client.ts index 0c57fce34..6ec95a8fe 100644 --- a/packages/testcontainers/src/container-runtime/clients/image/docker-image-client.ts +++ b/packages/testcontainers/src/container-runtime/clients/image/docker-image-client.ts @@ -1,14 +1,11 @@ import Dockerode, { ImageBuildOptions } from "dockerode"; import byline from "byline"; -import tar from "tar-fs"; -import path from "path"; -import { existsSync, promises as fs } from "fs"; -import dockerIgnore from "@balena/dockerignore"; import { getAuthConfig } from "../../auth/get-auth-config"; import { ImageName } from "../../image-name"; import { ImageClient } from "./image-client"; import AsyncLock from "async-lock"; import { log, buildLog, pullLog } from "../../../common"; +import stream from "stream"; export class DockerImageClient implements ImageClient { private readonly existingImages = new Set(); @@ -16,23 +13,12 @@ export class DockerImageClient implements ImageClient { constructor(protected readonly dockerode: Dockerode, protected readonly indexServerAddress: string) {} - async build(context: string, opts: ImageBuildOptions): Promise { + async build(context: stream.Readable, opts: ImageBuildOptions): Promise { try { - log.debug(`Building image "${opts.t}" with context "${context}"...`); - const isDockerIgnored = await this.createIsDockerIgnoredFunction(context); - const tarStream = tar.pack(context, { - ignore: (aPath) => { - const relativePath = path.relative(context, aPath); - if (relativePath === opts.dockerfile) { - return false; - } else { - return isDockerIgnored(relativePath); - } - }, - }); + log.debug(`Building image "${opts.t}"...`); await new Promise((resolve) => { this.dockerode - .buildImage(tarStream, opts) + .buildImage(context, opts) .then((stream) => byline(stream)) .then((stream) => { stream.setEncoding("utf-8"); @@ -44,27 +30,13 @@ export class DockerImageClient implements ImageClient { stream.on("end", () => resolve()); }); }); - log.debug(`Built image "${opts.t}" with context "${context}"`); + log.debug(`Built image "${opts.t}"`); } catch (err) { log.error(`Failed to build image: ${err}`); throw err; } } - private async createIsDockerIgnoredFunction(context: string): Promise<(path: string) => boolean> { - const dockerIgnoreFilePath = path.join(context, ".dockerignore"); - if (!existsSync(dockerIgnoreFilePath)) { - return () => false; - } - - const dockerIgnorePatterns = await fs.readFile(dockerIgnoreFilePath, { encoding: "utf-8" }); - const instance = dockerIgnore({ ignorecase: false }); - instance.add(dockerIgnorePatterns); - const filter = instance.createFilter(); - - return (aPath: string) => !filter(aPath); - } - async exists(imageName: ImageName): Promise { return this.imageExistsLock.acquire(imageName.string, async () => { if (this.existingImages.has(imageName.string)) { diff --git a/packages/testcontainers/src/container-runtime/clients/image/image-client.ts b/packages/testcontainers/src/container-runtime/clients/image/image-client.ts index 15a1eed9b..70d421804 100644 --- a/packages/testcontainers/src/container-runtime/clients/image/image-client.ts +++ b/packages/testcontainers/src/container-runtime/clients/image/image-client.ts @@ -2,7 +2,7 @@ import { ImageBuildOptions } from "dockerode"; import { ImageName } from "../../image-name"; export interface ImageClient { - build(context: string, opts: ImageBuildOptions): Promise; + build(context: NodeJS.ReadableStream, opts: ImageBuildOptions): Promise; pull(imageName: ImageName, opts?: { force: boolean }): Promise; exists(imageName: ImageName): Promise; } diff --git a/packages/testcontainers/src/generic-container/generic-container-builder.ts b/packages/testcontainers/src/generic-container/generic-container-builder.ts index a30acfe75..d7ad0e638 100644 --- a/packages/testcontainers/src/generic-container/generic-container-builder.ts +++ b/packages/testcontainers/src/generic-container/generic-container-builder.ts @@ -7,6 +7,10 @@ import { getAuthConfig, getContainerRuntimeClient, ImageName } from "../containe import { getReaper } from "../reaper/reaper"; import { getDockerfileImages } from "../utils/dockerfile-parser"; import { createLabels, LABEL_TESTCONTAINERS_SESSION_ID } from "../utils/labels"; +import tar from "tar-fs"; +import { existsSync, promises as fs } from "fs"; +import dockerIgnore from "@balena/dockerignore"; +import Dockerode from "dockerode"; export type BuildOptions = { deleteOnExit: boolean; @@ -19,12 +23,12 @@ export class GenericContainerBuilder { private target?: string; constructor( - private readonly context: string, + private readonly context: NodeJS.ReadableStream | string, private readonly dockerfileName: string, private readonly uuid: Uuid = new RandomUuid() ) {} - public withBuildArgs(buildArgs: BuildArgs): GenericContainerBuilder { + public withBuildArgs(buildArgs: BuildArgs): this { this.buildArgs = buildArgs; return this; } @@ -44,6 +48,12 @@ export class GenericContainerBuilder { return this; } + /** + * Build the image. + * + * @param image - The image name to tag the built image with. + * @param options - Options for the build. Defaults to `{ deleteOnExit: true }`. + */ public async build( image = `localhost/${this.uuid.nextUuid()}:${this.uuid.nextUuid()}`, options: BuildOptions = { deleteOnExit: true } @@ -52,26 +62,49 @@ export class GenericContainerBuilder { const reaper = await getReaper(client); const imageName = ImageName.fromString(image); - const dockerfile = path.resolve(this.context, this.dockerfileName); - - const imageNames = await getDockerfileImages(dockerfile, this.buildArgs); - const registryConfig = await this.getRegistryConfig(client.info.containerRuntime.indexServerAddress, imageNames); const labels = createLabels(); if (options.deleteOnExit) { labels[LABEL_TESTCONTAINERS_SESSION_ID] = reaper.sessionId; } - log.info(`Building Dockerfile "${dockerfile}" as image "${imageName}"...`); - await client.image.build(this.context, { + let contextStream: NodeJS.ReadableStream; + const imageBuildOptions: Dockerode.ImageBuildOptions = { t: imageName.string, dockerfile: this.dockerfileName, buildargs: this.buildArgs, pull: this.pullPolicy ? "true" : undefined, nocache: !this.cache, - registryconfig: registryConfig, labels, target: this.target, - }); + }; + + if (typeof this.context !== "string") { + contextStream = this.context; + } else { + // Get the registry config for the images in the Dockerfile + const dockerfile = path.resolve(this.context, this.dockerfileName); + const imageNames = await getDockerfileImages(dockerfile, this.buildArgs); + imageBuildOptions.registryconfig = await this.getRegistryConfig( + client.info.containerRuntime.indexServerAddress, + imageNames + ); + + // Create a tar stream of the context directory, excluding the files that are ignored by .dockerignore + const dockerignoreFilter = await newDockerignoreFilter(this.context); + contextStream = tar.pack(this.context, { + ignore: (aPath) => { + const relativePath = path.relative(this.context, aPath); + if (relativePath === this.dockerfileName) { + return false; + } else { + return dockerignoreFilter(relativePath); + } + }, + }); + } + + log.info(`Building Dockerfile "${this.dockerfileName}" as image "${imageName.string}"...`); + await client.image.build(contextStream, imageBuildOptions); const container = new GenericContainer(imageName.string); if (!(await client.image.exists(imageName))) { @@ -105,3 +138,17 @@ export class GenericContainerBuilder { .reduce((prev, next) => ({ ...prev, ...next }), {} as RegistryConfig); } } + +async function newDockerignoreFilter(context: string): Promise<(path: string) => boolean> { + const dockerIgnoreFilePath = path.join(context, ".dockerignore"); + if (!existsSync(dockerIgnoreFilePath)) { + return () => false; + } + + const dockerIgnorePatterns = await fs.readFile(dockerIgnoreFilePath, { encoding: "utf-8" }); + const instance = dockerIgnore({ ignorecase: false }); + instance.add(dockerIgnorePatterns); + const filter = instance.createFilter(); + + return (aPath: string) => !filter(aPath); +} diff --git a/packages/testcontainers/src/generic-container/generic-container-dockerfile.test.ts b/packages/testcontainers/src/generic-container/generic-container-dockerfile.test.ts index 7f24163ea..98aba091c 100644 --- a/packages/testcontainers/src/generic-container/generic-container-dockerfile.test.ts +++ b/packages/testcontainers/src/generic-container/generic-container-dockerfile.test.ts @@ -1,4 +1,6 @@ +import fs from "fs"; import path from "path"; +import tar from "tar-stream"; import { GenericContainer } from "./generic-container"; import { Wait } from "../wait-strategies/wait"; import { PullPolicy } from "../utils/pull-policy"; @@ -103,3 +105,27 @@ describe("GenericContainer Dockerfile", () => { await startedContainer.stop(); }); }); + +describe("GenericContainer fromContextArchive", () => { + jest.setTimeout(180_000); + + const fixtures = path.resolve(__dirname, "..", "..", "fixtures", "docker"); + + it("should build and start", async () => { + const fixturesCtx = path.resolve(fixtures, "docker"); + + // Generate a context stream with a Dockerfile named "alpine.Dockerfile" + const tarStream = tar.pack(); + tarStream.entry({ name: "alpine.Dockerfile" }, fs.readFileSync(path.join(fixturesCtx, "Dockerfile"))); + tarStream.entry({ name: "index.js" }, fs.readFileSync(path.join(fixturesCtx, "index.js"))); + tarStream.entry({ name: "test.txt" }, "hello world"); + tarStream.finalize(); + + const container = await GenericContainer.fromContextArchive(tarStream, "alpine.Dockerfile").build(); + const startedContainer = await container.withExposedPorts(8080).start(); + + await checkContainerIsHealthy(startedContainer); + + await startedContainer.stop(); + }); +}); diff --git a/packages/testcontainers/src/generic-container/generic-container.ts b/packages/testcontainers/src/generic-container/generic-container.ts index 70da281af..63b9b04a1 100644 --- a/packages/testcontainers/src/generic-container/generic-container.ts +++ b/packages/testcontainers/src/generic-container/generic-container.ts @@ -36,10 +36,29 @@ import { mapInspectResult } from "../utils/map-inspect-result"; const reusableContainerCreationLock = new AsyncLock(); export class GenericContainer implements TestContainer { + /** + * Create a docker image from a Dockerfile. + * + * @param context - The build context directory path. + * @param dockerfileName - The name of the Dockerfile. Default is `Dockerfile`. + */ public static fromDockerfile(context: string, dockerfileName = "Dockerfile"): GenericContainerBuilder { return new GenericContainerBuilder(context, dockerfileName); } + /** + * Create a docker image from a context archive stream. + * + * @param context - The build context archive stream. + * @param dockerfileName - The name of the Dockerfile. Default is `Dockerfile`. + */ + public static fromContextArchive( + context: NodeJS.ReadableStream, + dockerfileName = "Dockerfile" + ): GenericContainerBuilder { + return new GenericContainerBuilder(context, dockerfileName); + } + protected createOpts: ContainerCreateOptions; protected hostConfig: HostConfig;