From dd37f2a48e185c5da9f04bc9b4a869365a767650 Mon Sep 17 00:00:00 2001 From: Chris Wiggins Date: Thu, 9 Jan 2025 14:32:47 +1300 Subject: [PATCH] feat: Add toxiproxy module --- docs/modules/toxiproxy.md | 68 +++++++ mkdocs.yml | 1 + packages/modules/toxiproxy/jest.config.ts | 11 ++ packages/modules/toxiproxy/package.json | 39 ++++ packages/modules/toxiproxy/src/index.ts | 1 + .../toxiproxy/src/toxiproxy-container.test.ts | 183 ++++++++++++++++++ .../toxiproxy/src/toxiproxy-container.ts | 92 +++++++++ .../modules/toxiproxy/tsconfig.build.json | 13 ++ packages/modules/toxiproxy/tsconfig.json | 21 ++ 9 files changed, 429 insertions(+) create mode 100644 docs/modules/toxiproxy.md create mode 100644 packages/modules/toxiproxy/jest.config.ts create mode 100644 packages/modules/toxiproxy/package.json create mode 100644 packages/modules/toxiproxy/src/index.ts create mode 100644 packages/modules/toxiproxy/src/toxiproxy-container.test.ts create mode 100644 packages/modules/toxiproxy/src/toxiproxy-container.ts create mode 100644 packages/modules/toxiproxy/tsconfig.build.json create mode 100644 packages/modules/toxiproxy/tsconfig.json diff --git a/docs/modules/toxiproxy.md b/docs/modules/toxiproxy.md new file mode 100644 index 000000000..c0658fa94 --- /dev/null +++ b/docs/modules/toxiproxy.md @@ -0,0 +1,68 @@ +# Toxiproxy Module + +Testcontainers module for Shopify's [Toxiproxy](https://github.com/Shopify/toxiproxy). +This TCP proxy can be used to simulate network failure conditions. + +You can simulate network failures: + +* between NodeJS code and containers, ideal for testing resilience features of client code +* between containers, for testing resilience and emergent behaviour of multi-container systems +* if desired, between NodeJS code/containers and external resources (non-Dockerized!), for scenarios where not all dependencies can be/have been dockerized + +Testcontainers Toxiproxy support allows resilience features to be easily verified as part of isolated dev/CI testing. This allows earlier testing of resilience features, and broader sets of failure conditions to be covered. + +## Install +```bash +npm install @testcontainers/toxiproxy --save-dev +``` + +## Usage example + +A Toxiproxy container can be placed in between test code and a container, or in between containers. +In either scenario, it is necessary to create a `ToxiProxyContainer` instance on the same Docker network. + +Next, it is necessary to instruct Toxiproxy to start proxying connections. +Each `ToxiProxyContainer` can proxy to many target containers if necessary. + +A proxy is created by calling `createProxy` on the `ToxiProxyContainer` instance. + +The client connecting to the proxied endpoint then needs to use the exposed port from the returned proxy. + +All of this is done as follows: + +[Creating, starting and using the container:](../../packages/modules/toxiproxy/src/toxiproxy-container.test.ts) inside_block:create_proxy + + +!!! note + Currently, `ToxiProxyContainer` will reserve 31 ports, starting at 8666. After this, trying to create a new proxy instance will throw an error. + + +Having done all of this, it is possible to trigger failure conditions ('Toxics') through the `proxy.instance.addToxic()` object: + +`TPClient` is the internal `toxiproxy-node-client` re-exported in this package. + +* `bandwidth` - Limit a connection to a maximum number of kilobytes per second. +* `latency` - Add a delay to all data going through the proxy. The delay is equal to `latency +/- jitter`. +* `slicer` - Slices TCP data up into small bits, optionally adding a delay between each sliced "packet". +* `slow_close` - Delay the TCP socket from closing until `delay` milliseconds has elapsed. +* `timeout` - Stops all data from getting through, and closes the connection after `timeout`. If `timeout` is `0`, the connection won't close, and data will be delayed until the toxic is removed. +* `limit_data` - Closes connection when transmitted data exceeded limit. +* `reset_peer` - Simulate TCP RESET (Connection reset by peer) on the connections + +Please see the [Toxiproxy documentation](https://github.com/Shopify/toxiproxy#toxics) and the [toxiproxy-node-client](https://github.com/ihsw/toxiproxy-node-client) for full details on the available Toxics. + +As one example, we can introduce latency and random jitter to proxied connections as follows: + + +[Adding latency to a connection](../../packages/modules/toxiproxy/src/toxiproxy-container.test.ts) inside_block:adding_toxic + + +There is also a helper method to enable / disable specific proxy instances (for more fine-grained control instead of using the `reset_peer` toxic). This can also be done by calling the `proxy.instance.update` method, however it is more complicated as you'll need to supply the upstream again and the internal listening port. + + +[Enable and disable the proxy:](../../packages/modules/toxiproxy/src/toxiproxy-container.test.ts) inside_block:enabled_disabled + + +## Acknowledgements + +This module was inspired by the Java implementation, and under the hood uses the [toxiproxy-node-client](https://github.com/ihsw/toxiproxy-node-client). diff --git a/mkdocs.yml b/mkdocs.yml index 65cdb41c4..9baa130ac 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -67,5 +67,6 @@ nav: - Redpanda: modules/redpanda.md - ScyllaDB: modules/scylladb.md - Selenium: modules/selenium.md + - ToxiProxy: modules/toxiproxy.md - Weaviate: modules/weaviate.md - Configuration: configuration.md diff --git a/packages/modules/toxiproxy/jest.config.ts b/packages/modules/toxiproxy/jest.config.ts new file mode 100644 index 000000000..1f677baaf --- /dev/null +++ b/packages/modules/toxiproxy/jest.config.ts @@ -0,0 +1,11 @@ +import type { Config } from "jest"; +import * as path from "path"; + +const config: Config = { + preset: "ts-jest", + moduleNameMapper: { + "^testcontainers$": path.resolve(__dirname, "../../testcontainers/src"), + }, +}; + +export default config; diff --git a/packages/modules/toxiproxy/package.json b/packages/modules/toxiproxy/package.json new file mode 100644 index 000000000..b9cb4257e --- /dev/null +++ b/packages/modules/toxiproxy/package.json @@ -0,0 +1,39 @@ +{ + "name": "@testcontainers/toxiproxy", + "version": "10.16.0", + "license": "MIT", + "keywords": [ + "toxiproxy", + "testing", + "docker", + "testcontainers" + ], + "description": "Toxiproxy module for Testcontainers", + "homepage": "https://github.com/testcontainers/testcontainers-node#readme", + "repository": { + "type": "git", + "url": "https://github.com/testcontainers/testcontainers-node" + }, + "bugs": { + "url": "https://github.com/testcontainers/testcontainers-node/issues" + }, + "main": "build/index.js", + "files": [ + "build" + ], + "publishConfig": { + "access": "public" + }, + "scripts": { + "prepack": "shx cp ../../../README.md . && shx cp ../../../LICENSE .", + "build": "tsc --project tsconfig.build.json" + }, + "dependencies": { + "testcontainers": "^10.16.0", + "toxiproxy-node-client": "^4.0.0" + }, + "devDependencies": { + "@testcontainers/redis": "^10.16.0", + "redis": "^4.7.0" + } +} diff --git a/packages/modules/toxiproxy/src/index.ts b/packages/modules/toxiproxy/src/index.ts new file mode 100644 index 000000000..aa92c227b --- /dev/null +++ b/packages/modules/toxiproxy/src/index.ts @@ -0,0 +1 @@ +export { ToxiProxyContainer, StartedToxiProxyContainer, CreatedProxy } from "./toxiproxy-container"; diff --git a/packages/modules/toxiproxy/src/toxiproxy-container.test.ts b/packages/modules/toxiproxy/src/toxiproxy-container.test.ts new file mode 100644 index 000000000..72f2ca351 --- /dev/null +++ b/packages/modules/toxiproxy/src/toxiproxy-container.test.ts @@ -0,0 +1,183 @@ +import { ToxiProxyContainer, TPClient } from "./toxiproxy-container"; +import { GenericContainer, Network } from "testcontainers"; +import { createClient } from "redis"; + +describe("ToxiProxyContainer", () => { + jest.setTimeout(240_000); + + // Helper to connect to redis + async function connectTo(url: string) { + const client = createClient({ + url, + }); + client.on("error", () => {}); // Ignore errors + await client.connect(); + expect(client.isOpen).toBeTruthy(); + return client; + } + + // create_proxy { + it("Should create a proxy to an endpoint", async () => { + const containerNetwork = await new Network().start(); + const redisContainer = await new GenericContainer("redis:7.2") + .withNetwork(containerNetwork) + .withNetworkAliases("redis") + .start(); + + const toxiproxyContainer = await new ToxiProxyContainer().withNetwork(containerNetwork).start(); + + // Create the proxy between Toxiproxy and Redis + const redisProxy = await toxiproxyContainer.createProxy({ + name: "redis", + upstream: "redis:6379", + }); + + const url = `redis://${redisProxy.host}:${redisProxy.port}`; + const client = await connectTo(url); + await client.set("key", "val"); + expect(await client.get("key")).toBe("val"); + + await client.disconnect(); + await toxiproxyContainer.stop(); + await redisContainer.stop(); + }); + // } + + // enabled_disabled { + it("Should enable and disable a proxy", async () => { + const containerNetwork = await new Network().start(); + const redisContainer = await new GenericContainer("redis:7.2") + .withNetwork(containerNetwork) + .withNetworkAliases("redis") + .start(); + + const toxiproxyContainer = await new ToxiProxyContainer().withNetwork(containerNetwork).start(); + + // Create the proxy between Toxiproxy and Redis + const redisProxy = await toxiproxyContainer.createProxy({ + name: "redis", + upstream: "redis:6379", + }); + + const url = `redis://${redisProxy.host}:${redisProxy.port}`; + const client = await connectTo(url); + + await client.set("key", "val"); + expect(await client.get("key")).toBe("val"); + + // Disable any new connections to the proxy + await redisProxy.setEnabled(false); + + await expect(client.ping()).rejects.toThrow(); + + // Enable the proxy again + await redisProxy.setEnabled(true); + + expect(await client.ping()).toBe("PONG"); + + await client.disconnect(); + await toxiproxyContainer.stop(); + await redisContainer.stop(); + }); + // } + + // adding_toxic { + it("Should add a toxic to a proxy and then remove", async () => { + const containerNetwork = await new Network().start(); + const redisContainer = await new GenericContainer("redis:7.2") + .withNetwork(containerNetwork) + .withNetworkAliases("redis") + .start(); + + const toxiproxyContainer = await new ToxiProxyContainer().withNetwork(containerNetwork).start(); + + // Create the proxy between Toxiproxy and Redis + const redisProxy = await toxiproxyContainer.createProxy({ + name: "redis", + upstream: "redis:6379", + }); + + const url = `redis://${redisProxy.host}:${redisProxy.port}`; + const client = await connectTo(url); + + // See https://github.com/ihsw/toxiproxy-node-client for details on the instance interface + const toxic = await redisProxy.instance.addToxic({ + attributes: { + jitter: 50, + latency: 1500, + }, + name: "upstream-latency", + stream: "upstream", + toxicity: 1, // 1 is 100% + type: "latency", + }); + + const before = Date.now(); + await client.ping(); + const after = Date.now(); + expect(after - before).toBeGreaterThan(1000); + + await toxic.remove(); + + await client.disconnect(); + await toxiproxyContainer.stop(); + await redisContainer.stop(); + }); + // } + + it("Should create multiple proxies", async () => { + const containerNetwork = await new Network().start(); + const redisContainer = await new GenericContainer("redis:7.2") + .withNetwork(containerNetwork) + .withNetworkAliases("redis") + .start(); + + const toxiproxyContainer = await new ToxiProxyContainer().withNetwork(containerNetwork).start(); + + // Create the proxy between Toxiproxy and Redis + const redisProxy = await toxiproxyContainer.createProxy({ + name: "redis", + upstream: "redis:6379", + }); + + // Create the proxy between Toxiproxy and Redis + const redisProxy2 = await toxiproxyContainer.createProxy({ + name: "redis2", + upstream: "redis:6379", + }); + + const url = `redis://${redisProxy.host}:${redisProxy.port}`; + const client = await connectTo(url); + await client.set("key", "val"); + expect(await client.get("key")).toBe("val"); + + const url2 = `redis://${redisProxy2.host}:${redisProxy2.port}`; + const client2 = await connectTo(url2); + expect(await client2.get("key")).toBe("val"); + + await client.disconnect(); + await client2.disconnect(); + await toxiproxyContainer.stop(); + await redisContainer.stop(); + }); + + it("Throws an error when too many proxies are created", async () => { + const toxiproxyContainer = await new ToxiProxyContainer().start(); + + for (let i = 0; i < 32; i++) { + await toxiproxyContainer.createProxy({ + name: "test-" + i, + upstream: `google.com:80`, + }); + } + + await expect( + toxiproxyContainer.createProxy({ + name: "test-32", + upstream: `google.com:80`, + }) + ).rejects.toThrow(); + + await toxiproxyContainer.stop(); + }); +}); diff --git a/packages/modules/toxiproxy/src/toxiproxy-container.ts b/packages/modules/toxiproxy/src/toxiproxy-container.ts new file mode 100644 index 000000000..6630708c5 --- /dev/null +++ b/packages/modules/toxiproxy/src/toxiproxy-container.ts @@ -0,0 +1,92 @@ +import { AbstractStartedContainer, GenericContainer, StartedTestContainer, Wait } from "testcontainers"; +import * as TPClient from "toxiproxy-node-client"; + +const CONTROL_PORT = 8474; +const FIRST_PROXIED_PORT = 8666; + +const PORT_ARRAY = Array.from({ length: 32 }, (_, i) => i + FIRST_PROXIED_PORT); + +export interface CreatedProxy { + host: string; + port: number; + instance: TPClient.Proxy; + setEnabled: (enabled: boolean) => Promise; +} + +// Export this so that types can be used externally +export { TPClient }; + +export class ToxiProxyContainer extends GenericContainer { + constructor(image = "ghcr.io/shopify/toxiproxy:2.11.0") { + super(image); + + this.withExposedPorts(CONTROL_PORT, ...PORT_ARRAY) + .withWaitStrategy(Wait.forHttp("/version", CONTROL_PORT)) + .withStartupTimeout(30_000); + } + + public override async start(): Promise { + return new StartedToxiProxyContainer(await super.start()); + } +} + +export class StartedToxiProxyContainer extends AbstractStartedContainer { + /** + * + */ + public readonly client: TPClient.Toxiproxy; + + /** + * + * @param startedTestContainer + */ + constructor(startedTestContainer: StartedTestContainer) { + super(startedTestContainer); + + this.client = new TPClient.Toxiproxy(`http://${this.getHost()}:${this.getMappedPort(CONTROL_PORT)}`); + } + + public async createProxy(createProxyBody: Omit): Promise { + // Firstly get the list of proxies to find the next available port + const proxies = await this.client.getAll(); + + const usedPorts = PORT_ARRAY.reduce((acc, port) => { + acc[port] = false; + return acc; + }, {} as Record); + + for (const proxy of Object.values(proxies)) { + const lastColon = proxy.listen.lastIndexOf(":"); + const port = parseInt(proxy.listen.substring(lastColon + 1), 10); + usedPorts[port] = true; + } + + // Find the first available port + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const port = Object.entries(usedPorts).find(([_, used]) => !used); + if (!port) { + throw new Error("No available ports left"); + } + + const listen = `0.0.0.0:${port[0]}`; + + const proxy = await this.client.createProxy({ + ...createProxyBody, + listen, + }); + + const setEnabled = (enabled: boolean) => + proxy.update({ + enabled, + listen, + upstream: createProxyBody.upstream, + }); + + return { + host: this.getHost(), + port: this.getMappedPort(parseInt(port[0], 10)), + instance: proxy, + setEnabled, + }; + } +} diff --git a/packages/modules/toxiproxy/tsconfig.build.json b/packages/modules/toxiproxy/tsconfig.build.json new file mode 100644 index 000000000..0222f6ff1 --- /dev/null +++ b/packages/modules/toxiproxy/tsconfig.build.json @@ -0,0 +1,13 @@ +{ + "extends": "./tsconfig.json", + "exclude": [ + "build", + "jest.config.ts", + "src/**/*.test.ts" + ], + "references": [ + { + "path": "../../testcontainers" + } + ] +} \ No newline at end of file diff --git a/packages/modules/toxiproxy/tsconfig.json b/packages/modules/toxiproxy/tsconfig.json new file mode 100644 index 000000000..39b165817 --- /dev/null +++ b/packages/modules/toxiproxy/tsconfig.json @@ -0,0 +1,21 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "build", + "paths": { + "testcontainers": [ + "../../testcontainers/src" + ] + } + }, + "exclude": [ + "build", + "jest.config.ts" + ], + "references": [ + { + "path": "../../testcontainers" + } + ] +} \ No newline at end of file