diff --git a/packages/pg-config/src/PgConfig.ts b/packages/pg-config/src/PgConfig.ts index 9dbf4326..b1a069a1 100644 --- a/packages/pg-config/src/PgConfig.ts +++ b/packages/pg-config/src/PgConfig.ts @@ -18,6 +18,14 @@ export interface TestConfig { * @default "postgres:10.6-alpine" */ image: string; + /** + * Set this to a volume name to automatically + * persist data to a docker volume of that name. + * + * N.B. this will automatically remove "-ram" + * from the end of any image name. + */ + persistVolume?: string; /** * The default name to give the docker * containers run by @database/pg-test diff --git a/packages/pg-config/src/PgConfig.validator.ts b/packages/pg-config/src/PgConfig.validator.ts index ad7e0c32..5d225a74 100644 --- a/packages/pg-config/src/PgConfig.validator.ts +++ b/packages/pg-config/src/PgConfig.validator.ts @@ -61,6 +61,11 @@ export const PgConfigSchema = { description: 'Optional script to run after the database\nhas been started but before running tests', }, + persistVolume: { + description: + 'Set this to a volume name to automatically\npersist data to a docker volume of that name.\n\nN.B. this will automatically remove "-ram"\nfrom the end of any image name.', + type: 'string', + }, pgDb: { default: 'test-db', description: 'The db to create in the test docker container', diff --git a/packages/pg-test/package.json b/packages/pg-test/package.json index aa4922a6..4342ccfb 100644 --- a/packages/pg-test/package.json +++ b/packages/pg-test/package.json @@ -4,9 +4,13 @@ "description": "", "main": "./lib/index.js", "types": "./lib/index.d.ts", + "bin": { + "pg-test": "./lib/cli.js" + }, "dependencies": { "@databases/pg-config": "^0.0.0", "@databases/with-container": "^0.0.0", + "generate-alphabetic-name": "^1.0.0", "modern-spawn": "^1.0.0" }, "scripts": {}, @@ -16,7 +20,8 @@ "access": "public" }, "devDependencies": { - "@types/node": "^13.13.4" + "@types/node": "^13.13.4", + "@databases/pg": "^0.0.0" }, "bugs": "https://github.com/ForbesLindesay/atdatabases/issues", "homepage": "https://www.atdatabases.org/docs/pg-test", diff --git a/packages/pg-test/src/__tests__/.gitignore b/packages/pg-test/src/__tests__/.gitignore new file mode 100644 index 00000000..c4618616 --- /dev/null +++ b/packages/pg-test/src/__tests__/.gitignore @@ -0,0 +1 @@ +env-example-* \ No newline at end of file diff --git a/packages/pg-test/src/__tests__/__snapshots__/integration.test.ts.snap b/packages/pg-test/src/__tests__/__snapshots__/integration.test.ts.snap new file mode 100644 index 00000000..c305f4ff --- /dev/null +++ b/packages/pg-test/src/__tests__/__snapshots__/integration.test.ts.snap @@ -0,0 +1,5 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`integration: DATABASE_URL for atdatabases-test-volume-1 1`] = `"postgres://test-user@localhost:47936/test-db"`; + +exports[`integration: DATABASE_URL for atdatabases-test-volume-2 1`] = `"postgres://test-user@localhost:9833/test-db"`; diff --git a/packages/pg-test/src/__tests__/integration.test.ts b/packages/pg-test/src/__tests__/integration.test.ts new file mode 100644 index 00000000..ff367877 --- /dev/null +++ b/packages/pg-test/src/__tests__/integration.test.ts @@ -0,0 +1,65 @@ +import connect, {sql} from '@databases/pg'; +import {readFileSync} from 'fs'; +import call from '../cli-function'; +import {spawnBuffered} from 'modern-spawn'; + +jest.setTimeout(60_000); +if (!process.env.CI) { + const cleanup = async () => { + for (const volume of [ + 'atdatabases-test-volume-1', + 'atdatabases-test-volume-2', + ]) { + // these may fail without harm, so we don't check the exit code + await spawnBuffered('docker', ['kill', volume], { + debug: false, + }); + await spawnBuffered('docker', ['volume', 'rm', volume], { + debug: false, + }); + } + }; + afterAll(cleanup); + beforeAll(cleanup); + test('integration', async () => { + for (const volume of [ + 'atdatabases-test-volume-1', + 'atdatabases-test-volume-2', + ]) { + await spawnBuffered('docker', ['volume', 'create', volume], { + debug: true, + }).getResult(); + await call([ + 'start', + '--persistVolume', + volume, + '--writeEnv', + `${__dirname}/env-example-1`, + ]); + + const dbURL = readFileSync(`${__dirname}/env-example-1`, 'utf8') + .trim() + .replace(/DATABASE_URL=/, ''); + expect(dbURL).toMatchSnapshot(`DATABASE_URL for ${volume}`); + const db = connect(dbURL); + await db.query(sql`CREATE TABLE entries (id INT NOT NULL PRIMARY KEY)`); + await db.query(sql`INSERT INTO entries (id) VALUES (1), (2)`); + await db.dispose(); + + await call(['stop', '--persistVolume', volume]); + + await call(['start', '--persistVolume', volume]); + + const db2 = connect(dbURL); + expect(await db2.query(sql`SELECT * FROM entries`)).toEqual([ + {id: 1}, + {id: 2}, + ]); + await db2.dispose(); + + await call(['stop', '--persistVolume', volume]); + } + }); +} else { + test.skip('Cannot run in CI because we need to control docker images'); +} diff --git a/packages/pg-test/src/__tests__/numberToValidPort.test.ts b/packages/pg-test/src/__tests__/numberToValidPort.test.ts new file mode 100644 index 00000000..681df1b7 --- /dev/null +++ b/packages/pg-test/src/__tests__/numberToValidPort.test.ts @@ -0,0 +1,10 @@ +import numberToValidPort from '../numberToValidPort'; + +test('numberToValidPort', () => { + expect(numberToValidPort(0, 5, 7)).toBe(5); + expect(numberToValidPort(1, 5, 7)).toBe(6); + expect(numberToValidPort(2, 5, 7)).toBe(7); + expect(numberToValidPort(3, 5, 7)).toBe(5); + expect(numberToValidPort(4, 5, 7)).toBe(6); + expect(numberToValidPort(5, 5, 7)).toBe(7); +}); diff --git a/packages/pg-test/src/__tests__/stringToValidPort.test.ts b/packages/pg-test/src/__tests__/stringToValidPort.test.ts new file mode 100644 index 00000000..95a0270c --- /dev/null +++ b/packages/pg-test/src/__tests__/stringToValidPort.test.ts @@ -0,0 +1,10 @@ +import stringToValidPort from '../stringToValidPort'; + +test('numberToValidPort', () => { + expect(stringToValidPort('volume-a', 1000, 2000)).toMatchInlineSnapshot( + `1980`, + ); + expect(stringToValidPort('volume-b', 1000, 2000)).toMatchInlineSnapshot( + `1224`, + ); +}); diff --git a/packages/pg-test/src/cli-function.ts b/packages/pg-test/src/cli-function.ts new file mode 100644 index 00000000..302e1589 --- /dev/null +++ b/packages/pg-test/src/cli-function.ts @@ -0,0 +1,96 @@ +#!/usr/bin/env node + +import getDatabase, {stopDatabase} from '.'; +import {readFileSync, writeFileSync} from 'fs'; +import stringToValidPort from './stringToValidPort'; + +export default async function call(args: string[]): Promise { + if (args[0] === 'help' || args.includes('--help') || args.includes('-h')) { + usage(); + return 0; + } + + const PORT_RANGE = [1024, 49151] as const; + const debug = args.includes('--debug') ? true : undefined; + function get(key: string) { + return args.includes(key) ? args[args.indexOf(key) + 1] : undefined; + } + + const persistVolume = get('--persistVolume'); + const containerName = + get('--containerName') || + (persistVolume ? `pg-test-${persistVolume}` : undefined); + + const portStr = get('port'); + if ( + portStr && + (!/^\d+$/.test(portStr) || + portStr.length > PORT_RANGE[1].toString().length || + parseInt(portStr, 10) < PORT_RANGE[0] || + parseInt(portStr, 10) > PORT_RANGE[1]) + ) { + console.error( + `The port must be a valid integer between ${PORT_RANGE[0]} and ${PORT_RANGE[1]} (inclusive)`, + ); + return 1; + } + + const defaultExternalPort = portStr + ? parseInt(portStr, 10) + : persistVolume + ? stringToValidPort(persistVolume, PORT_RANGE[0], PORT_RANGE[1]) + : undefined; + + const command = args[0]; + switch (command) { + case 'start': + const result = await getDatabase({ + detached: true, + debug, + containerName, + pgUser: get('--pgUser'), + pgDb: get('--pgDb'), + persistVolume, + externalPort: portStr ? defaultExternalPort : undefined, + defaultExternalPort, + refreshImage: args.includes('--refreshImage') ? true : undefined, + }); + const envLine = 'DATABASE_URL=' + result.databaseURL; + const writeEnv = get('--writeEnv'); + if (args.includes('--writeEnv')) { + const envFile = writeEnv && writeEnv[0] !== '-' ? writeEnv : '.env'; + console.info('Writing to .env'); + console.info(''); + console.info(envLine); + try { + let dotenv = readFileSync(envFile, 'utf8'); + + if (/^DATABASE_URL *=.*$/gm.test(dotenv)) { + dotenv = dotenv.replace(/^DATABASE_URL *=.*$/gm, envLine); + } else { + dotenv += '\n' + envLine + '\n'; + } + writeFileSync(envFile, dotenv); + } catch (ex) { + if (ex.code !== 'ENOENT') { + throw ex; + } + writeFileSync(envFile, envLine + '\n'); + } + } else { + console.info(envLine); + } + return 0; + case 'stop': + await stopDatabase({debug, containerName}); + return 0; + default: + usage(); + return 1; + } + + function usage() { + console.info('Usage: pg-test start'); + console.info('Usage: pg-test stop'); + } +} diff --git a/packages/pg-test/src/cli.ts b/packages/pg-test/src/cli.ts new file mode 100644 index 00000000..0424f5d3 --- /dev/null +++ b/packages/pg-test/src/cli.ts @@ -0,0 +1,10 @@ +#!/usr/bin/env node + +import call from './cli-function'; + +call(process.argv.slice(2)) + .then(code => process.exit(code)) + .catch(ex => { + console.error(ex); + process.exit(1); + }); diff --git a/packages/pg-test/src/index.ts b/packages/pg-test/src/index.ts index f20645fc..e584737c 100644 --- a/packages/pg-test/src/index.ts +++ b/packages/pg-test/src/index.ts @@ -1,5 +1,6 @@ import startContainer, { Options as WithContainerOptions, + killOldContainers, } from '@databases/with-container'; import {getPgConfigSync} from '@databases/pg-config'; @@ -31,10 +32,34 @@ export interface Options > { pgUser: string; pgDb: string; + persistVolume?: string; } +export async function stopDatabase( + options: Partial> = {}, +) { + await killOldContainers({ + debug: options.debug !== undefined ? options.debug : DEFAULT_PG_DEBUG, + containerName: options.containerName || DEFAULT_CONTAINER_NAME, + }); +} +function withDefaults(overrides: Partial, defaults: T): T { + const result = {...defaults}; + Object.keys(overrides).forEach(key => { + if ((overrides as any)[key] !== undefined) { + (result as any)[key] = (overrides as any)[key]; + } + }); + return result; +} export default async function getDatabase(options: Partial = {}) { - const {pgUser, pgDb, environment, ...rawOptions}: Options = { + const { + pgUser, + pgDb, + environment, + persistVolume, + ...rawOptions + }: Options = withDefaults(options, { debug: DEFAULT_PG_DEBUG, image: DEFAULT_IMAGE, containerName: DEFAULT_CONTAINER_NAME, @@ -43,11 +68,32 @@ export default async function getDatabase(options: Partial = {}) { pgDb: DEFAULT_PG_DB, defaultExternalPort: DEFAULT_PG_PORT, externalPort: config.test.port, - ...options, - }; + persistVolume: config.test.persistVolume, + }); + if (persistVolume) { + rawOptions.image = rawOptions.image.replace(/\-ram$/, ''); + } + if (options.persistVolume) { + console.info(`Using ${options.persistVolume} to store sql data`); + console.info( + `Run "docker volume rm ${options.persistVolume}" to clear data`, + ); + } const {proc, externalPort, kill} = await startContainer({ ...rawOptions, + mount: [ + ...(options.mount || []), + ...(options.persistVolume + ? [ + { + type: 'volume', + src: options.persistVolume, + dst: '/var/lib/postgresql/data', + } as const, + ] + : []), + ], internalPort: DEFAULT_PG_PORT, environment: {...environment, POSTGRES_USER: pgUser, POSTGRES_DB: pgDb}, }); diff --git a/packages/pg-test/src/jest/globalSetup.ts b/packages/pg-test/src/jest/globalSetup.ts index f0c44bf5..7292e18a 100644 --- a/packages/pg-test/src/jest/globalSetup.ts +++ b/packages/pg-test/src/jest/globalSetup.ts @@ -20,7 +20,7 @@ export default async function setup( const migrationsScript = opts.migrationsScript || (process.env.PG_TEST_MIGRATIONS_SCRIPT - ? process.env.PG_TEST_MIGRATIONS_SCRIPT.split(' ') + ? process.env.PG_TEST_MIGRATIONS_SCRIPT : config.test.migrationsScript); if (process.env[envVar]) { console.info(`Using existing pg database from: ${envVar}`); diff --git a/packages/pg-test/src/numberToValidPort.ts b/packages/pg-test/src/numberToValidPort.ts new file mode 100644 index 00000000..bde8cdaa --- /dev/null +++ b/packages/pg-test/src/numberToValidPort.ts @@ -0,0 +1,9 @@ +export default function numberToValidPort( + value: number, + minPort: number, + maxPort: number, +) { + const range = maxPort + 1 - minPort; + const index = value % range; + return minPort + index; +} diff --git a/packages/pg-test/src/stringToValidPort.ts b/packages/pg-test/src/stringToValidPort.ts new file mode 100644 index 00000000..9be366bc --- /dev/null +++ b/packages/pg-test/src/stringToValidPort.ts @@ -0,0 +1,10 @@ +import {hash} from 'generate-alphabetic-name'; +import numberToValidPort from './numberToValidPort'; + +export default function stringToValidPort( + value: string, + minPort: number, + maxPort: number, +) { + return numberToValidPort(hash(value), minPort, maxPort); +} diff --git a/packages/with-container/src/index.ts b/packages/with-container/src/index.ts index 7ae9faf9..1a0c3ccf 100644 --- a/packages/with-container/src/index.ts +++ b/packages/with-container/src/index.ts @@ -23,6 +23,12 @@ export interface Options { */ refreshImage?: boolean; detached?: boolean; + mount?: ReadonlyArray<{ + type: 'bind' | 'volume' | 'tmpfs'; + src: string; + dst: string; + readonly?: boolean; + }>; } export interface NormalizedOptions @@ -82,6 +88,15 @@ export function startDockerContainer(options: NormalizedOptions) { envArgs.push('--env'); envArgs.push(`${key}=${env[key]}`); }); + const mounts: string[] = []; + (options.mount || []).forEach(mount => { + mounts.push('--mount'); + mounts.push( + `type=${mount.type},source=${mount.src},target=${mount.dst}${ + mount.readonly ? `,readonly` : `` + }`, + ); + }); return spawn( 'docker', [ @@ -93,6 +108,7 @@ export function startDockerContainer(options: NormalizedOptions) { '-p', // forward appropriate port `${options.externalPort}:${options.internalPort}`, ...(options.detached ? ['--detach'] : []), + ...mounts, // set enviornment variables ...envArgs, options.image, @@ -118,16 +134,32 @@ export async function waitForDatabaseToStart(options: NormalizedOptions) { console.info( `Waiting for ${options.containerName} on port ${options.externalPort}...`, ); + let currentTestFinished = false; const connection = connect(options.externalPort) .on('error', () => { - if (finished) return; + if (finished || currentTestFinished) return; + currentTestFinished = true; + setTimeout(test, 500); + }) + .on('end', () => { + if (finished || currentTestFinished) return; + currentTestFinished = true; + setTimeout(test, 500); + }) + .on('close', () => { + if (finished || currentTestFinished) return; + currentTestFinished = true; setTimeout(test, 500); }) .on('connect', () => { - finished = true; - clearTimeout(timeout); - connection.end(); - setTimeout(resolve, 1000); + setTimeout(() => { + if (finished || currentTestFinished) return; + finished = true; + currentTestFinished = true; + clearTimeout(timeout); + connection.end(); + resolve(); + }, 2000); }); } test(); diff --git a/yarn.lock b/yarn.lock index 55424fe4..f7c4a94d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2505,6 +2505,11 @@ gauge@~2.7.3: strip-ansi "^3.0.1" wide-align "^1.1.0" +generate-alphabetic-name@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/generate-alphabetic-name/-/generate-alphabetic-name-1.0.0.tgz#20390ac8dc8325caf9edb4b5643beb5385210e5a" + integrity sha512-/0o6Y1wZ2NgvF9Ev8mYEqi2TGQjyQM0Z9BuyQ8ulZ8ObkDV9inRDrLFTH9nFmMIRswDTavyWz60L4tRndgFyFA== + generate-function@^2.3.1: version "2.3.1" resolved "https://registry.yarnpkg.com/generate-function/-/generate-function-2.3.1.tgz#f069617690c10c868e73b8465746764f97c3479f"