feat: allow persistent data in postgres test

packages/pg-config/src/PgConfig.ts

@@ -18,6 +18,14 @@ export interface TestConfig {
    * @default "postgres:10.6-alpine"
    */
   image: string;
+  /**
+   * Set this to a volume name to automatically
+   * persist data to a docker volume of that name.
+   *
+   * N.B. this will automatically remove "-ram"
+   * from the end of any image name.
+   */
+  persistVolume?: string;
   /**
    * The default name to give the docker
    * containers run by @database/pg-test

packages/pg-config/src/PgConfig.validator.ts

@@ -61,6 +61,11 @@ export const PgConfigSchema = {
           description:
             'Optional script to run after the database\nhas been started but before running tests',
         },
+        persistVolume: {
+          description:
+            'Set this to a volume name to automatically\npersist data to a docker volume of that name.\n\nN.B. this will automatically remove "-ram"\nfrom the end of any image name.',
+          type: 'string',
+        },
         pgDb: {
           default: 'test-db',
           description: 'The db to create in the test docker container',

packages/pg-test/package.json

@@ -4,9 +4,13 @@
   "description": "",
   "main": "./lib/index.js",
   "types": "./lib/index.d.ts",
+  "bin": {
+    "pg-test": "./lib/cli.js"
+  },
   "dependencies": {
     "@databases/pg-config": "^0.0.0",
     "@databases/with-container": "^0.0.0",
+    "generate-alphabetic-name": "^1.0.0",
     "modern-spawn": "^1.0.0"
   },
   "scripts": {},
@@ -16,7 +20,8 @@
     "access": "public"
   },
   "devDependencies": {
-    "@types/node": "^13.13.4"
+    "@types/node": "^13.13.4",
+    "@databases/pg": "^0.0.0"
   },
   "bugs": "https://github.com/ForbesLindesay/atdatabases/issues",
   "homepage": "https://www.atdatabases.org/docs/pg-test",

packages/pg-test/src/__tests__/.gitignore

@@ -0,0 +1 @@
+env-example-*

packages/pg-test/src/__tests__/__snapshots__/integration.test.ts.snap

@@ -0,0 +1,5 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`integration: DATABASE_URL for atdatabases-test-volume-1 1`] = `"postgres://test-user@localhost:47936/test-db"`;
+
+exports[`integration: DATABASE_URL for atdatabases-test-volume-2 1`] = `"postgres://test-user@localhost:9833/test-db"`;

packages/pg-test/src/__tests__/integration.test.ts

@@ -0,0 +1,65 @@
+import connect, {sql} from '@databases/pg';
+import {readFileSync} from 'fs';
+import call from '../cli-function';
+import {spawnBuffered} from 'modern-spawn';
+
+jest.setTimeout(60_000);
+if (!process.env.CI) {
+  const cleanup = async () => {
+    for (const volume of [
+      'atdatabases-test-volume-1',
+      'atdatabases-test-volume-2',
+    ]) {
+      // these may fail without harm, so we don't check the exit code
+      await spawnBuffered('docker', ['kill', volume], {
+        debug: false,
+      });
+      await spawnBuffered('docker', ['volume', 'rm', volume], {
+        debug: false,
+      });
+    }
+  };
+  afterAll(cleanup);
+  beforeAll(cleanup);
+  test('integration', async () => {
+    for (const volume of [
+      'atdatabases-test-volume-1',
+      'atdatabases-test-volume-2',
+    ]) {
+      await spawnBuffered('docker', ['volume', 'create', volume], {
+        debug: true,
+      }).getResult();
+      await call([
+        'start',
+        '--persistVolume',
+        volume,
+        '--writeEnv',
+        `${__dirname}/env-example-1`,
+      ]);
+
+      const dbURL = readFileSync(`${__dirname}/env-example-1`, 'utf8')
+        .trim()
+        .replace(/DATABASE_URL=/, '');
+      expect(dbURL).toMatchSnapshot(`DATABASE_URL for ${volume}`);
+      const db = connect(dbURL);
+      await db.query(sql`CREATE TABLE entries (id INT NOT NULL PRIMARY KEY)`);
+      await db.query(sql`INSERT INTO entries (id) VALUES (1), (2)`);
+      await db.dispose();
+
+      await call(['stop', '--persistVolume', volume]);
+
+      await call(['start', '--persistVolume', volume]);
+
+      const db2 = connect(dbURL);
+      expect(await db2.query(sql`SELECT * FROM entries`)).toEqual([
+        {id: 1},
+        {id: 2},
+      ]);
+      await db2.dispose();
+
+      await call(['stop', '--persistVolume', volume]);
+    }
+  });
+} else {
+  test.skip('Cannot run in CI because we need to control docker images');
+}

packages/pg-test/src/__tests__/numberToValidPort.test.ts

@@ -0,0 +1,10 @@
+import numberToValidPort from '../numberToValidPort';
+
+test('numberToValidPort', () => {
+  expect(numberToValidPort(0, 5, 7)).toBe(5);
+  expect(numberToValidPort(1, 5, 7)).toBe(6);
+  expect(numberToValidPort(2, 5, 7)).toBe(7);
+  expect(numberToValidPort(3, 5, 7)).toBe(5);
+  expect(numberToValidPort(4, 5, 7)).toBe(6);
+  expect(numberToValidPort(5, 5, 7)).toBe(7);
+});

packages/pg-test/src/__tests__/stringToValidPort.test.ts

@@ -0,0 +1,10 @@
+import stringToValidPort from '../stringToValidPort';
+
+test('numberToValidPort', () => {
+  expect(stringToValidPort('volume-a', 1000, 2000)).toMatchInlineSnapshot(
+    `1980`,
+  );
+  expect(stringToValidPort('volume-b', 1000, 2000)).toMatchInlineSnapshot(
+    `1224`,
+  );
+});

packages/pg-test/src/cli-function.ts

@@ -0,0 +1,96 @@
+#!/usr/bin/env node
+
+import getDatabase, {stopDatabase} from '.';
+import {readFileSync, writeFileSync} from 'fs';
+import stringToValidPort from './stringToValidPort';
+
+export default async function call(args: string[]): Promise<number> {
+  if (args[0] === 'help' || args.includes('--help') || args.includes('-h')) {
+    usage();
+    return 0;
+  }
+
+  const PORT_RANGE = [1024, 49151] as const;
+  const debug = args.includes('--debug') ? true : undefined;
+  function get(key: string) {
+    return args.includes(key) ? args[args.indexOf(key) + 1] : undefined;
+  }
+
+  const persistVolume = get('--persistVolume');
+  const containerName =
+    get('--containerName') ||
+    (persistVolume ? `pg-test-${persistVolume}` : undefined);
+
+  const portStr = get('port');
+  if (
+    portStr &&
+    (!/^\d+$/.test(portStr) ||
+      portStr.length > PORT_RANGE[1].toString().length ||
+      parseInt(portStr, 10) < PORT_RANGE[0] ||
+      parseInt(portStr, 10) > PORT_RANGE[1])
+  ) {
+    console.error(
+      `The port must be a valid integer between ${PORT_RANGE[0]} and ${PORT_RANGE[1]} (inclusive)`,
+    );
+    return 1;
+  }
+
+  const defaultExternalPort = portStr
+    ? parseInt(portStr, 10)
+    : persistVolume
+    ? stringToValidPort(persistVolume, PORT_RANGE[0], PORT_RANGE[1])
+    : undefined;
+
+  const command = args[0];
+  switch (command) {
+    case 'start':
+      const result = await getDatabase({
+        detached: true,
+        debug,
+        containerName,
+        pgUser: get('--pgUser'),
+        pgDb: get('--pgDb'),
+        persistVolume,
+        externalPort: portStr ? defaultExternalPort : undefined,
+        defaultExternalPort,
+        refreshImage: args.includes('--refreshImage') ? true : undefined,
+      });
+      const envLine = 'DATABASE_URL=' + result.databaseURL;
+      const writeEnv = get('--writeEnv');
+      if (args.includes('--writeEnv')) {
+        const envFile = writeEnv && writeEnv[0] !== '-' ? writeEnv : '.env';
+        console.info('Writing to .env');
+        console.info('');
+        console.info(envLine);
+        try {
+          let dotenv = readFileSync(envFile, 'utf8');
+
+          if (/^DATABASE_URL *=.*$/gm.test(dotenv)) {
+            dotenv = dotenv.replace(/^DATABASE_URL *=.*$/gm, envLine);
+          } else {
+            dotenv += '\n' + envLine + '\n';
+          }
+          writeFileSync(envFile, dotenv);
+        } catch (ex) {
+          if (ex.code !== 'ENOENT') {
+            throw ex;
+          }
+          writeFileSync(envFile, envLine + '\n');
+        }
+      } else {
+        console.info(envLine);
+      }
+      return 0;
+    case 'stop':
+      await stopDatabase({debug, containerName});
+      return 0;
+    default:
+      usage();
+      return 1;
+  }
+
+  function usage() {
+    console.info('Usage: pg-test start');
+    console.info('Usage: pg-test stop');
+  }
+}

packages/pg-test/src/cli.ts

@@ -0,0 +1,10 @@
+#!/usr/bin/env node
+
+import call from './cli-function';
+
+call(process.argv.slice(2))
+  .then(code => process.exit(code))
+  .catch(ex => {
+    console.error(ex);
+    process.exit(1);
+  });

packages/pg-test/src/index.ts

@@ -1,5 +1,6 @@
 import startContainer, {
   Options as WithContainerOptions,
+  killOldContainers,
 } from '@databases/with-container';
 import {getPgConfigSync} from '@databases/pg-config';
 
@@ -31,10 +32,34 @@ export interface Options
   > {
   pgUser: string;
   pgDb: string;
+  persistVolume?: string;
 }
 
+export async function stopDatabase(
+  options: Partial<Pick<Options, 'debug' | 'containerName'>> = {},
+) {
+  await killOldContainers({
+    debug: options.debug !== undefined ? options.debug : DEFAULT_PG_DEBUG,
+    containerName: options.containerName || DEFAULT_CONTAINER_NAME,
+  });
+}
+function withDefaults<T>(overrides: Partial<T>, defaults: T): T {
+  const result = {...defaults};
+  Object.keys(overrides).forEach(key => {
+    if ((overrides as any)[key] !== undefined) {
+      (result as any)[key] = (overrides as any)[key];
+    }
+  });
+  return result;
+}
 export default async function getDatabase(options: Partial<Options> = {}) {
-  const {pgUser, pgDb, environment, ...rawOptions}: Options = {
+  const {
+    pgUser,
+    pgDb,
+    environment,
+    persistVolume,
+    ...rawOptions
+  }: Options = withDefaults(options, {
     debug: DEFAULT_PG_DEBUG,
     image: DEFAULT_IMAGE,
     containerName: DEFAULT_CONTAINER_NAME,
@@ -43,11 +68,32 @@ export default async function getDatabase(options: Partial<Options> = {}) {
     pgDb: DEFAULT_PG_DB,
     defaultExternalPort: DEFAULT_PG_PORT,
     externalPort: config.test.port,
-    ...options,
-  };
+    persistVolume: config.test.persistVolume,
+  });
+  if (persistVolume) {
+    rawOptions.image = rawOptions.image.replace(/\-ram$/, '');
+  }
 
+  if (options.persistVolume) {
+    console.info(`Using ${options.persistVolume} to store sql data`);
+    console.info(
+      `Run "docker volume rm ${options.persistVolume}" to clear data`,
+    );
+  }
   const {proc, externalPort, kill} = await startContainer({
     ...rawOptions,
+    mount: [
+      ...(options.mount || []),
+      ...(options.persistVolume
+        ? [
+            {
+              type: 'volume',
+              src: options.persistVolume,
+              dst: '/var/lib/postgresql/data',
+            } as const,
+          ]
+        : []),
+    ],
     internalPort: DEFAULT_PG_PORT,
     environment: {...environment, POSTGRES_USER: pgUser, POSTGRES_DB: pgDb},
   });

packages/pg-test/src/jest/globalSetup.ts

@@ -20,7 +20,7 @@ export default async function setup(
   const migrationsScript =
     opts.migrationsScript ||
     (process.env.PG_TEST_MIGRATIONS_SCRIPT
-      ? process.env.PG_TEST_MIGRATIONS_SCRIPT.split(' ')
+      ? process.env.PG_TEST_MIGRATIONS_SCRIPT
       : config.test.migrationsScript);
   if (process.env[envVar]) {
     console.info(`Using existing pg database from: ${envVar}`);

packages/pg-test/src/numberToValidPort.ts

@@ -0,0 +1,9 @@
+export default function numberToValidPort(
+  value: number,
+  minPort: number,
+  maxPort: number,
+) {
+  const range = maxPort + 1 - minPort;
+  const index = value % range;
+  return minPort + index;
+}

packages/pg-test/src/stringToValidPort.ts

@@ -0,0 +1,10 @@
+import {hash} from 'generate-alphabetic-name';
+import numberToValidPort from './numberToValidPort';
+
+export default function stringToValidPort(
+  value: string,
+  minPort: number,
+  maxPort: number,
+) {
+  return numberToValidPort(hash(value), minPort, maxPort);
+}