feat: add @slonik/lazy-dataloader

packages/slonik-lazy-dataloader/.eslintrc

@@ -0,0 +1,26 @@
+{
+  "extends": "canonical/auto",
+  "ignorePatterns": [
+    "dist",
+    "package-lock.json"
+  ],
+  "overrides": [
+    {
+      "files": "*.ts",
+      "rules": {
+        "import/no-cycle": 0,
+        "no-restricted-imports": ["error", "pg"]
+      }
+    },
+    {
+      "files": ["*.test.ts", "**/*.test/*"],
+      "extends": "canonical/ava",
+      "rules": {
+        "@typescript-eslint/no-explicit-any": 0,
+        "@typescript-eslint/no-unused-expressions": 0,
+        "ava/no-ignored-test-files": 0
+      }
+    }
+  ],
+  "root": true
+}

packages/slonik-lazy-dataloader/README.md

@@ -0,0 +1,83 @@
+# Lazy DataLoader
+
+Connection pool wrapper with seamless query batching.
+
+## Usage
+
+```ts
+import { createLazyDataLoader } from '@slonik/lazy-dataloader';
+import {
+  createPool,
+  sql,
+} from 'slonik';
+import { z } from 'zod';
+
+const pool = createPool('postgres://');
+
+const lazyDataLoader = createLazyDataLoader(pool);
+
+const results = await Promise.all([
+  lazyDataLoader.oneFirst(
+    sql.type(
+      z.object({
+        id: z.number(),
+        name: z.string(),
+      })
+    )`
+      SELECT id, name
+      FROM person
+      WHERE id = ${1}
+    `
+  ),
+  lazyDataLoader.oneFirst(
+    sql.type(
+      z.object({
+        id: z.number(),
+        name: z.string(),
+        website: z.string().nullable(),
+      })
+    )`
+      SELECT id, name, website
+      FROM company
+      WHERE id = ${2}
+    `
+  ),
+]);
+
+console.log(results);
+```
+
+In this example:
+
+* Both queries will be batched into a single query.
+* `results` will be an array with the results of the two queries.
+
+## How it works
+
+Using the same idea as [DataLoader](https://github.com/graphql/dataloader), `LazyDataLoader` will batch all queries that are executed in the same tick. This is done by using sub-queries for every query. Example:
+
+```sql
+SELECT
+  (
+    SELECT json_agg(row_to_json(t))
+    FROM (
+      SELECT id, name
+      FROM person
+      WHERE id = 1
+    ) t
+  ) query_1,
+  (
+    SELECT json_agg(row_to_json(t))
+    FROM (
+      SELECT id, name, website
+      FROM company
+      WHERE id = 2
+    ) t
+  ) query_2
+```
+
+## Use cases
+
+This is experimental approach to help with the N+1 problem that is common in GraphQL APIs.
+
+The same problem can be solved more efficiently by using a [DataLoader](https://github.com/graphql/dataloader) directly and hand crafting the queries. This approach is more flexible and efficient, but requires more work. This library is a middle ground that can be used in some cases to reduce the impact of the N+1 problem by reducing the number of round trips to the database.

packages/slonik-lazy-dataloader/package.json

@@ -0,0 +1,86 @@
+{
+  "author": {
+    "email": "[email protected]",
+    "name": "Gajus Kuizinas",
+    "url": "http://gajus.com"
+  },
+  "ava": {
+    "extensions": [
+      "ts"
+    ],
+    "files": [
+      "src/**/*.test.ts"
+    ],
+    "require": [
+      "ts-node/register/transpile-only"
+    ]
+  },
+  "dependencies": {
+    "dataloader": "^2.2.2",
+    "pg-query-stream": "^4.5.5",
+    "slonik": "^43.0.7"
+  },
+  "description": "Connection pool wrapper with seamless query batching.",
+  "devDependencies": {
+    "@types/node": "^18.15.3",
+    "@types/pg": "^8.11.6",
+    "ava": "^5.3.1",
+    "cspell": "^8.6.0",
+    "eslint": "^8.57.0",
+    "eslint-config-canonical": "^42.8.1",
+    "nyc": "^15.1.0",
+    "ts-node": "^10.9.1",
+    "typescript": "^5.4.3",
+    "zod": "^3.23.8"
+  },
+  "engines": {
+    "node": ">=18"
+  },
+  "files": [
+    "./src",
+    "./dist"
+  ],
+  "keywords": [
+    "postgresql",
+    "promise",
+    "types"
+  ],
+  "license": "BSD-3-Clause",
+  "main": "./dist/index.js",
+  "name": "@slonik/lazy-dataloader",
+  "nyc": {
+    "all": true,
+    "exclude": [
+      "**/*.d.ts"
+    ],
+    "include": [
+      "src/**/*.ts"
+    ],
+    "reporter": [
+      "html",
+      "text-summary"
+    ],
+    "require": [
+      "ts-node/register/transpile-only"
+    ],
+    "silent": true,
+    "sourceMap": false
+  },
+  "peerDependencies": {
+    "zod": "^3"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/gajus/slonik"
+  },
+  "scripts": {
+    "build": "rm -fr ./dist && tsc --project ./tsconfig.json",
+    "lint": "npm run lint:cspell && npm run lint:eslint && npm run lint:tsc",
+    "lint:cspell": "cspell . --no-progress --gitignore",
+    "lint:eslint": "eslint --cache ./src",
+    "lint:tsc": "tsc --noEmit",
+    "test": "nyc ava --verbose --serial"
+  },
+  "types": "./dist/index.d.ts",
+  "version": "43.0.7"
+}

packages/slonik-lazy-dataloader/src/factories/createLazyDataLoader.test.ts

@@ -0,0 +1,55 @@
+import { createLazyDataLoader } from './createLazyDataLoader';
+import { createTestRunner } from './helpers.test/createTestRunner';
+import { createPool, sql } from 'slonik';
+import { z } from 'zod';
+
+const { test } = createTestRunner();
+
+test('fetches a single query', async (t) => {
+  const pool = await createPool(t.context.dsn);
+
+  const lazy = await createLazyDataLoader(pool);
+
+  const result = await lazy.one(sql.type(
+    z.object({
+      id: z.number(),
+    }),
+  )`
+    SELECT 1 AS id
+  `);
+
+  t.deepEqual(result, {
+    id: 1,
+  });
+});
+
+test('fetches multiple queries', async (t) => {
+  const pool = await createPool(t.context.dsn);
+
+  const lazy = await createLazyDataLoader(pool);
+
+  const [a, b] = await Promise.all([
+    lazy.one(sql.type(
+      z.object({
+        id: z.number(),
+      }),
+    )`
+      SELECT 1 AS id
+    `),
+    lazy.one(sql.type(
+      z.object({
+        id: z.number(),
+      }),
+    )`
+      SELECT 2 AS id
+    `),
+  ]);
+
+  t.deepEqual(a, {
+    id: 1,
+  });
+
+  t.deepEqual(b, {
+    id: 2,
+  });
+});

packages/slonik-lazy-dataloader/src/factories/createLazyDataLoader.ts

@@ -0,0 +1,62 @@
+import DataLoader from 'dataloader';
+import {
+  type CommonQueryMethods,
+  type DatabasePool,
+  DataIntegrityError,
+  NotFoundError,
+  type QuerySqlToken,
+  sql,
+} from 'slonik';
+import { type ZodTypeAny } from 'zod';
+
+// TODO add other methods
+// TODO add middlewares
+
+type LazyDataLoader = {
+  one: CommonQueryMethods['one'];
+};
+
+export const createLazyDataLoader = (pool: DatabasePool): LazyDataLoader => {
+  const dataLoader = new DataLoader(
+    async (queries: ReadonlyArray<QuerySqlToken<ZodTypeAny>>) => {
+      // console.log('queries', queries);
+
+      const result = await pool.one(sql.unsafe`
+        SELECT
+          ${sql.join(
+            queries.map(
+              (query, index) => sql.fragment`
+              (
+                SELECT json_agg(row_to_json(t))
+                FROM (
+                  ${query}
+                ) t
+              ) AS ${sql.identifier(['query_' + String(index + 1)])}
+            `,
+            ),
+            sql.fragment`, `,
+          )}
+      `);
+
+      return queries.map((query, index) => {
+        return result['query_' + String(index + 1)];
+      });
+    },
+  );
+
+  return {
+    one: async (slonikQuery) => {
+      const rows = await dataLoader.load(slonikQuery);
+
+      if (rows.length === 0) {
+        throw new NotFoundError(slonikQuery);
+      }
+
+      if (rows.length > 1) {
+        throw new DataIntegrityError(slonikQuery);
+      }
+
+      return rows[0];
+    },
+  };
+};

packages/slonik-lazy-dataloader/src/factories/helpers.test/createTestRunner.ts

@@ -0,0 +1,64 @@
+import anyTest, { type TestFn } from 'ava';
+import { createPool, sql } from 'slonik';
+
+// TODO deduplicate with slonik/src/factories/createTestRunner.ts
+
+// eslint-disable-next-line n/no-process-env
+const POSTGRES_DSN = process.env.POSTGRES_DSN ?? 'postgres@localhost:5432';
+
+export type TestContextType = {
+  dsn: string;
+  testDatabaseName: string;
+};
+
+export const createTestRunner = () => {
+  let testId = 0;
+
+  const test = anyTest as TestFn<TestContextType>;
+  const { beforeEach } = test;
+
+  const TEMPLATE_DATABASE_NAME = 'slonik_test';
+
+  beforeEach(async (t) => {
+    ++testId;
+
+    const TEST_DATABASE_NAME = [
+      'slonik_test',
+      'lazy-dataloader',
+      String(testId),
+    ].join('_');
+
+    t.context = {
+      dsn: 'postgresql://' + POSTGRES_DSN + '/' + TEST_DATABASE_NAME,
+      testDatabaseName: TEST_DATABASE_NAME,
+    };
+
+    const pool0 = await createPool('postgresql://' + POSTGRES_DSN, {
+      maximumPoolSize: 1,
+    });
+
+    await pool0.connect(async (connection) => {
+      await connection.query(sql.unsafe`
+        SELECT pg_terminate_backend(pid)
+        FROM pg_stat_activity
+        WHERE
+          pid != pg_backend_pid() AND
+          datname = ${TEMPLATE_DATABASE_NAME}
+      `);
+      await connection.query(
+        sql.unsafe`DROP DATABASE IF EXISTS ${sql.identifier([
+          TEST_DATABASE_NAME,
+        ])}`,
+      );
+      await connection.query(
+        sql.unsafe`CREATE DATABASE ${sql.identifier([TEST_DATABASE_NAME])}`,
+      );
+    });
+
+    await pool0.end();
+  });
+
+  return {
+    test,
+  };
+};

packages/slonik-lazy-dataloader/src/index.ts

@@ -0,0 +1 @@
+export { createLazyDataLoader } from './factories/createLazyDataLoader';