Merge pull request #219 from digirati-co-uk/feature/performance

Performance

Dockerfile

@@ -21,7 +21,7 @@ WORKDIR /home/node/app
 ADD ./package.json /home/node/app/package.json
 ADD ./yarn.lock /home/node/app/yarn.lock
 
-RUN LDFLAGS='-static-libgcc -static-libstdc++' yarn install --production --no-interactive --frozen-lockfile
+RUN yarn install --production --no-interactive --frozen-lockfile
 
 FROM node:lts-bullseye-slim
 
@@ -56,4 +56,3 @@ COPY ./migrations /home/node/app/migrations
 COPY ./config.json /home/node/app/config.json
 
 CMD ["node_modules/.bin/pm2-runtime", "start", "./ecosystem.config.js", "--only", "tasks-api-prod"]
-

__tests__/root-statistics.test.ts

@@ -27,6 +27,8 @@ function track(t: any) {
   createdTasks.push(t.body.id);
 }
 
+jest.setTimeout(30000);
+
 describe('Root statistics', function () {
   test('some statistics', async () => {
     const root = await global.asAdmin.post('/tasks', {
@@ -102,4 +104,33 @@ describe('Root statistics', function () {
       }
     `);
   });
+
+  test('lots of statistics', async () => {
+    const root = await global.asAdmin.post('/tasks', {
+      ...baseTask,
+      subject: 'urn:madoc:manifest:1',
+    });
+    track(root);
+    const rootId: string = (root.body as any).id;
+
+    // 100,000x status=1
+    for (let i = 0; i < 1000; i++) {
+      await global.asAdmin.post('/tasks', {
+        ...baseTask,
+        subject: 'urn:madoc:manifest:1',
+        root_task: rootId,
+        parent_task: rootId,
+        status: 1,
+      });
+    }
+
+    const newList = await global.asAdmin.get(`/tasks/${rootId}?root_statistics=true`);
+    expect((newList.body as any).root_statistics).toEqual({
+      error: 0,
+      not_started: 0,
+      accepted: 1000,
+      progress: 0,
+      done: 0,
+    });
+  });
 });

docker-compose.yaml

@@ -0,0 +1,55 @@
+services:
+
+  gateway-redis:
+    image: redis:5-alpine
+
+  shared-postgres:
+    image: ghcr.io/digirati-co-uk/madoc-postgres:main
+    platform: linux/amd64
+    environment:
+      - POSTGRES_DB=postgres
+      - POSTGRES_USER=postgres
+      - POSTGRES_PASSWORD=postgres_password
+      - POSTGRES_MADOC_TS_USER=madoc_ts
+      - POSTGRES_MADOC_TS_SCHEMA=madoc_ts
+      - POSTGRES_MADOC_TS_PASSWORD=madoc_ts_password
+      - POSTGRES_TASKS_API_USER=tasks_api
+      - POSTGRES_TASKS_API_SCHEMA=tasks_api
+      - POSTGRES_TASKS_API_PASSWORD=tasks_api_password
+      - POSTGRES_MODELS_API_USER=models_api
+      - POSTGRES_MODELS_API_SCHEMA=models_api
+      - POSTGRES_MODELS_API_PASSWORD=models_api_password
+      - POSTGRES_CONFIG_SERVICE_USER=config_service
+      - POSTGRES_CONFIG_SERVICE_SCHEMA=config_service
+      - POSTGRES_CONFIG_SERVICE_PASSWORD=config_service_password
+      - POSTGRES_SEARCH_API_USER=search_api
+      - POSTGRES_SEARCH_API_SCHEMA=search_api
+      - POSTGRES_SEARCH_API_PASSWORD=search_api_password
+    volumes:
+      - shared_postgres_data:/var/lib/postgresql/data:Z
+    ports:
+      - "${PORTS_SHARED_POSTGRES:-5401}:5432"
+
+  tasks-api:
+    build:
+      dockerfile: Dockerfile
+    restart: on-failure
+    environment:
+      - SERVER_PORT=3000
+      - DATABASE_HOST=shared-postgres
+      - DATABASE_NAME=postgres
+      - DATABASE_PORT=5432
+      - DATABASE_USER=tasks_api
+      - DATABASE_SCHEMA=tasks_api
+      - DATABASE_PASSWORD=tasks_api_password
+      - QUEUE_LIST=tasks-api,madoc-ts
+      - REDIS_HOST=gateway-redis
+    links:
+      - shared-postgres
+      - gateway-redis
+    ports:
+      - "3000:3000"
+
+volumes:
+  # Databases
+  shared_postgres_data: {}

migrations/2024-11-20T13-10.type-index.sql

@@ -0,0 +1,13 @@
+--type-index (up)
+
+-- Create index on type column
+create index tasks_type_index
+    on tasks (type);
+
+-- Create index on status column
+create index tasks_status_index
+    on tasks (status);
+
+-- Crete GIN index on context column
+create index tasks_context_gin_index
+    on tasks using GIN (context jsonb_ops);

migrations/down/2024-11-20T13-10.type-index.sql

@@ -0,0 +1,4 @@
+--type-index (down)
+drop index if exists tasks_type_index;
+drop index if exists tasks_status_index;
+drop index if exists tasks_context_gin_index;

src/database/get-task.ts

@@ -50,7 +50,7 @@ export async function getTask(
       select t.*
       from tasks t
       left join tasks dt on t.delegated_task = dt.id
-      where t.context ?& ${sql.array(context, 'text')}
+      where t.context ?& ${sql.array(context, 'text')}::text[]
         ${userCheck}
         and (t.id = ${id} or (t.parent_task = ${id} ${statusQuery} ${subjectsQuery})) order by t.created_at
     `;
@@ -65,7 +65,7 @@ export async function getTask(
         from task_list
         where task_list.id = ${id}
         union
-        ( 
+        (
             select *
             from task_list
             where parent_task = ${id}
@@ -77,15 +77,15 @@ export async function getTask(
   // Root statistics
   const rootStats = rootStatistics
     ? await connection.one<Exclude<FullSingleTask['root_statistics'], undefined>>(sql`
-    select 
+    select
       sum((status = -1)::int) as error,
       sum((status = 0)::int) as not_started,
       sum((status = 1)::int) as accepted,
       sum((status = 2 or status > 3)::int) as progress,
       sum((status = 3)::int) as done
     from tasks t
-    where t.context ?& ${sql.array(context, 'text')}
-    and t.root_task = ${id} 
+    where t.context ?& ${sql.array(context, 'text')}::text[]
+    and t.root_task = ${id}
   `)
     : undefined;
 

src/middleware/db-connection.ts

@@ -4,8 +4,6 @@ import { DatabasePoolType } from 'slonik';
 export const dbConnection =
   (pool: DatabasePoolType): Middleware =>
   async (context, next) => {
-    await pool.connect(async (connection) => {
-      context.connection = connection;
-      await next();
-    });
+    context.connection = pool;
+    await next();
   };

src/routes/event.ts

@@ -20,10 +20,10 @@ export const postEvent: RouteMiddleware<{ id: string; event: string }, { subject
     events?: string[];
     type: string;
   }>`
-      SELECT t.assignee_id, t.creator_id, t.events, t.type 
-      FROM tasks t 
-      WHERE id = ${id} 
-        AND context ?& ${sql.array(context.state.jwt.context, 'text')}
+      SELECT t.assignee_id, t.creator_id, t.events, t.type
+      FROM tasks t
+      WHERE id = ${id}
+        AND context ?& ${sql.array(context.state.jwt.context, 'text')}::text[]
     `);
 
   const taskWithId = { id, type, events };

src/routes/export-tasks.ts

@@ -10,7 +10,7 @@ export const exportTasks: RouteMiddleware = async (context) => {
   }
 
   const tasks = await context.connection.any(sql`
-    select * from tasks where context ?& ${sql.array(context.state.jwt.context, 'text')}
+    select * from tasks where context ?& ${sql.array(context.state.jwt.context, 'text')}::text[]
   `);
 
   context.response.body = { tasks };

src/routes/get-all-tasks.ts

@@ -138,9 +138,9 @@ export const getAllTasks: RouteMiddleware = async (context) => {
 
   try {
     const countQuery = sql<{ total_items: number }>`
-        select COUNT(*) as total_items from tasks t 
+        select COUNT(*) as total_items from tasks t
         ${dtJoin}
-        where t.context ?& ${sql.array(context.state.jwt.context, 'text')}
+        where t.context ?& ${sql.array(context.state.jwt.context, 'text')}::text[]
         ${subtaskExclusion}
         ${userExclusion}
         ${typeFilter}
@@ -160,7 +160,7 @@ export const getAllTasks: RouteMiddleware = async (context) => {
       SELECT t.id, t.name, t.status, t.status_text, t.metadata, t.type ${detailedFields}
       FROM tasks t
       ${dtJoin}
-      WHERE t.context ?& ${sql.array(context.state.jwt.context, 'text')}
+      WHERE t.context ?& ${sql.array(context.state.jwt.context, 'text')}::text[]
         ${subtaskExclusion}
         ${userExclusion}
         ${typeFilter}

src/routes/get-statistics.ts

@@ -11,9 +11,12 @@ export const getStatistics: RouteMiddleware<{ id?: string }> = async (context) =
   const whereUser = context.query.user_id
     ? sql`(creator_id = ${context.query.user_id} or assignee_id = ${context.query.user_id})`
     : undefined;
-  const whereContext = sql`context ?& ${sql.array(context.state.jwt.context, 'text')}`;
+  const whereContext = sql`context ?& ${sql.array(context.state.jwt.context, 'text')}::text[]`;
   const counter = context.query.distinct_subjects ? sql`count(distinct subject)` : sql`count(*)`;
-  const fullWhere = sql.join([whereRoot, whereType, whereUser, whereContext, whereStatus].filter(Boolean) as any[], sql` and `);
+  const fullWhere = sql.join(
+    [whereRoot, whereType, whereUser, whereContext, whereStatus].filter(Boolean) as any[],
+    sql` and `
+  );
   const isAdmin = context.state.jwt.scope.indexOf('tasks.admin') !== -1;
   const canCreate = context.state.jwt.scope.indexOf('tasks.create') !== -1;
   const groupBy = context.query.group_by;
@@ -51,7 +54,6 @@ export const getStatistics: RouteMiddleware<{ id?: string }> = async (context) =
   );
 
   if (returnField === 'status') {
-
     let total = 0;
     const statuses = query.reduce((state, next) => {
       state[next.status] = next.total;

src/routes/get-subject-statistics.ts

@@ -26,15 +26,15 @@ export const getSubjectStatistics: RouteMiddleware<{ id: string }> = async (ctx)
   const statusQuery = typeof status !== 'undefined' && !Number.isNaN(status) ? sql`and t.status = ${status}` : sql``;
 
   const results = await ctx.connection.any(sql`
-    select t.subject, t.status ${assigneeFields} from tasks t 
+    select t.subject, t.status ${assigneeFields} from tasks t
         ${parentJoin}
-        where  ${parentTask ? sql`t.parent_task = ${taskId}` : sql`t.root_task = ${taskId}`} 
+        where  ${parentTask ? sql`t.parent_task = ${taskId}` : sql`t.root_task = ${taskId}`}
         ${subjectQuery}
         ${typeQuery}
         ${parentQuery}
         ${assignedToQuery}
         ${statusQuery}
-        and t.context ?& ${sql.array(context, 'text')}
+        and t.context ?& ${sql.array(context, 'text')}::text[]
   `);
 
   ctx.response.body = {

src/routes/update-metadata.ts

@@ -18,9 +18,9 @@ export const updateMetadata: RouteMiddleware<{ id: string }> = async (context) =
     metadata: any | null;
   }>(sql`
       SELECT t.id, t.metadata
-      FROM tasks t 
-      WHERE t.id = ${id} 
-        AND t.context ?& ${sql.array(context.state.jwt.context, 'text')}
+      FROM tasks t
+      WHERE t.id = ${id}
+        AND t.context ?& ${sql.array(context.state.jwt.context, 'text')}::text[]
     `);
 
   if (!currentTask) {

src/routes/update-single-task.ts

@@ -23,10 +23,10 @@ export const updateSingleTask: RouteMiddleware<{ id: string }> = async (context)
     type: string;
   }>(sql`
       SELECT t.assignee_id, t.creator_id, t.events, t.type, dt.assignee_id as delegated_assignee
-      FROM tasks t 
+      FROM tasks t
       left join tasks dt on t.delegated_task = dt.id
-      WHERE t.id = ${id} 
-        AND t.context ?& ${sql.array(context.state.jwt.context, 'text')}
+      WHERE t.id = ${id}
+        AND t.context ?& ${sql.array(context.state.jwt.context, 'text')}::text[]
     `);
 
   const taskWithId = { id, type, events };

src/types.ts

@@ -1,7 +1,7 @@
 import { RouterParamContext } from '@koa/router';
 import * as Koa from 'koa';
 import { router } from './router';
-import { DatabasePoolConnectionType } from 'slonik';
+import { DatabasePoolType } from 'slonik';
 import { Ajv } from 'ajv';
 import { JobsOptions, ConnectionOptions, Queue, QueueOptions } from 'bullmq';
 import { EventPrefix } from './utility/events';
@@ -75,7 +75,7 @@ export interface ApplicationState {
 
 export interface ApplicationContext {
   routes: typeof router;
-  connection: DatabasePoolConnectionType;
+  connection: DatabasePoolType;
   getQueue?: (name: string) => Queue;
   ajv: Ajv;
 }