feat: database management

package.json

@@ -6,6 +6,7 @@
   "author": "Nevermined <[email protected]>",
   "license": "Apache-2.0",
   "scripts": {
+    "clean": "nest clean",
     "prebuild": "rimraf dist",
     "build": "nest build",
     "format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\"",
@@ -18,20 +19,28 @@
     "test:watch": "jest --watch",
     "test:cov": "jest --coverage",
     "test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand",
-    "test:e2e": "jest --config ./test/jest-e2e.json"
+    "test:e2e": "jest --config ./test/jest-e2e.json",
+    "database:typeorm": "ts-node ./node_modules/typeorm/cli",
+    "database:run-migrations": "yarn run database:typeorm migration:run -d ./src/database/typeorm.config.ts",
+    "database:create-migration": "yarn run database:typeorm migration:create",
+    "database:generate-migration": "yarn run database:typeorm -d ./src/database/typeorm.config.ts migration:generate",
+    "database:revert-migration": "yarn run database:typeorm -d ./src/database/typeorm.config.ts migration:revert"
   },
   "dependencies": {
     "@nestjs/common": "^10.3.9",
-    "@nestjs/core": "^10.3.9",
     "@nestjs/config": "^3.2.2",
+    "@nestjs/core": "^10.3.9",
     "@nestjs/platform-express": "^10.3.9",
     "@nestjs/swagger": "^7.3.1",
-    "reflect-metadata": "^0.1.13",
-    "rimraf": "^3.0.2",
+    "@nestjs/typeorm": "^10.0.2",
+    "sqlite3": "^5.1.7",
     "class-transformer": "^0.5.1",
     "class-validator": "^0.14.1",
-    "uuid": "^10.0.0",
-    "rxjs": "^7.2.0"
+    "reflect-metadata": "^0.1.13",
+    "rimraf": "^5.0.7",
+    "rxjs": "^7.2.0",
+    "typeorm": "^0.3.20",
+    "uuid": "^10.0.0"
   },
   "devDependencies": {
     "@nestjs/cli": "^10.3.2",
@@ -40,7 +49,7 @@
     "@types/express": "^4.17.21",
     "@types/jest": "^29.5.12",
     "@types/node": "^16.0.0",
-    "@types/supertest": "^2.0.11",
+    "@types/supertest": "^6.0.2",
     "@typescript-eslint/eslint-plugin": "^7.13.1",
     "@typescript-eslint/parser": "^7.13.1",
     "eslint": "^8.56.0",
@@ -49,7 +58,7 @@
     "jest": "^29.7.0",
     "prettier": "^3.3.2",
     "source-map-support": "^0.5.20",
-    "supertest": "^6.1.3",
+    "supertest": "^7.0.0",
     "ts-jest": "^29.1.5",
     "ts-node": "^10.9.2",
     "tsconfig-paths": "^3.10.1",

src/common/guards/auth.guard.ts

@@ -10,7 +10,10 @@ import { ConfigService } from '@nestjs/config'
 
 @Injectable()
 export class AuthorizationGuard implements CanActivate {
-  constructor(private config: ConfigService, private _reflector: Reflector) {}
+  constructor(
+    private config: ConfigService,
+    private _reflector: Reflector
+  ) {}
   async canActivate(context: ExecutionContext): Promise<boolean> {
     const apiAuthToken = this.config.get('api.authToken')
     if (!apiAuthToken) return true

src/common/models/agent-models.ts

@@ -11,6 +11,11 @@ export interface Task extends ExecutionOptions {
    * The steps executed by the agent to complete the task
    */
   steps: Step[]
+
+  /**
+   * The name of the task
+   */
+  name?: string
 }
 
 export interface Step extends ExecutionOptions {

src/common/utils/agents-factory.ts

@@ -11,6 +11,7 @@ export abstract class AgentsFactory {
     const task_id = `task-${uuidv4()}`
     const task: Task = {
       task_id,
+      name: '',
       input,
       status: ExecutionStatus.PENDING,
       steps: [AgentsFactory.getStepFromTemplate(input, task_id)],
@@ -31,6 +32,7 @@ export abstract class AgentsFactory {
       task_id,
       input,
       status: ExecutionStatus.PENDING,
+      is_last: false,
       created_at: new Date(),
       updated_at: new Date()
     }

src/configuration.ts

@@ -1,4 +1,4 @@
-export default () => ({
+export const appConfig = {
   api: {
     host: process.env.API_HOST || 'localhost',
     port: parseInt(process.env.API_PORT, 10) || 4100,
@@ -19,8 +19,13 @@ export default () => ({
     level: process.env.LOG_LEVEL || ['error', 'log', 'warn']
   },
   database: {
-    type: process.env.DATABASE_TYPE,
+    type: process.env.DATABASE_TYPE || 'sqlite', // sqlite, mysql, postgres, etc
+    name: process.env.DATABASE_NAME || '/tmp/agent-template.db', // path to the database file if sqlite if is a different database type put the database name instead
     host: process.env.DATABASE_HOST,
-    port: parseInt(process.env.DATABASE_PORT, 10) || 5432
+    port: parseInt(process.env.DATABASE_PORT, 10),
+    username: process.env.DATABASE_USERNAME,
+    password: process.env.DATABASE_PASSWORD,
+    schena: process.env.DATABASE_SCHEMA
   }
-})
+}
+export default () => appConfig

src/database/1718898482722-AgentBase.ts

@@ -0,0 +1,158 @@
+import {
+  MigrationInterface,
+  QueryRunner,
+  Table,
+  TableForeignKey
+} from 'typeorm'
+import { TABLE_STEPS, TABLE_TASKS, baseSchema } from './schema'
+import { ExecutionStatus } from '../common/models/agent-models'
+
+export class AgentBase1718898482722 implements MigrationInterface {
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.createTable(
+      new Table({
+        name: TABLE_TASKS,
+        columns: [
+          {
+            name: 'task_id',
+            type: 'varchar',
+            isPrimary: true,
+            isUnique: true,
+            generationStrategy: 'uuid',
+            default: 'uuid_generate_v4()'
+          },
+          {
+            name: 'task_status',
+            type: 'enum',
+            default: "'PENDING'",
+            enum: Object.values(ExecutionStatus)
+          },
+          {
+            name: 'name',
+            type: 'varchar',
+            isNullable: true
+          },
+          {
+            name: 'input_query',
+            type: 'text',
+            isNullable: false
+          },
+          {
+            name: 'input_params',
+            type: 'text',
+            isNullable: true
+          },
+          {
+            name: 'input_artifacts',
+            type: 'text',
+            isNullable: true
+          },
+          {
+            name: 'output',
+            type: 'text',
+            isNullable: true
+          },
+          {
+            name: 'output_additional',
+            type: 'text',
+            isNullable: true
+          },
+          {
+            name: 'output_artifacts',
+            type: 'text',
+            isNullable: true
+          },
+          ...baseSchema
+        ]
+      }),
+      true
+    )
+
+    await queryRunner.createTable(
+      new Table({
+        name: TABLE_STEPS,
+        columns: [
+          {
+            name: 'step_id',
+            type: 'varchar',
+            isPrimary: true,
+            isUnique: true,
+            generationStrategy: 'uuid',
+            default: 'uuid_generate_v4()'
+          },
+          {
+            name: 'task_id',
+            type: 'varchar',
+            isNullable: false
+          },
+          {
+            name: 'name',
+            type: 'varchar',
+            isNullable: true
+          },
+          {
+            name: 'is_last',
+            type: 'boolean',
+            default: false,
+            isNullable: false
+          },
+          {
+            name: 'step_status',
+            type: 'enum',
+            default: "'PENDING'",
+            enum: Object.values(ExecutionStatus)
+          },
+          {
+            name: 'input_query',
+            type: 'text',
+            isNullable: false
+          },
+          {
+            name: 'input_params',
+            type: 'text',
+            isNullable: true
+          },
+          {
+            name: 'input_artifacts',
+            type: 'text',
+            isNullable: true
+          },
+          {
+            name: 'output',
+            type: 'text',
+            isNullable: true
+          },
+          {
+            name: 'output_additional',
+            type: 'text',
+            isNullable: true
+          },
+          {
+            name: 'output_artifacts',
+            type: 'text',
+            isNullable: true
+          },
+          ...baseSchema
+        ]
+      }),
+      true
+    )
+
+    const foreignKeyRuleTasks = new TableForeignKey({
+      name: 'tasksStepsByTaskId',
+      columnNames: ['task_id'],
+      referencedColumnNames: ['task_id'],
+      referencedTableName: TABLE_TASKS,
+      onUpdate: 'CASCADE',
+      onDelete: 'RESTRICT'
+    })
+    await queryRunner.createForeignKey(TABLE_STEPS, foreignKeyRuleTasks)
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.dropForeignKey(TABLE_STEPS, 'tasksStepsByTaskId')
+
+    await queryRunner.dropTable(TABLE_STEPS)
+    await queryRunner.dropTable(TABLE_TASKS)
+  }
+}

src/database/entities/base.entity.ts

@@ -0,0 +1,10 @@
+import { Entity, CreateDateColumn, UpdateDateColumn } from 'typeorm'
+
+@Entity()
+export class BaseEntity {
+  @CreateDateColumn()
+  created_at: Date
+
+  @UpdateDateColumn()
+  updated_at: Date
+}

src/database/entities/step.entity.ts

@@ -0,0 +1,50 @@
+import { Column, Entity, PrimaryColumn } from 'typeorm'
+import { BaseEntity } from './base.entity'
+import { ExecutionStatus } from 'src/common/models/agent-models'
+import { v4 as uuidv4 } from 'uuid'
+
+@Entity('steps')
+export class StepEntity extends BaseEntity {
+  @PrimaryColumn({ unique: true })
+  step_id: string
+
+  @Column('varchar')
+  task_id: string
+
+  @Column('varchar')
+  name: string
+
+  @Column('boolean')
+  is_last: boolean
+
+  @Column({
+    // enum: Object.values(ExecutionStatus),
+    enum: ExecutionStatus,
+    type: 'simple-enum'
+  })
+  step_status: ExecutionStatus
+
+  @Column('text')
+  input_query: string
+
+  @Column('text')
+  input_params: string
+
+  @Column('text')
+  input_artifacts: string
+
+  @Column('text')
+  output: string
+
+  @Column('text')
+  output_additional: string
+
+  @Column('text')
+  output_artifacts: string
+
+  constructor() {
+    super()
+    this.step_id = `step-${uuidv4()}`
+    this.step_status = ExecutionStatus.PENDING
+  }
+}

src/database/entities/task.entity.ts

@@ -0,0 +1,44 @@
+import { Column, Entity, PrimaryColumn } from 'typeorm'
+import { BaseEntity } from './base.entity'
+import { ExecutionStatus } from 'src/common/models/agent-models'
+import { v4 as uuidv4 } from 'uuid'
+
+@Entity('tasks')
+export class TaskEntity extends BaseEntity {
+  @PrimaryColumn({ unique: true })
+  task_id: string
+
+  @Column({
+    // enum: Object.values(ExecutionStatus),
+    enum: ExecutionStatus,
+    type: 'simple-enum'
+  })
+  task_status: ExecutionStatus
+
+  @Column('varchar')
+  name: string
+
+  @Column('text')
+  input_query: string
+
+  @Column('text')
+  input_params: string
+
+  @Column('text')
+  input_artifacts: string
+
+  @Column('text')
+  output: string
+
+  @Column('text')
+  output_additional: string
+
+  @Column('text')
+  output_artifacts: string
+
+  constructor() {
+    super()
+    this.task_id = `task-${uuidv4()}`
+    this.task_status = ExecutionStatus.PENDING
+  }
+}