diff --git a/api/.env.template b/api/.env.template index bd45022d24..ff433dfd8c 100644 --- a/api/.env.template +++ b/api/.env.template @@ -52,3 +52,5 @@ THROTTLE_TTL=3600000 THROTTLE_LIMIT=100 # API passkey, requests missing this will not be alllowed to progress API_PASS_KEY="some-key-here" +# this is used to test the script runner's data transfer job +TEST_CONNECTION_STRING="" diff --git a/api/src/controllers/auth.controller.ts b/api/src/controllers/auth.controller.ts index 3647713b3a..73fdd5cfac 100644 --- a/api/src/controllers/auth.controller.ts +++ b/api/src/controllers/auth.controller.ts @@ -31,12 +31,10 @@ import { mapTo } from '../utilities/mapTo'; import { User } from '../dtos/users/user.dto'; import { LoginViaSingleUseCode } from '../dtos/auth/login-single-use-code.dto'; import { SingleUseCodeAuthGuard } from '../guards/single-use-code.guard'; -import { ApiKeyGuard } from '../guards/api-key.guard'; @Controller('auth') @ApiTags('auth') @UsePipes(new ValidationPipe(defaultValidationPipeOptions)) -@UseGuards(ApiKeyGuard) export class AuthController { constructor(private readonly authService: AuthService) {} diff --git a/api/src/controllers/script-runner.controller.ts b/api/src/controllers/script-runner.controller.ts index 78c4e0d5f3..808c31b688 100644 --- a/api/src/controllers/script-runner.controller.ts +++ b/api/src/controllers/script-runner.controller.ts @@ -1,4 +1,5 @@ import { + Body, Controller, Put, Request, @@ -13,6 +14,7 @@ import { defaultValidationPipeOptions } from '../utilities/default-validation-pi import { SuccessDTO } from '../dtos/shared/success.dto'; import { OptionalAuthGuard } from '../guards/optional.guard'; import { AdminOrJurisdictionalAdminGuard } from '../guards/admin-or-jurisdiction-admin.guard'; +import { DataTransferDTO } from '../dtos/script-runner/data-transfer.dto'; @Controller('scriptRunner') @ApiTags('scriptRunner') @@ -30,4 +32,17 @@ export class ScirptRunnerController { async update(@Request() req: ExpressRequest): Promise { return await this.scriptRunnerService.example(req); } + + @Put('dataTransfer') + @ApiOperation({ + summary: 'A script that pulls data from one source into the current db', + operationId: 'dataTransfer', + }) + @ApiOkResponse({ type: SuccessDTO }) + async dataTransfer( + @Body() dataTransferDTO: DataTransferDTO, + @Request() req: ExpressRequest, + ): Promise { + return await this.scriptRunnerService.dataTransfer(req, dataTransferDTO); + } } diff --git a/api/src/dtos/script-runner/data-transfer.dto.ts b/api/src/dtos/script-runner/data-transfer.dto.ts new file mode 100644 index 0000000000..9a4b54891e --- /dev/null +++ b/api/src/dtos/script-runner/data-transfer.dto.ts @@ -0,0 +1,12 @@ +import { ApiProperty } from '@nestjs/swagger'; +import { Expose } from 'class-transformer'; +import { IsDefined, IsString } from 'class-validator'; +import { ValidationsGroupsEnum } from '../../enums/shared/validation-groups-enum'; + +export class DataTransferDTO { + @Expose() + @IsString({ groups: [ValidationsGroupsEnum.default] }) + @IsDefined({ groups: [ValidationsGroupsEnum.default] }) + @ApiProperty() + connectionString: string; +} diff --git a/api/src/services/script-runner.service.ts b/api/src/services/script-runner.service.ts index f4ca94c4b8..0d34256194 100644 --- a/api/src/services/script-runner.service.ts +++ b/api/src/services/script-runner.service.ts @@ -1,9 +1,11 @@ import { Injectable, BadRequestException } from '@nestjs/common'; +import { PrismaClient } from '@prisma/client'; import { Request as ExpressRequest } from 'express'; import { PrismaService } from './prisma.service'; import { SuccessDTO } from '../dtos/shared/success.dto'; import { User } from '../dtos/users/user.dto'; import { mapTo } from '../utilities/mapTo'; +import { DataTransferDTO } from '../dtos/script-runner/data-transfer.dto'; /** this is the service for running scripts @@ -13,6 +15,44 @@ import { mapTo } from '../utilities/mapTo'; export class ScriptRunnerService { constructor(private prisma: PrismaService) {} + /** + * + * @param req incoming request object + * @param dataTransferDTO data transfer endpoint args. Should contain foreign db connection string + * @returns successDTO + * @description transfers data from foreign data into the database this api normally connects to + */ + async dataTransfer( + req: ExpressRequest, + dataTransferDTO: DataTransferDTO, + ): Promise { + // script runner standard start up + const requestingUser = mapTo(User, req['user']); + await this.markScriptAsRunStart('data transfer', requestingUser); + + // connect to foreign db based on incoming connection string + const client = new PrismaClient({ + datasources: { + db: { + url: dataTransferDTO.connectionString, + }, + }, + }); + await client.$connect(); + + // get data + const res = + await client.$queryRaw`SELECT id, name FROM jurisdictions WHERE name = 'San Mateo'`; + console.log(res); + + // disconnect from foreign db + await client.$disconnect(); + + // script runner standard spin down + await this.markScriptAsComplete('data transfer', requestingUser); + return { success: true }; + } + /** this is simply an example */ diff --git a/api/test/unit/services/script-runner.service.spec.ts b/api/test/unit/services/script-runner.service.spec.ts index 8d0b08f316..76bce2db7d 100644 --- a/api/test/unit/services/script-runner.service.spec.ts +++ b/api/test/unit/services/script-runner.service.spec.ts @@ -1,7 +1,8 @@ import { Test, TestingModule } from '@nestjs/testing'; import { Logger } from '@nestjs/common'; -import { randomUUID } from 'crypto'; import { SchedulerRegistry } from '@nestjs/schedule'; +import { randomUUID } from 'crypto'; +import { Request as ExpressRequest } from 'express'; import { ScriptRunnerService } from '../../../src/services/script-runner.service'; import { PrismaService } from '../../../src/services/prisma.service'; import { User } from '../../../src/dtos/users/user.dto'; @@ -23,6 +24,49 @@ describe('Testing script runner service', () => { prisma = module.get(PrismaService); }); + it('should transfer data', async () => { + prisma.scriptRuns.findUnique = jest.fn().mockResolvedValue(null); + prisma.scriptRuns.create = jest.fn().mockResolvedValue(null); + prisma.scriptRuns.update = jest.fn().mockResolvedValue(null); + + const id = randomUUID(); + const scriptName = 'data transfer'; + + const res = await service.dataTransfer( + { + user: { + id, + } as unknown as User, + } as unknown as ExpressRequest, + { + connectionString: process.env.TEST_CONNECTION_STRING, + }, + ); + + expect(res.success).toBe(true); + + expect(prisma.scriptRuns.findUnique).toHaveBeenCalledWith({ + where: { + scriptName, + }, + }); + expect(prisma.scriptRuns.create).toHaveBeenCalledWith({ + data: { + scriptName, + triggeringUser: id, + }, + }); + expect(prisma.scriptRuns.update).toHaveBeenCalledWith({ + data: { + didScriptRun: true, + triggeringUser: id, + }, + where: { + scriptName, + }, + }); + }); + // | ---------- HELPER TESTS BELOW ---------- | // it('should mark script run as started if no script run present in db', async () => { prisma.scriptRuns.findUnique = jest.fn().mockResolvedValue(null); diff --git a/shared-helpers/src/types/backend-swagger.ts b/shared-helpers/src/types/backend-swagger.ts index 414710ef3c..ef311c4cca 100644 --- a/shared-helpers/src/types/backend-swagger.ts +++ b/shared-helpers/src/types/backend-swagger.ts @@ -2015,6 +2015,28 @@ export class ScriptRunnerService { configs.data = data + axios(configs, resolve, reject) + }) + } + /** + * A script that pulls data from one source into the current db + */ + dataTransfer( + params: { + /** requestBody */ + body?: DataTransferDTO + } = {} as any, + options: IRequestOptions = {} + ): Promise { + return new Promise((resolve, reject) => { + let url = basePath + "/scriptRunner/dataTransfer" + + const configs: IRequestConfig = getConfigs("put", "application/json", url, options) + + let data = params.body + + configs.data = data + axios(configs, resolve, reject) }) } @@ -5199,6 +5221,11 @@ export interface MapLayer { jurisdictionId: string } +export interface DataTransferDTO { + /** */ + connectionString: string +} + export enum ListingViews { "fundamentals" = "fundamentals", "base" = "base",