generated from bcgov/quickstart-openshift
-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: add object store integration (#24)
- Loading branch information
Showing
16 changed files
with
2,046 additions
and
688 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,22 +1,23 @@ | ||
| import { Test, TestingModule } from '@nestjs/testing'; | ||
| import { AppController } from './app.controller'; | ||
| import { AppService } from './app.service'; | ||
| import { Test, TestingModule } from "@nestjs/testing"; | ||
| import { AppController } from "./app.controller"; | ||
| import { AppService } from "./app.service"; | ||
| import { ObjectStoreService } from "./v1/object-store/object.store.service"; | ||
|
|
||
| describe('AppController', () => { | ||
| describe("AppController", () => { | ||
| let appController: AppController; | ||
|
|
||
| beforeEach(async () => { | ||
| const app: TestingModule = await Test.createTestingModule({ | ||
| controllers: [AppController], | ||
| providers: [AppService], | ||
| providers: [AppService, ObjectStoreService], | ||
| }).compile(); | ||
|
|
||
| appController = app.get<AppController>(AppController); | ||
| }); | ||
|
|
||
| describe('root', () => { | ||
| describe("root", () => { | ||
| it('should return "Hello Backend!"', () => { | ||
| expect(appController.getHello()).toBe('Hello Backend!'); | ||
| expect(appController.getHello()).toBe("Hello Backend!"); | ||
| }); | ||
| }); | ||
| }); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,12 +1,19 @@ | ||
| import { Controller, Get } from '@nestjs/common'; | ||
| import { AppService } from './app.service'; | ||
| import { ObjectStoreService } from './v1/object-store/object.store.service' | ||
| import { OmrrData } from './v1/types/omrr-data' | ||
|
|
||
| @Controller() | ||
| export class AppController { | ||
| constructor(private readonly appService: AppService) {} | ||
| constructor(private readonly appService: AppService, | ||
| private readonly objectStoreService: ObjectStoreService) {} | ||
|
|
||
| @Get() | ||
| getHello(): string { | ||
| return this.appService.getHello(); | ||
| } | ||
| @Get('/omrr') | ||
| async getAllOmrrRecords(): Promise<OmrrData[]> { | ||
| return this.objectStoreService.getLatestOMRRFileContents(); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,10 @@ | ||
| import { Module } from '@nestjs/common' | ||
| import { ObjectStoreService } from './object.store.service' | ||
|
|
||
| @Module({ | ||
| providers: [ObjectStoreService], | ||
| exports: [ObjectStoreService], | ||
| }) | ||
| export class ObjectStoreModule{ | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,104 @@ | ||
| import { Injectable, OnModuleDestroy, OnModuleInit } from '@nestjs/common' | ||
| import { GetObjectCommand, ListObjectsCommand, S3Client } from '@aws-sdk/client-s3' | ||
| import * as process from 'process' | ||
| import { logger } from '../../logger' | ||
| import { Readable } from 'stream' | ||
| import Papa from 'papaparse' | ||
| import { OmrrData } from '../types/omrr-data' | ||
|
|
||
| @Injectable() | ||
| export class ObjectStoreService implements OnModuleDestroy, OnModuleInit { | ||
| private readonly _s3Client: S3Client | ||
| private _omrrData: OmrrData[] = [] | ||
|
|
||
| constructor() { | ||
| this._s3Client = new S3Client({ | ||
| credentials: { | ||
| accessKeyId: process.env.OS_ACCESS_KEY_ID, | ||
| secretAccessKey: process.env.OS_SECRET_ACCESS_KEY_ID, | ||
| }, | ||
| endpoint: process.env.OS_ENDPOINT, | ||
| forcePathStyle: true, | ||
| region: 'ca-central-1', | ||
| }) | ||
| } | ||
|
|
||
| async onModuleInit() { | ||
| try { | ||
| logger.info('Initializing ObjectStoreService'); | ||
| await this.getLatestOmrrDataFromObjectStore() | ||
| } catch (e) { | ||
| logger.error(e) | ||
| process.exit(1) | ||
| } | ||
| } | ||
|
|
||
| onModuleDestroy() { | ||
| this._s3Client.destroy() | ||
| } | ||
|
|
||
| async parseCSVToObject(csv: string): Promise<any[]> { | ||
| return new Promise((resolve, reject) => { | ||
| Papa.parse(csv, { | ||
| header: true, | ||
| complete: (results: any) => { | ||
| resolve(results.data) | ||
| }, | ||
| error: (error: any) => { | ||
| reject(error) | ||
| }, | ||
| }) | ||
| }) | ||
| } | ||
|
|
||
| async convertCSVToJson(stream: Readable): Promise<any[]> { | ||
| return new Promise((resolve, reject) => { | ||
| let csvData = '' | ||
| stream.on('data', (chunk) => { | ||
| csvData += chunk | ||
| }) | ||
| stream.on('end', async () => { | ||
| try { | ||
| const jsonData = await this.parseCSVToObject(csvData) | ||
| resolve(jsonData) | ||
| } catch (error) { | ||
| reject(error) | ||
| } | ||
| }) | ||
| stream.on('error', (error) => { | ||
| reject(error) | ||
| }) | ||
| }) | ||
| } | ||
|
|
||
| async getLatestOMRRFileContents(): Promise<OmrrData[]> { | ||
| return this._omrrData | ||
| } | ||
|
|
||
| async getLatestOmrrDataFromObjectStore(): Promise<OmrrData[]> { | ||
| try { | ||
| const response = await this._s3Client.send( | ||
| new ListObjectsCommand({ Bucket: process.env.OS_BUCKET }), | ||
| ) | ||
| let sortedData: any = response.Contents.sort((a: any, b: any) => { | ||
| const modifiedDateA: any = new Date(a.LastModified) | ||
| const modifiedDateB: any = new Date(b.LastModified) | ||
| return modifiedDateB - modifiedDateA | ||
| }) | ||
| const fileName = sortedData[0]?.Key | ||
| logger.info(`fileName is ${fileName}`) | ||
|
|
||
| const result = await this._s3Client.send( | ||
| new GetObjectCommand({ Bucket: process.env.OS_BUCKET, Key: fileName }), | ||
| ) | ||
| const fileStream: any = result?.Body | ||
| if (fileStream) { | ||
| this._omrrData = await this.convertCSVToJson(fileStream) | ||
| return this._omrrData | ||
| } | ||
| } catch (e) { | ||
| } | ||
|
|
||
| throw new Error('No file found') | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,9 @@ | ||
| import { Module } from '@nestjs/common'; | ||
| import { TasksService } from './task.service'; | ||
| import { ObjectStoreModule } from '../object-store/object.store.module' | ||
|
|
||
| @Module({ | ||
| imports: [ObjectStoreModule], | ||
| providers: [TasksService], | ||
| }) | ||
| export class TasksModule {} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,20 @@ | ||
| import { Injectable, Logger } from '@nestjs/common' | ||
| import { Cron } from '@nestjs/schedule' | ||
| import { ObjectStoreService } from '../object-store/object.store.service' | ||
| import { logger } from '../../logger' | ||
|
|
||
| @Injectable() | ||
| export class TasksService { | ||
|
|
||
| constructor(private readonly objectStoreService: ObjectStoreService) { | ||
|
|
||
| } | ||
|
|
||
| @Cron('0 0 0/4 * * *') | ||
| async refreshCache() { | ||
| logger.info('refresh cache every 4 hours') | ||
| await this.objectStoreService.getLatestOmrrDataFromObjectStore() | ||
|
|
||
| } | ||
|
|
||
| } |
Oops, something went wrong.