-
Notifications
You must be signed in to change notification settings - Fork 1.2k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
chore: Integration with MinIO and S3
Adds support for MinIO and S3 for storing media files. Modified several files to implement this feature, including package.json, prisma/postgresql-schema.prisma, src/api/integrations/typebot/services/typebot.service.ts, src/api/routes/index.router.ts, src/api/services/channels/whatsapp.baileys.service.ts, and src/config/env.config.ts. Added untracked files for the new S3 integration. Also added a new S3Controller and S3Service for handling S3 related operations. This change allows for more flexible media storage options and enables the use of MinIO or S3 for storing media files.
- Loading branch information
1 parent
f7a731a
commit e73d9c1
Showing
14 changed files
with
364 additions
and
15 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
24 changes: 24 additions & 0 deletions
24
prisma/migrations/20240713184337_add_media_table/migration.sql
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,24 @@ | ||
-- CreateTable | ||
CREATE TABLE "Media" ( | ||
"id" TEXT NOT NULL, | ||
"fileName" VARCHAR(500) NOT NULL, | ||
"type" VARCHAR(100) NOT NULL, | ||
"mimetype" VARCHAR(100) NOT NULL, | ||
"createdAt" DATE DEFAULT CURRENT_TIMESTAMP, | ||
"messageId" TEXT NOT NULL, | ||
"instanceId" TEXT NOT NULL, | ||
|
||
CONSTRAINT "Media_pkey" PRIMARY KEY ("id") | ||
); | ||
|
||
-- CreateIndex | ||
CREATE UNIQUE INDEX "Media_fileName_key" ON "Media"("fileName"); | ||
|
||
-- CreateIndex | ||
CREATE UNIQUE INDEX "Media_messageId_key" ON "Media"("messageId"); | ||
|
||
-- AddForeignKey | ||
ALTER TABLE "Media" ADD CONSTRAINT "Media_messageId_fkey" FOREIGN KEY ("messageId") REFERENCES "Message"("id") ON DELETE CASCADE ON UPDATE CASCADE; | ||
|
||
-- AddForeignKey | ||
ALTER TABLE "Media" ADD CONSTRAINT "Media_instanceId_fkey" FOREIGN KEY ("instanceId") REFERENCES "Instance"("id") ON DELETE CASCADE ON UPDATE CASCADE; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,15 @@ | ||
import { InstanceDto } from '../../../dto/instance.dto'; | ||
import { MediaDto } from '../dto/media.dto'; | ||
import { S3Service } from '../services/s3.service'; | ||
|
||
export class S3Controller { | ||
constructor(private readonly s3Service: S3Service) {} | ||
|
||
public async getMedia(instance: InstanceDto, data: MediaDto) { | ||
return this.s3Service.getMedia(instance, data); | ||
} | ||
|
||
public async getMediaUrl(instance: InstanceDto, data: MediaDto) { | ||
return this.s3Service.getMediaUrl(instance, data); | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,6 @@ | ||
export class MediaDto { | ||
id?: string; | ||
type?: string; | ||
messageId?: number; | ||
expiry?: number; | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,88 @@ | ||
import * as MinIo from 'minio'; | ||
import { join } from 'path'; | ||
import { Readable, Transform } from 'stream'; | ||
|
||
import { ConfigService, S3 } from '../../../../config/env.config'; | ||
import { Logger } from '../../../../config/logger.config'; | ||
import { BadRequestException } from '../../../../exceptions'; | ||
|
||
const logger = new Logger('S3 Service'); | ||
|
||
const BUCKET = new ConfigService().get<S3>('S3'); | ||
|
||
interface Metadata extends MinIo.ItemBucketMetadata { | ||
'Content-Type': string; | ||
} | ||
|
||
const minioClient = (() => { | ||
if (BUCKET?.ENABLE) { | ||
return new MinIo.Client({ | ||
endPoint: BUCKET.ENDPOINT, | ||
port: BUCKET.PORT, | ||
useSSL: BUCKET.USE_SSL, | ||
accessKey: BUCKET.ACCESS_KEY, | ||
secretKey: BUCKET.SECRET_KEY, | ||
}); | ||
} | ||
})(); | ||
|
||
const bucketName = process.env.S3_BUCKET; | ||
|
||
const bucketExists = async () => { | ||
if (minioClient) { | ||
try { | ||
const list = await minioClient.listBuckets(); | ||
return list.find((bucket) => bucket.name === bucketName); | ||
} catch (error) { | ||
return false; | ||
} | ||
} | ||
}; | ||
|
||
const createBucket = async () => { | ||
if (minioClient) { | ||
try { | ||
const exists = await bucketExists(); | ||
if (!exists) { | ||
await minioClient.makeBucket(bucketName); | ||
} | ||
|
||
logger.info(`S3 Bucket ${bucketName} - ON`); | ||
return true; | ||
} catch (error) { | ||
console.log('S3 ERROR: ', error); | ||
return false; | ||
} | ||
} | ||
}; | ||
|
||
createBucket(); | ||
|
||
const uploadFile = async (fileName: string, file: Buffer | Transform | Readable, size: number, metadata: Metadata) => { | ||
if (minioClient) { | ||
const objectName = join('evolution-api', fileName); | ||
try { | ||
metadata['custom-header-application'] = 'evolution-api'; | ||
return await minioClient.putObject(bucketName, objectName, file, size, metadata); | ||
} catch (error) { | ||
console.log('ERROR: ', error); | ||
return error; | ||
} | ||
} | ||
}; | ||
|
||
const getObjectUrl = async (fileName: string, expiry?: number) => { | ||
if (minioClient) { | ||
try { | ||
const objectName = join('evolution-api', fileName); | ||
if (expiry) { | ||
return await minioClient.presignedGetObject(bucketName, objectName, expiry); | ||
} | ||
return await minioClient.presignedGetObject(bucketName, objectName); | ||
} catch (error) { | ||
throw new BadRequestException(error?.message); | ||
} | ||
} | ||
}; | ||
|
||
export { BUCKET, getObjectUrl, uploadFile }; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,36 @@ | ||
import { RequestHandler, Router } from 'express'; | ||
|
||
import { RouterBroker } from '../../../abstract/abstract.router'; | ||
import { HttpStatus } from '../../../routes/index.router'; | ||
import { s3Controller } from '../../../server.module'; | ||
import { MediaDto } from '../dto/media.dto'; | ||
import { s3Schema, s3UrlSchema } from '../validate/s3.schema'; | ||
|
||
export class S3Router extends RouterBroker { | ||
constructor(...guards: RequestHandler[]) { | ||
super(); | ||
this.router | ||
.post(this.routerPath('getMedia'), ...guards, async (req, res) => { | ||
const response = await this.dataValidate<MediaDto>({ | ||
request: req, | ||
schema: s3Schema, | ||
ClassRef: MediaDto, | ||
execute: (instance, data) => s3Controller.getMedia(instance, data), | ||
}); | ||
|
||
res.status(HttpStatus.CREATED).json(response); | ||
}) | ||
.post(this.routerPath('getMediaUrl'), ...guards, async (req, res) => { | ||
const response = await this.dataValidate<MediaDto>({ | ||
request: req, | ||
schema: s3UrlSchema, | ||
ClassRef: MediaDto, | ||
execute: (instance, data) => s3Controller.getMediaUrl(instance, data), | ||
}); | ||
|
||
res.status(HttpStatus.OK).json(response); | ||
}); | ||
} | ||
|
||
public readonly router = Router(); | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,50 @@ | ||
import { Logger } from '../../../../config/logger.config'; | ||
import { BadRequestException } from '../../../../exceptions'; | ||
import { InstanceDto } from '../../../dto/instance.dto'; | ||
import { PrismaRepository } from '../../../repository/repository.service'; | ||
import { MediaDto } from '../dto/media.dto'; | ||
import { getObjectUrl } from '../libs/minio.server'; | ||
|
||
export class S3Service { | ||
constructor(private readonly prismaRepository: PrismaRepository) {} | ||
|
||
private readonly logger = new Logger(S3Service.name); | ||
|
||
public async getMedia(instance: InstanceDto, query?: MediaDto) { | ||
try { | ||
const where: any = { | ||
instanceId: instance.instanceId, | ||
...query, | ||
}; | ||
|
||
const media = await this.prismaRepository.media.findMany({ | ||
where, | ||
select: { | ||
id: true, | ||
fileName: true, | ||
type: true, | ||
mimetype: true, | ||
createdAt: true, | ||
Message: true, | ||
}, | ||
}); | ||
|
||
if (!media || media.length === 0) { | ||
throw 'Media not found'; | ||
} | ||
|
||
return media; | ||
} catch (error) { | ||
throw new BadRequestException(error); | ||
} | ||
} | ||
|
||
public async getMediaUrl(instance: InstanceDto, data: MediaDto) { | ||
const media = (await this.getMedia(instance, { id: data.id }))[0]; | ||
const mediaUrl = await getObjectUrl(media.fileName, data.expiry); | ||
return { | ||
mediaUrl, | ||
...media, | ||
}; | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,43 @@ | ||
import { JSONSchema7 } from 'json-schema'; | ||
import { v4 } from 'uuid'; | ||
|
||
const isNotEmpty = (...propertyNames: string[]): JSONSchema7 => { | ||
const properties = {}; | ||
propertyNames.forEach( | ||
(property) => | ||
(properties[property] = { | ||
minLength: 1, | ||
description: `The "${property}" cannot be empty`, | ||
}), | ||
); | ||
return { | ||
if: { | ||
propertyNames: { | ||
enum: [...propertyNames], | ||
}, | ||
}, | ||
then: { properties }, | ||
}; | ||
}; | ||
|
||
export const s3Schema: JSONSchema7 = { | ||
$id: v4(), | ||
type: 'object', | ||
properties: { | ||
id: { type: 'string' }, | ||
type: { type: 'string' }, | ||
messageId: { type: 'integer' }, | ||
}, | ||
...isNotEmpty('id', 'type', 'messageId'), | ||
}; | ||
|
||
export const s3UrlSchema: JSONSchema7 = { | ||
$id: v4(), | ||
type: 'object', | ||
properties: { | ||
id: { type: 'string', pattern: '\\d+', minLength: 1 }, | ||
expiry: { type: 'string', pattern: '\\d+', minLength: 1 }, | ||
}, | ||
...isNotEmpty('id'), | ||
required: ['id'], | ||
}; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.