Merge pull request #436 from BinaryStudioAcademy/task/OV-423-generate…

…-preview-with-bg OV-423: Generate preview with background
BinaryStudioAcademy · Sep 27, 2024 · 74f7e26 · 74f7e26
2 parents c077a8e + 08de0b0
commit 74f7e26
Show file tree

Hide file tree

Showing 13 changed files with 575 additions and 12 deletions.
diff --git a/backend/package.json b/backend/package.json
@@ -53,6 +53,7 @@
         "pino": "9.3.2",
         "pino-pretty": "10.3.1",
         "shared": "*",
+        "sharp": "0.33.5",
         "socket.io": "4.7.5",
         "swagger-jsdoc": "6.2.8",
         "tiktoken": "1.0.16"

diff --git a/backend/src/bundles/videos/types/types.ts b/backend/src/bundles/videos/types/types.ts
@@ -1,5 +1,6 @@
 export {
     type CreateVideoRequestDto,
+    type Scene,
     type UpdateVideoRequestDto,
     type UserGetCurrentResponseDto,
     type VideoGetAllItemResponseDto,

diff --git a/backend/src/bundles/videos/video.repository.ts b/backend/src/bundles/videos/video.repository.ts
@@ -1,13 +1,22 @@
-import { type UpdateVideoRequestDto } from '~/bundles/videos/types/types.js';
+import {
+    type Scene,
+    type UpdateVideoRequestDto,
+} from '~/bundles/videos/types/types.js';
 import { VideoEntity } from '~/bundles/videos/video.entity.js';
 import { type VideoModel } from '~/bundles/videos/video.model.js';
+import { type ImageService } from '~/common/services/image/image.service.js';
 import { type Repository } from '~/common/types/types.js';
 
 class VideoRepository implements Repository {
     private videoModel: typeof VideoModel;
+    private imageService: ImageService;
 
-    public constructor(videoModel: typeof VideoModel) {
+    public constructor(
+        videoModel: typeof VideoModel,
+        imageService: ImageService,
+    ) {
         this.videoModel = videoModel;
+        this.imageService = imageService;
     }
 
     public async findById(id: string): Promise<VideoEntity | null> {
@@ -58,7 +67,9 @@ class VideoRepository implements Repository {
 
         if (payload.composition) {
             data.composition = payload.composition;
-            data.previewUrl = payload.composition.scenes[0]?.avatar?.url ?? '';
+            data.previewUrl = await this.imageService.generatePreview(
+                payload.composition.scenes[0] as Scene,
+            );
         }
 
         if (payload.name) {

diff --git a/backend/src/bundles/videos/video.service.ts b/backend/src/bundles/videos/video.service.ts
@@ -1,12 +1,14 @@
 import { VideoEntity } from '~/bundles/videos/video.entity.js';
 import { type VideoRepository } from '~/bundles/videos/video.repository.js';
 import { HTTPCode, HttpError } from '~/common/http/http.js';
+import { type ImageService } from '~/common/services/image/image.service.js';
 import { type RemotionService } from '~/common/services/remotion/remotion.service.js';
 import { type Service } from '~/common/types/types.js';
 
 import { VideoValidationMessage } from './enums/enums.js';
 import {
     type CreateVideoRequestDto,
+    type Scene,
     type UpdateVideoRequestDto,
     type VideoGetAllItemResponseDto,
     type VideoGetAllResponseDto,
@@ -15,13 +17,16 @@ import {
 class VideoService implements Service {
     private videoRepository: VideoRepository;
     private remotionService: RemotionService;
+    private imageService: ImageService;
 
     public constructor(
         videoRepository: VideoRepository,
         remotionService: RemotionService,
+        imageService: ImageService,
     ) {
         this.videoRepository = videoRepository;
         this.remotionService = remotionService;
+        this.imageService = imageService;
     }
 
     public async findById(id: string): Promise<VideoGetAllItemResponseDto> {
@@ -56,11 +61,15 @@ class VideoService implements Service {
     public async create(
         payload: CreateVideoRequestDto & { userId: string },
     ): Promise<VideoGetAllItemResponseDto> {
+        const previewUrl = await this.imageService.generatePreview(
+            payload.composition.scenes[0] as Scene,
+        );
+
         const video = await this.videoRepository.create(
             VideoEntity.initializeNew({
                 name: payload.name,
                 composition: payload.composition,
-                previewUrl: payload.composition?.scenes[0]?.avatar?.url ?? '',
+                previewUrl,
                 userId: payload.userId,
             }),
         );

diff --git a/backend/src/bundles/videos/videos.ts b/backend/src/bundles/videos/videos.ts
@@ -1,13 +1,17 @@
 import { logger } from '~/common/logger/logger.js';
-import { remotionService } from '~/common/services/services.js';
+import { imageService, remotionService } from '~/common/services/services.js';
 
 import { VideoController } from './video.controller.js';
 import { VideoModel } from './video.model.js';
 import { VideoRepository } from './video.repository.js';
 import { VideoService } from './video.service.js';
 
-const videoRepository = new VideoRepository(VideoModel);
-const videoService = new VideoService(videoRepository, remotionService);
+const videoRepository = new VideoRepository(VideoModel, imageService);
+const videoService = new VideoService(
+    videoRepository,
+    remotionService,
+    imageService,
+);
 const videoController = new VideoController(logger, videoService);
 
 export { videoController, videoService };
diff --git a/backend/src/common/services/image/constants/constants.ts b/backend/src/common/services/image/constants/constants.ts
@@ -0,0 +1,4 @@
+const PREVIEW_WIDTH = 1920;
+const PREVIEW_HEIGHT = 1080;
+
+export { PREVIEW_HEIGHT, PREVIEW_WIDTH };
diff --git a/backend/src/common/services/image/image-base.ts b/backend/src/common/services/image/image-base.ts
@@ -0,0 +1,25 @@
+import { type Http, ContentType, HTTPMethod } from 'shared';
+
+import { BaseHttpApi } from '~/common/http/base-http-api.js';
+
+type Constructor = {
+    http: Http;
+};
+
+class ImageApi extends BaseHttpApi {
+    public constructor({ http }: Constructor) {
+        super({ path: '', baseUrl: '', http });
+    }
+
+    public async getImageBuffer(url: string): Promise<Buffer> {
+        const response = await this.load(url, {
+            method: HTTPMethod.GET,
+            contentType: ContentType.IMAGE,
+        });
+
+        const arrayBuffer = await response.arrayBuffer();
+        return Buffer.from(arrayBuffer);
+    }
+}
+
+export { ImageApi };
diff --git a/backend/src/common/services/image/image.service.ts b/backend/src/common/services/image/image.service.ts
@@ -0,0 +1,109 @@
+import sharp from 'sharp';
+
+import { type FileService } from '~/common/services/file/file.service.js';
+
+import { PREVIEW_HEIGHT, PREVIEW_WIDTH } from './constants/constants.js';
+import { type ImageApi } from './image-base.js';
+import { type Scene } from './types/types.js';
+
+type Constructor = {
+    fileService: FileService;
+    imageApi: ImageApi;
+};
+
+class ImageService {
+    private fileService: FileService;
+    private imageApi: ImageApi;
+
+    public constructor({ fileService, imageApi }: Constructor) {
+        this.fileService = fileService;
+        this.imageApi = imageApi;
+    }
+
+    public async generatePreview(scene: Scene): Promise<string> {
+        const avatarImage = scene.avatar?.url ?? '';
+        const background = scene.background;
+
+        if (background?.url) {
+            const backgroundImageBuffer = await this.imageApi.getImageBuffer(
+                background.url,
+            );
+
+            return await this.combineAvatarWithBackground(
+                avatarImage,
+                backgroundImageBuffer,
+            );
+        }
+
+        if (background?.color) {
+            const backgroundColorImageBuffer =
+                await this.createImageWithBackgroundColor(background.color);
+
+            return await this.combineAvatarWithBackground(
+                avatarImage,
+                backgroundColorImageBuffer,
+            );
+        }
+
+        return avatarImage;
+    }
+
+    private async combineAvatarWithBackground(
+        avatarImage: string,
+        background: Buffer,
+    ): Promise<string> {
+        const avatarImageBuffer =
+            await this.imageApi.getImageBuffer(avatarImage);
+
+        const previewBuffer = await this.composeImages(
+            avatarImageBuffer,
+            background,
+        );
+
+        const fileName = `preview_${Date.now()}.jpg`;
+
+        await this.fileService.uploadFile(previewBuffer, fileName);
+
+        return this.fileService.getCloudFrontFileUrl(fileName);
+    }
+
+    private async composeImages(
+        avatar: Buffer,
+        background: Buffer,
+    ): Promise<Buffer> {
+        const resizedBackground = await sharp(background)
+            .resize(PREVIEW_WIDTH, PREVIEW_HEIGHT, {
+                fit: 'cover',
+                position: 'center',
+            })
+            .toBuffer();
+
+        const resizedAvatar = await sharp(avatar)
+            .resize(PREVIEW_WIDTH, PREVIEW_HEIGHT, {
+                fit: 'inside',
+                position: 'bottom',
+            })
+            .toBuffer();
+
+        return await sharp(resizedBackground)
+            .composite([{ input: resizedAvatar, blend: 'over' }])
+            .toBuffer();
+    }
+
+    private async createImageWithBackgroundColor(
+        backgroundColor: string,
+    ): Promise<Buffer> {
+        return await sharp({
+            create: {
+                width: PREVIEW_WIDTH,
+                height: PREVIEW_HEIGHT,
+                channels: 3,
+                background: backgroundColor,
+            },
+        })
+            .toFormat('png')
+            .toBuffer();
+    }
+}
+
+export { ImageService };
diff --git a/backend/src/common/services/image/image.ts b/backend/src/common/services/image/image.ts
@@ -0,0 +1,7 @@
+import { baseHttp } from '~/common/http/http.js';
+
+import { ImageApi } from './image-base.js';
+
+const imageApi = new ImageApi({ http: baseHttp });
+
+export { imageApi };
diff --git a/backend/src/common/services/image/types/types.ts b/backend/src/common/services/image/types/types.ts
@@ -0,0 +1 @@
+export { type Scene } from 'shared';
diff --git a/backend/src/common/services/services.ts b/backend/src/common/services/services.ts
@@ -5,6 +5,8 @@ import { AzureAIService } from './azure-ai/azure-ai.service.js';
 import { textToSpeechApi } from './azure-ai/text-to-speech/text-to-speech.js';
 import { CryptService } from './crypt/crypt.service.js';
 import { FileService } from './file/file.service.js';
+import { imageApi } from './image/image.js';
+import { ImageService } from './image/image.service.js';
 import { OpenAIService } from './open-ai/open-ai.service.js';
 import { RemotionService } from './remotion/remotion.service.js';
 import { TokenService } from './token/token.services.js';
@@ -23,11 +25,13 @@ const azureAIService = new AzureAIService({
 const secretKey = config.ENV.TOKEN.SECRET_KEY;
 const expirationTime = config.ENV.TOKEN.EXPIRATION_TIME;
 const tokenService = new TokenService(secretKey, expirationTime);
+const imageService = new ImageService({ fileService, imageApi });
 
 export {
     azureAIService,
     cryptService,
     fileService,
+    imageService,
     openAIService,
     remotionService,
     tokenService,