From 67a8ec9f4b5ac077afb6f7ca214e058dab0f6e0f Mon Sep 17 00:00:00 2001 From: Oleksandra Okhotnykova Date: Wed, 25 Sep 2024 17:47:36 +0200 Subject: [PATCH 1/7] OV-423: + create image service --- backend/src/bundles/videos/video.service.ts | 10 ++++++- backend/src/bundles/videos/videos.ts | 8 +++-- .../common/services/image/image.service.ts | 29 +++++++++++++++++++ .../src/common/services/image/types/types.ts | 1 + backend/src/common/services/services.ts | 3 ++ 5 files changed, 48 insertions(+), 3 deletions(-) create mode 100644 backend/src/common/services/image/image.service.ts create mode 100644 backend/src/common/services/image/types/types.ts diff --git a/backend/src/bundles/videos/video.service.ts b/backend/src/bundles/videos/video.service.ts index fad0bacb5..84ea007fb 100644 --- a/backend/src/bundles/videos/video.service.ts +++ b/backend/src/bundles/videos/video.service.ts @@ -2,6 +2,7 @@ import { VideoEntity } from '~/bundles/videos/video.entity.js'; import { type VideoRepository } from '~/bundles/videos/video.repository.js'; import { HTTPCode, HttpError } from '~/common/http/http.js'; import { type FileService } from '~/common/services/file/file.service.js'; +import { type ImageService } from '~/common/services/image/image.service.js'; import { type Service } from '~/common/types/types.js'; import { VideoValidationMessage } from './enums/enums.js'; @@ -15,13 +16,16 @@ import { class VideoService implements Service { private videoRepository: VideoRepository; private fileService: FileService; + private imageService: ImageService; public constructor( videoRepository: VideoRepository, fileService: FileService, + imageService: ImageService, ) { this.videoRepository = videoRepository; this.fileService = fileService; + this.imageService = imageService; } public async findById(id: string): Promise { @@ -56,11 +60,15 @@ class VideoService implements Service { public async create( payload: CreateVideoRequestDto & { userId: string }, ): Promise { + const previewUrl = this.imageService.generatePreview( + payload.composition, + ); + const video = await this.videoRepository.create( VideoEntity.initializeNew({ name: payload.name, composition: payload.composition, - previewUrl: payload.composition?.scenes[0]?.avatar?.url ?? '', + previewUrl, userId: payload.userId, }), ); diff --git a/backend/src/bundles/videos/videos.ts b/backend/src/bundles/videos/videos.ts index 133a4dcab..adb063902 100644 --- a/backend/src/bundles/videos/videos.ts +++ b/backend/src/bundles/videos/videos.ts @@ -1,5 +1,5 @@ import { logger } from '~/common/logger/logger.js'; -import { fileService } from '~/common/services/services.js'; +import { fileService, imageService } from '~/common/services/services.js'; import { VideoController } from './video.controller.js'; import { VideoModel } from './video.model.js'; @@ -7,7 +7,11 @@ import { VideoRepository } from './video.repository.js'; import { VideoService } from './video.service.js'; const videoRepository = new VideoRepository(VideoModel); -const videoService = new VideoService(videoRepository, fileService); +const videoService = new VideoService( + videoRepository, + fileService, + imageService, +); const videoController = new VideoController(logger, videoService); export { videoController, videoService }; diff --git a/backend/src/common/services/image/image.service.ts b/backend/src/common/services/image/image.service.ts new file mode 100644 index 000000000..a2eb288f9 --- /dev/null +++ b/backend/src/common/services/image/image.service.ts @@ -0,0 +1,29 @@ +import { type FileService } from '~/common/services/file/file.service.js'; + +import { type Composition } from './types/types.js'; + +class ImageService { + private fileService: FileService; + + public constructor(fileService: FileService) { + this.fileService = fileService; + } + + public generatePreview(composition: Composition): string { + const avatarImage = composition.scenes[0]?.avatar?.url ?? ''; + const background = composition.scenes[0]?.background; + + if (background?.url) { + // TODO: combine avatar and bg + } + + if (background?.color) { + // TODO: create empty image with bg color + // then combine avatar and this new image + } + + return avatarImage; + } +} + +export { ImageService }; diff --git a/backend/src/common/services/image/types/types.ts b/backend/src/common/services/image/types/types.ts new file mode 100644 index 000000000..91ee6dd10 --- /dev/null +++ b/backend/src/common/services/image/types/types.ts @@ -0,0 +1 @@ +export { type Composition } from 'shared'; diff --git a/backend/src/common/services/services.ts b/backend/src/common/services/services.ts index 8d4bc32d8..667efa895 100644 --- a/backend/src/common/services/services.ts +++ b/backend/src/common/services/services.ts @@ -5,6 +5,7 @@ import { AzureAIService } from './azure-ai/azure-ai.service.js'; import { textToSpeechApi } from './azure-ai/text-to-speech/text-to-speech.js'; import { CryptService } from './crypt/crypt.service.js'; import { FileService } from './file/file.service.js'; +import { ImageService } from './image/image.service.js'; import { OpenAIService } from './open-ai/open-ai.service.js'; import { RemotionService } from './remotion/remotion.service.js'; import { TokenService } from './token/token.services.js'; @@ -23,11 +24,13 @@ const azureAIService = new AzureAIService({ const secretKey = config.ENV.TOKEN.SECRET_KEY; const expirationTime = config.ENV.TOKEN.EXPIRATION_TIME; const tokenService = new TokenService(secretKey, expirationTime); +const imageService = new ImageService(fileService); export { azureAIService, cryptService, fileService, + imageService, openAIService, remotionService, tokenService, From 0ea029c8532a93f818b8d3e65d341d225b74a736 Mon Sep 17 00:00:00 2001 From: Oleksandra Okhotnykova Date: Wed, 25 Sep 2024 18:28:59 +0200 Subject: [PATCH 2/7] OV-423: + image api --- .../src/common/services/image/image-base.ts | 25 +++++++++++++++++++ .../common/services/image/image.service.ts | 10 +++++++- backend/src/common/services/image/image.ts | 7 ++++++ backend/src/common/services/services.ts | 3 ++- shared/src/enums/content-type.enum.ts | 1 + 5 files changed, 44 insertions(+), 2 deletions(-) create mode 100644 backend/src/common/services/image/image-base.ts create mode 100644 backend/src/common/services/image/image.ts diff --git a/backend/src/common/services/image/image-base.ts b/backend/src/common/services/image/image-base.ts new file mode 100644 index 000000000..73694ff10 --- /dev/null +++ b/backend/src/common/services/image/image-base.ts @@ -0,0 +1,25 @@ +import { type Http, ContentType, HTTPMethod } from 'shared'; + +import { BaseHttpApi } from '~/common/http/base-http-api.js'; + +type Constructor = { + http: Http; +}; + +class ImageApi extends BaseHttpApi { + public constructor({ http }: Constructor) { + super({ path: '', baseUrl: '', http }); + } + + public async getImageBuffer(url: string): Promise { + const response = await this.load(url, { + method: HTTPMethod.GET, + contentType: ContentType.IMAGE, + }); + + const arrayBuffer = await response.arrayBuffer(); + return Buffer.from(arrayBuffer); + } +} + +export { ImageApi }; diff --git a/backend/src/common/services/image/image.service.ts b/backend/src/common/services/image/image.service.ts index a2eb288f9..9c0dcd9a5 100644 --- a/backend/src/common/services/image/image.service.ts +++ b/backend/src/common/services/image/image.service.ts @@ -1,12 +1,20 @@ import { type FileService } from '~/common/services/file/file.service.js'; +import { type ImageApi } from './image-base.js'; import { type Composition } from './types/types.js'; +type Constructor = { + fileService: FileService; + imageApi: ImageApi; +}; + class ImageService { private fileService: FileService; + private imageApi: ImageApi; - public constructor(fileService: FileService) { + public constructor({ fileService, imageApi }: Constructor) { this.fileService = fileService; + this.imageApi = imageApi; } public generatePreview(composition: Composition): string { diff --git a/backend/src/common/services/image/image.ts b/backend/src/common/services/image/image.ts new file mode 100644 index 000000000..14b4c7f9d --- /dev/null +++ b/backend/src/common/services/image/image.ts @@ -0,0 +1,7 @@ +import { baseHttp } from '~/common/http/http.js'; + +import { ImageApi } from './image-base.js'; + +const imageApi = new ImageApi({ http: baseHttp }); + +export { imageApi }; diff --git a/backend/src/common/services/services.ts b/backend/src/common/services/services.ts index 667efa895..ffe6ceee6 100644 --- a/backend/src/common/services/services.ts +++ b/backend/src/common/services/services.ts @@ -5,6 +5,7 @@ import { AzureAIService } from './azure-ai/azure-ai.service.js'; import { textToSpeechApi } from './azure-ai/text-to-speech/text-to-speech.js'; import { CryptService } from './crypt/crypt.service.js'; import { FileService } from './file/file.service.js'; +import { imageApi } from './image/image.js'; import { ImageService } from './image/image.service.js'; import { OpenAIService } from './open-ai/open-ai.service.js'; import { RemotionService } from './remotion/remotion.service.js'; @@ -24,7 +25,7 @@ const azureAIService = new AzureAIService({ const secretKey = config.ENV.TOKEN.SECRET_KEY; const expirationTime = config.ENV.TOKEN.EXPIRATION_TIME; const tokenService = new TokenService(secretKey, expirationTime); -const imageService = new ImageService(fileService); +const imageService = new ImageService({ fileService, imageApi }); export { azureAIService, diff --git a/shared/src/enums/content-type.enum.ts b/shared/src/enums/content-type.enum.ts index 333af187e..22e011104 100644 --- a/shared/src/enums/content-type.enum.ts +++ b/shared/src/enums/content-type.enum.ts @@ -1,5 +1,6 @@ const ContentType = { JSON: 'application/json', + IMAGE: 'image/*', } as const; export { ContentType }; From 768646d27c8e25accdae0cc6cd7e97e1a87fcdc9 Mon Sep 17 00:00:00 2001 From: Oleksandra Okhotnykova Date: Wed, 25 Sep 2024 21:12:29 +0200 Subject: [PATCH 3/7] OV-423: * compose images, upload file and get preview url --- backend/package.json | 1 + backend/src/bundles/videos/video.service.ts | 2 +- .../common/services/image/image.service.ts | 47 ++- package-lock.json | 396 +++++++++++++++++- 4 files changed, 438 insertions(+), 8 deletions(-) diff --git a/backend/package.json b/backend/package.json index 589e2364e..4708cba5f 100644 --- a/backend/package.json +++ b/backend/package.json @@ -53,6 +53,7 @@ "pino": "9.3.2", "pino-pretty": "10.3.1", "shared": "*", + "sharp": "0.33.5", "socket.io": "4.7.5", "swagger-jsdoc": "6.2.8", "tiktoken": "1.0.16" diff --git a/backend/src/bundles/videos/video.service.ts b/backend/src/bundles/videos/video.service.ts index 84ea007fb..1745497db 100644 --- a/backend/src/bundles/videos/video.service.ts +++ b/backend/src/bundles/videos/video.service.ts @@ -60,7 +60,7 @@ class VideoService implements Service { public async create( payload: CreateVideoRequestDto & { userId: string }, ): Promise { - const previewUrl = this.imageService.generatePreview( + const previewUrl = await this.imageService.generatePreview( payload.composition, ); diff --git a/backend/src/common/services/image/image.service.ts b/backend/src/common/services/image/image.service.ts index 9c0dcd9a5..85b580a2c 100644 --- a/backend/src/common/services/image/image.service.ts +++ b/backend/src/common/services/image/image.service.ts @@ -1,3 +1,5 @@ +import sharp from 'sharp'; + import { type FileService } from '~/common/services/file/file.service.js'; import { type ImageApi } from './image-base.js'; @@ -17,12 +19,27 @@ class ImageService { this.imageApi = imageApi; } - public generatePreview(composition: Composition): string { + public async generatePreview(composition: Composition): Promise { const avatarImage = composition.scenes[0]?.avatar?.url ?? ''; const background = composition.scenes[0]?.background; if (background?.url) { - // TODO: combine avatar and bg + const avatarImageBuffer = + await this.imageApi.getImageBuffer(avatarImage); + const backgroundImageBuffer = await this.imageApi.getImageBuffer( + background.url, + ); + + const previewBuffer = await this.composeImages( + avatarImageBuffer, + backgroundImageBuffer, + ); + + const fileName = `preview_${Date.now()}.jpg`; + + await this.fileService.uploadFile(previewBuffer, fileName); + + return this.fileService.getCloudFrontFileUrl(fileName); } if (background?.color) { @@ -32,6 +49,32 @@ class ImageService { return avatarImage; } + + private async composeImages( + avatar: Buffer, + background: Buffer, + ): Promise { + const finalWidth = 1920; + const finalHeight = 1080; + + const resizedBackground = await sharp(background) + .resize(finalWidth, finalHeight, { + fit: 'cover', + position: 'center', + }) + .toBuffer(); + + const resizedAvatar = await sharp(avatar) + .resize(finalWidth, finalHeight, { + fit: 'inside', + position: 'bottom', + }) + .toBuffer(); + + return await sharp(resizedBackground) + .composite([{ input: resizedAvatar, blend: 'over' }]) + .toBuffer(); + } } export { ImageService }; diff --git a/package-lock.json b/package-lock.json index 4654a6dd9..c97c8bb5f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -66,6 +66,7 @@ "pino": "9.3.2", "pino-pretty": "10.3.1", "shared": "*", + "sharp": "0.33.5", "socket.io": "4.7.5", "swagger-jsdoc": "6.2.8", "tiktoken": "1.0.16" @@ -84,6 +85,44 @@ "npm": "10.x.x" } }, + "backend/node_modules/sharp": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.33.5.tgz", + "integrity": "sha512-haPVm1EkS9pgvHrQ/F3Xy+hgcuMV0Wm9vfIBSiwZ05k+xgb0PkBQpGsAA/oWdDobNaZTH5ppvHtzCFbnSEwHVw==", + "hasInstallScript": true, + "dependencies": { + "color": "^4.2.3", + "detect-libc": "^2.0.3", + "semver": "^7.6.3" + }, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-darwin-arm64": "0.33.5", + "@img/sharp-darwin-x64": "0.33.5", + "@img/sharp-libvips-darwin-arm64": "1.0.4", + "@img/sharp-libvips-darwin-x64": "1.0.4", + "@img/sharp-libvips-linux-arm": "1.0.5", + "@img/sharp-libvips-linux-arm64": "1.0.4", + "@img/sharp-libvips-linux-s390x": "1.0.4", + "@img/sharp-libvips-linux-x64": "1.0.4", + "@img/sharp-libvips-linuxmusl-arm64": "1.0.4", + "@img/sharp-libvips-linuxmusl-x64": "1.0.4", + "@img/sharp-linux-arm": "0.33.5", + "@img/sharp-linux-arm64": "0.33.5", + "@img/sharp-linux-s390x": "0.33.5", + "@img/sharp-linux-x64": "0.33.5", + "@img/sharp-linuxmusl-arm64": "0.33.5", + "@img/sharp-linuxmusl-x64": "0.33.5", + "@img/sharp-wasm32": "0.33.5", + "@img/sharp-win32-ia32": "0.33.5", + "@img/sharp-win32-x64": "0.33.5" + } + }, "frontend": { "version": "1.0.0", "dependencies": { @@ -8173,6 +8212,15 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/@emnapi/runtime": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.2.0.tgz", + "integrity": "sha512-bV21/9LQmcQeCPEg3BDFtvwL6cwiTMksYNWQQ4KOxCZikEGalWtenoZ0wCiukJINlGCIi2KXx01g4FoH/LxpzQ==", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, "node_modules/@emotion/babel-plugin": { "version": "11.12.0", "resolved": "https://registry.npmjs.org/@emotion/babel-plugin/-/babel-plugin-11.12.0.tgz", @@ -9130,6 +9178,348 @@ "deprecated": "Use @eslint/object-schema instead", "dev": true }, + "node_modules/@img/sharp-darwin-arm64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.33.5.tgz", + "integrity": "sha512-UT4p+iz/2H4twwAoLCqfA9UH5pI6DggwKEGuaPy7nCVQ8ZsiY5PIcrRvD1DzuY3qYL07NtIQcWnBSY/heikIFQ==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-arm64": "1.0.4" + } + }, + "node_modules/@img/sharp-darwin-x64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.33.5.tgz", + "integrity": "sha512-fyHac4jIc1ANYGRDxtiqelIbdWkIuQaI84Mv45KvGRRxSAa7o7d1ZKAOBaYbnepLC1WqxfpimdeWfvqqSGwR2Q==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-x64": "1.0.4" + } + }, + "node_modules/@img/sharp-libvips-darwin-arm64": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.0.4.tgz", + "integrity": "sha512-XblONe153h0O2zuFfTAbQYAX2JhYmDHeWikp1LM9Hul9gVPjFY427k6dFEcOL72O01QxQsWi761svJ/ev9xEDg==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-darwin-x64": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.0.4.tgz", + "integrity": "sha512-xnGR8YuZYfJGmWPvmlunFaWJsb9T/AO2ykoP3Fz/0X5XV2aoYBPkX6xqCQvUTKKiLddarLaxpzNe+b1hjeWHAQ==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.0.5.tgz", + "integrity": "sha512-gvcC4ACAOPRNATg/ov8/MnbxFDJqf/pDePbBnuBDcjsI8PssmjoKMAz4LtLaVi+OnSb5FK/yIOamqDwGmXW32g==", + "cpu": [ + "arm" + ], + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm64": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.0.4.tgz", + "integrity": "sha512-9B+taZ8DlyyqzZQnoeIvDVR/2F4EbMepXMc/NdVbkzsJbzkUjhXv/70GQJ7tdLA4YJgNP25zukcxpX2/SueNrA==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-s390x": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.0.4.tgz", + "integrity": "sha512-u7Wz6ntiSSgGSGcjZ55im6uvTrOxSIS8/dgoVMoiGE9I6JAfU50yH5BoDlYA1tcuGS7g/QNtetJnxA6QEsCVTA==", + "cpu": [ + "s390x" + ], + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-x64": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.0.4.tgz", + "integrity": "sha512-MmWmQ3iPFZr0Iev+BAgVMb3ZyC4KeFc3jFxnNbEPas60e1cIfevbtuyf9nDGIzOaW9PdnDciJm+wFFaTlj5xYw==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-arm64": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.0.4.tgz", + "integrity": "sha512-9Ti+BbTYDcsbp4wfYib8Ctm1ilkugkA/uscUn6UXK1ldpC1JjiXbLfFZtRlBhjPZ5o1NCLiDbg8fhUPKStHoTA==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-x64": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.0.4.tgz", + "integrity": "sha512-viYN1KX9m+/hGkJtvYYp+CCLgnJXwiQB39damAO7WMdKWlIhmYTfHjwSbQeUK/20vY154mwezd9HflVFM1wVSw==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-linux-arm": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.33.5.tgz", + "integrity": "sha512-JTS1eldqZbJxjvKaAkxhZmBqPRGmxgu+qFKSInv8moZ2AmT5Yib3EQ1c6gp493HvrvV8QgdOXdyaIBrhvFhBMQ==", + "cpu": [ + "arm" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm": "1.0.5" + } + }, + "node_modules/@img/sharp-linux-arm64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.33.5.tgz", + "integrity": "sha512-JMVv+AMRyGOHtO1RFBiJy/MBsgz0x4AWrT6QoEVVTyh1E39TrCUpTRI7mx9VksGX4awWASxqCYLCV4wBZHAYxA==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm64": "1.0.4" + } + }, + "node_modules/@img/sharp-linux-s390x": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.33.5.tgz", + "integrity": "sha512-y/5PCd+mP4CA/sPDKl2961b+C9d+vPAveS33s6Z3zfASk2j5upL6fXVPZi7ztePZ5CuH+1kW8JtvxgbuXHRa4Q==", + "cpu": [ + "s390x" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-s390x": "1.0.4" + } + }, + "node_modules/@img/sharp-linux-x64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.33.5.tgz", + "integrity": "sha512-opC+Ok5pRNAzuvq1AG0ar+1owsu842/Ab+4qvU879ippJBHvyY5n2mxF1izXqkPYlGuP/M556uh53jRLJmzTWA==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-x64": "1.0.4" + } + }, + "node_modules/@img/sharp-linuxmusl-arm64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.33.5.tgz", + "integrity": "sha512-XrHMZwGQGvJg2V/oRSUfSAfjfPxO+4DkiRh6p2AFjLQztWUuY/o8Mq0eMQVIY7HJ1CDQUJlxGGZRw1a5bqmd1g==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-arm64": "1.0.4" + } + }, + "node_modules/@img/sharp-linuxmusl-x64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.33.5.tgz", + "integrity": "sha512-WT+d/cgqKkkKySYmqoZ8y3pxx7lx9vVejxW/W4DOFMYVSkErR+w7mf2u8m/y4+xHe7yY9DAXQMWQhpnMuFfScw==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-x64": "1.0.4" + } + }, + "node_modules/@img/sharp-wasm32": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.33.5.tgz", + "integrity": "sha512-ykUW4LVGaMcU9lu9thv85CbRMAwfeadCJHRsg2GmeRa/cJxsVY9Rbd57JcMxBkKHag5U/x7TSBpScF4U8ElVzg==", + "cpu": [ + "wasm32" + ], + "optional": true, + "dependencies": { + "@emnapi/runtime": "^1.2.0" + }, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-ia32": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.33.5.tgz", + "integrity": "sha512-T36PblLaTwuVJ/zw/LaH0PdZkRz5rd3SmMHX8GSmR7vtNSP5Z6bQkExdSK7xGWyxLw4sUknBuugTelgw2faBbQ==", + "cpu": [ + "ia32" + ], + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-x64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.33.5.tgz", + "integrity": "sha512-MpY/o8/8kj+EcnxwvrP4aTJSWw/aZ7JIGR4aBeZkZw5B7/Jn+tY9/VNwtcoGmdT7GfggGIU4kygOMSbYnOrAbg==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, "node_modules/@isaacs/cliui": { "version": "8.0.2", "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", @@ -14206,7 +14596,6 @@ "version": "4.2.3", "resolved": "https://registry.npmjs.org/color/-/color-4.2.3.tgz", "integrity": "sha512-1rXeuUUiGGrykh+CeBdu5Ie7OJwinCgQY0bc7GCRxy5xVHy+moaqkpL/jqQq0MtQOeYcrqEz4abc5f0KtU7W4A==", - "dev": true, "dependencies": { "color-convert": "^2.0.1", "color-string": "^1.9.0" @@ -14235,7 +14624,6 @@ "version": "1.9.1", "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.9.1.tgz", "integrity": "sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==", - "dev": true, "dependencies": { "color-name": "^1.0.0", "simple-swizzle": "^0.2.2" @@ -21944,7 +22332,6 @@ "version": "0.2.2", "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz", "integrity": "sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg==", - "dev": true, "dependencies": { "is-arrayish": "^0.3.1" } @@ -21952,8 +22339,7 @@ "node_modules/simple-swizzle/node_modules/is-arrayish": { "version": "0.3.2", "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz", - "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==", - "dev": true + "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==" }, "node_modules/simple-update-notifier": { "version": "2.0.0", From 03aef74f68a1935c0eb63bc0b67f4ec6d31e6fbb Mon Sep 17 00:00:00 2001 From: Oleksandra Okhotnykova Date: Wed, 25 Sep 2024 21:33:50 +0200 Subject: [PATCH 4/7] OV-423: + constants --- backend/src/common/services/image/constants/constants.ts | 4 ++++ backend/src/common/services/image/image.service.ts | 8 +++----- 2 files changed, 7 insertions(+), 5 deletions(-) create mode 100644 backend/src/common/services/image/constants/constants.ts diff --git a/backend/src/common/services/image/constants/constants.ts b/backend/src/common/services/image/constants/constants.ts new file mode 100644 index 000000000..8ee50667c --- /dev/null +++ b/backend/src/common/services/image/constants/constants.ts @@ -0,0 +1,4 @@ +const PREVIEW_WIDTH = 1920; +const PREVIEW_HEIGHT = 1080; + +export { PREVIEW_HEIGHT, PREVIEW_WIDTH }; diff --git a/backend/src/common/services/image/image.service.ts b/backend/src/common/services/image/image.service.ts index 85b580a2c..fa209230d 100644 --- a/backend/src/common/services/image/image.service.ts +++ b/backend/src/common/services/image/image.service.ts @@ -2,6 +2,7 @@ import sharp from 'sharp'; import { type FileService } from '~/common/services/file/file.service.js'; +import { PREVIEW_HEIGHT, PREVIEW_WIDTH } from './constants/constants.js'; import { type ImageApi } from './image-base.js'; import { type Composition } from './types/types.js'; @@ -54,18 +55,15 @@ class ImageService { avatar: Buffer, background: Buffer, ): Promise { - const finalWidth = 1920; - const finalHeight = 1080; - const resizedBackground = await sharp(background) - .resize(finalWidth, finalHeight, { + .resize(PREVIEW_WIDTH, PREVIEW_HEIGHT, { fit: 'cover', position: 'center', }) .toBuffer(); const resizedAvatar = await sharp(avatar) - .resize(finalWidth, finalHeight, { + .resize(PREVIEW_WIDTH, PREVIEW_HEIGHT, { fit: 'inside', position: 'bottom', }) From 0dcaca92cb0841aeb0a13e11f758d012e6053dae Mon Sep 17 00:00:00 2001 From: Oleksandra Okhotnykova Date: Wed, 25 Sep 2024 21:54:26 +0200 Subject: [PATCH 5/7] OV-423: * support background color for preview --- .../common/services/image/image.service.ts | 55 +++++++++++++++---- 1 file changed, 43 insertions(+), 12 deletions(-) diff --git a/backend/src/common/services/image/image.service.ts b/backend/src/common/services/image/image.service.ts index fa209230d..a84c3f745 100644 --- a/backend/src/common/services/image/image.service.ts +++ b/backend/src/common/services/image/image.service.ts @@ -25,32 +25,48 @@ class ImageService { const background = composition.scenes[0]?.background; if (background?.url) { - const avatarImageBuffer = - await this.imageApi.getImageBuffer(avatarImage); const backgroundImageBuffer = await this.imageApi.getImageBuffer( background.url, ); - const previewBuffer = await this.composeImages( - avatarImageBuffer, + return await this.combineAvatarWithBackground( + avatarImage, backgroundImageBuffer, ); - - const fileName = `preview_${Date.now()}.jpg`; - - await this.fileService.uploadFile(previewBuffer, fileName); - - return this.fileService.getCloudFrontFileUrl(fileName); } if (background?.color) { - // TODO: create empty image with bg color - // then combine avatar and this new image + const backgroundColorImageBuffer = + await this.createImageWithBackgroundColor(background.color); + + return await this.combineAvatarWithBackground( + avatarImage, + backgroundColorImageBuffer, + ); } return avatarImage; } + private async combineAvatarWithBackground( + avatarImage: string, + background: Buffer, + ): Promise { + const avatarImageBuffer = + await this.imageApi.getImageBuffer(avatarImage); + + const previewBuffer = await this.composeImages( + avatarImageBuffer, + background, + ); + + const fileName = `preview_${Date.now()}.jpg`; + + await this.fileService.uploadFile(previewBuffer, fileName); + + return this.fileService.getCloudFrontFileUrl(fileName); + } + private async composeImages( avatar: Buffer, background: Buffer, @@ -73,6 +89,21 @@ class ImageService { .composite([{ input: resizedAvatar, blend: 'over' }]) .toBuffer(); } + + private async createImageWithBackgroundColor( + backgroundColor: string, + ): Promise { + return await sharp({ + create: { + width: PREVIEW_WIDTH, + height: PREVIEW_HEIGHT, + channels: 3, + background: backgroundColor, + }, + }) + .toFormat('png') + .toBuffer(); + } } export { ImageService }; From 163b7bf84655992f49f0173a1a9bd8c884d537c1 Mon Sep 17 00:00:00 2001 From: Oleksandra Okhotnykova Date: Wed, 25 Sep 2024 21:58:32 +0200 Subject: [PATCH 6/7] OV-423: * generate preview on update --- backend/src/bundles/videos/video.repository.ts | 12 ++++++++++-- backend/src/bundles/videos/videos.ts | 2 +- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/backend/src/bundles/videos/video.repository.ts b/backend/src/bundles/videos/video.repository.ts index 656af139f..8e56b2730 100644 --- a/backend/src/bundles/videos/video.repository.ts +++ b/backend/src/bundles/videos/video.repository.ts @@ -1,13 +1,19 @@ import { type UpdateVideoRequestDto } from '~/bundles/videos/types/types.js'; import { VideoEntity } from '~/bundles/videos/video.entity.js'; import { type VideoModel } from '~/bundles/videos/video.model.js'; +import { type ImageService } from '~/common/services/image/image.service.js'; import { type Repository } from '~/common/types/types.js'; class VideoRepository implements Repository { private videoModel: typeof VideoModel; + private imageService: ImageService; - public constructor(videoModel: typeof VideoModel) { + public constructor( + videoModel: typeof VideoModel, + imageService: ImageService, + ) { this.videoModel = videoModel; + this.imageService = imageService; } public async findById(id: string): Promise { @@ -58,7 +64,9 @@ class VideoRepository implements Repository { if (payload.composition) { data.composition = payload.composition; - data.previewUrl = payload.composition.scenes[0]?.avatar?.url ?? ''; + data.previewUrl = await this.imageService.generatePreview( + payload.composition, + ); } if (payload.name) { diff --git a/backend/src/bundles/videos/videos.ts b/backend/src/bundles/videos/videos.ts index adb063902..6c9ee5c8a 100644 --- a/backend/src/bundles/videos/videos.ts +++ b/backend/src/bundles/videos/videos.ts @@ -6,7 +6,7 @@ import { VideoModel } from './video.model.js'; import { VideoRepository } from './video.repository.js'; import { VideoService } from './video.service.js'; -const videoRepository = new VideoRepository(VideoModel); +const videoRepository = new VideoRepository(VideoModel, imageService); const videoService = new VideoService( videoRepository, fileService, From 501b0673355a548f4102d3eacf644797d89fe4af Mon Sep 17 00:00:00 2001 From: Oleksandra Okhotnykova Date: Thu, 26 Sep 2024 13:55:06 +0200 Subject: [PATCH 7/7] OV-423: * pass scene instead of composition --- backend/src/bundles/videos/types/types.ts | 1 + backend/src/bundles/videos/video.repository.ts | 7 +++++-- backend/src/bundles/videos/video.service.ts | 3 ++- backend/src/common/services/image/image.service.ts | 8 ++++---- backend/src/common/services/image/types/types.ts | 2 +- 5 files changed, 13 insertions(+), 8 deletions(-) diff --git a/backend/src/bundles/videos/types/types.ts b/backend/src/bundles/videos/types/types.ts index 0e99b9510..86eea3daa 100644 --- a/backend/src/bundles/videos/types/types.ts +++ b/backend/src/bundles/videos/types/types.ts @@ -1,5 +1,6 @@ export { type CreateVideoRequestDto, + type Scene, type UpdateVideoRequestDto, type UserGetCurrentResponseDto, type VideoGetAllItemResponseDto, diff --git a/backend/src/bundles/videos/video.repository.ts b/backend/src/bundles/videos/video.repository.ts index 8e56b2730..ed0b8bc91 100644 --- a/backend/src/bundles/videos/video.repository.ts +++ b/backend/src/bundles/videos/video.repository.ts @@ -1,4 +1,7 @@ -import { type UpdateVideoRequestDto } from '~/bundles/videos/types/types.js'; +import { + type Scene, + type UpdateVideoRequestDto, +} from '~/bundles/videos/types/types.js'; import { VideoEntity } from '~/bundles/videos/video.entity.js'; import { type VideoModel } from '~/bundles/videos/video.model.js'; import { type ImageService } from '~/common/services/image/image.service.js'; @@ -65,7 +68,7 @@ class VideoRepository implements Repository { if (payload.composition) { data.composition = payload.composition; data.previewUrl = await this.imageService.generatePreview( - payload.composition, + payload.composition.scenes[0] as Scene, ); } diff --git a/backend/src/bundles/videos/video.service.ts b/backend/src/bundles/videos/video.service.ts index 1745497db..dc31ddc87 100644 --- a/backend/src/bundles/videos/video.service.ts +++ b/backend/src/bundles/videos/video.service.ts @@ -8,6 +8,7 @@ import { type Service } from '~/common/types/types.js'; import { VideoValidationMessage } from './enums/enums.js'; import { type CreateVideoRequestDto, + type Scene, type UpdateVideoRequestDto, type VideoGetAllItemResponseDto, type VideoGetAllResponseDto, @@ -61,7 +62,7 @@ class VideoService implements Service { payload: CreateVideoRequestDto & { userId: string }, ): Promise { const previewUrl = await this.imageService.generatePreview( - payload.composition, + payload.composition.scenes[0] as Scene, ); const video = await this.videoRepository.create( diff --git a/backend/src/common/services/image/image.service.ts b/backend/src/common/services/image/image.service.ts index a84c3f745..6cb0b6a9b 100644 --- a/backend/src/common/services/image/image.service.ts +++ b/backend/src/common/services/image/image.service.ts @@ -4,7 +4,7 @@ import { type FileService } from '~/common/services/file/file.service.js'; import { PREVIEW_HEIGHT, PREVIEW_WIDTH } from './constants/constants.js'; import { type ImageApi } from './image-base.js'; -import { type Composition } from './types/types.js'; +import { type Scene } from './types/types.js'; type Constructor = { fileService: FileService; @@ -20,9 +20,9 @@ class ImageService { this.imageApi = imageApi; } - public async generatePreview(composition: Composition): Promise { - const avatarImage = composition.scenes[0]?.avatar?.url ?? ''; - const background = composition.scenes[0]?.background; + public async generatePreview(scene: Scene): Promise { + const avatarImage = scene.avatar?.url ?? ''; + const background = scene.background; if (background?.url) { const backgroundImageBuffer = await this.imageApi.getImageBuffer( diff --git a/backend/src/common/services/image/types/types.ts b/backend/src/common/services/image/types/types.ts index 91ee6dd10..23de6349e 100644 --- a/backend/src/common/services/image/types/types.ts +++ b/backend/src/common/services/image/types/types.ts @@ -1 +1 @@ -export { type Composition } from 'shared'; +export { type Scene } from 'shared';