Skip to content

Commit

Permalink
Merge pull request #362 from BinaryStudioAcademy/task/OV-322-add-remo…
Browse files Browse the repository at this point in the history
…tion-render

OV-322: add remotion render
  • Loading branch information
nikita-remeslov authored Sep 20, 2024
2 parents c916093 + 26f60b2 commit c037d1c
Show file tree
Hide file tree
Showing 28 changed files with 10,358 additions and 7,041 deletions.
8 changes: 8 additions & 0 deletions backend/.env.example
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ AWS_SECRET_ACCESS_KEY=see-in-slack
AWS_S3_REGION=eu-north-1
AWS_S3_BUCKET_NAME=bsa-2024-outreachvids
AWS_CLOUDFRONT_DOMAIN_ID=d2tm5q3cg1nlwf
AWS_CLOUDFRONT_DOMAIN_ID_FOR_RENDERED_VIDEO=SOME_SECRET_KEY

#
# OPEN AI
Expand All @@ -48,3 +49,10 @@ ORIGIN=http://localhost:3000
AZURE_SUBSCRIPTION_KEY=see-in-slack
AZURE_SERVICE_REGION=see-in-slack
AZURE_SERVICE_ENDPOINT=see-in-slack

#
# REMOTION
#
REMOTION_LAMBDA_FUNCTION_NAME=SOME_SECRET_KEY
REMOTION_SERVE_URL=SOME_SECRET_KEY
REMOTION_BUCKET_NAME=SOME_SECRET_KEY
1 change: 1 addition & 0 deletions backend/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
"@fastify/static": "7.0.4",
"@fastify/swagger": "8.15.0",
"@fastify/swagger-ui": "4.0.1",
"@remotion/lambda": "4.0.201",
"bcrypt": "5.1.1",
"convict": "6.2.4",
"dotenv": "16.4.5",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@ class AvatarVideoController extends BaseController {
}>,
): Promise<ApiHandlerResponse> {
const userId = (options.user as UserGetCurrentResponseDto).id;

const videoRecord = await this.avatarVideoService.createVideo({
...options.body,
userId,
Expand All @@ -88,7 +89,6 @@ class AvatarVideoController extends BaseController {

await this.avatarVideoService.submitAvatarsConfigs(
avatarsConfigs,
userId,
videoRecord.id,
);

Expand Down
120 changes: 88 additions & 32 deletions backend/src/bundles/avatar-videos/avatar-videos.service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,40 +5,50 @@ import { v4 as uuidv4 } from 'uuid';
import { type AvatarData } from '~/common/services/azure-ai/avatar-video/types/avatar-data.js';
import { type AzureAIService } from '~/common/services/azure-ai/azure-ai.service.js';
import { type FileService } from '~/common/services/file/file.service.js';
import { type RemotionService } from '~/common/services/remotion/remotion.service.js';
import { type RemotionAvatarScene } from '~/common/services/remotion/type/types.js';

import { type VideoService } from '../videos/video.service.js';
import { REQUEST_DELAY } from './constants/constnats.js';
import {
GenerateAvatarResponseStatus,
RenderVideoErrorMessage,
} from './enums/enums.js';
import { distributeScriptsToScenes, getFileName } from './helpers/helpers.js';
import { generatedAvatarToRemotionScene } from './helpers/generated-avatars-to-remotion-scenes.helper.js';
import {
distributeScriptsToScenes,
getFileName,
getTotalDuration,
} from './helpers/helpers.js';
import {
type Composition,
type GeneratedAvatarData,
type RenderAvatarVideoRequestDto,
} from './types/types.js';

type HandleRenderVideoArguments = {
videoRecordId: string;
avatars: {
id: string;
url: string;
}[];
type Constructor = {
azureAIService: AzureAIService;
fileService: FileService;
videoService: VideoService;
remotionService: RemotionService;
};

class AvatarVideoService {
private azureAIService: AzureAIService;
private fileService: FileService;
private videoService: VideoService;
private remotionService: RemotionService;

public constructor(
azureAIService: AzureAIService,
fileService: FileService,
videoService: VideoService,
) {
public constructor({
azureAIService,
fileService,
remotionService,
videoService,
}: Constructor) {
this.azureAIService = azureAIService;
this.fileService = fileService;
this.videoService = videoService;
this.remotionService = remotionService;
}

private async saveAvatarVideo(url: string, id: string): Promise<string> {
Expand Down Expand Up @@ -70,7 +80,6 @@ class AvatarVideoService {

public async submitAvatarsConfigs(
configs: AvatarData[],
userId: string,
recordId: string,
): Promise<string[]> {
try {
Expand All @@ -87,7 +96,7 @@ class AvatarVideoService {
return response.id;
});

this.checkAvatarsProcessing(ids, userId, recordId).catch(() => {
this.checkAvatarsProcessing(ids, recordId).catch(() => {
throw new HttpError({
message: RenderVideoErrorMessage.RENDER_ERROR,
status: HTTPCode.BAD_REQUEST,
Expand All @@ -105,7 +114,6 @@ class AvatarVideoService {

public async checkAvatarsProcessing(
ids: string[],
userId: string,
videoRecordId: string,
): Promise<void> {
try {
Expand All @@ -116,7 +124,7 @@ class AvatarVideoService {
);

await this.handleSuccessfulAvatarsGeneration({
avatars: response,
generatedAvatars: response,
videoRecordId,
});
} catch {
Expand All @@ -127,9 +135,7 @@ class AvatarVideoService {
}
}

private checkAvatarStatus(
id: string,
): Promise<{ id: string; url: string }> {
private checkAvatarStatus(id: string): Promise<GeneratedAvatarData> {
return new Promise((resolve, reject) => {
const interval = setInterval(() => {
this.azureAIService
Expand All @@ -140,7 +146,12 @@ class AvatarVideoService {
GenerateAvatarResponseStatus.SUCCEEDED
) {
clearInterval(interval);
resolve({ id, url: response.outputs.result });
resolve({
id,
url: response.outputs.result,
durationInMilliseconds:
response.properties.durationInMilliseconds,
});
} else if (
response.status ===
GenerateAvatarResponseStatus.FAILED
Expand Down Expand Up @@ -170,21 +181,38 @@ class AvatarVideoService {

private async handleSuccessfulAvatarsGeneration({
videoRecordId,
avatars,
}: HandleRenderVideoArguments): Promise<void> {
// TODO: REPLACE THIS LOGIC WITH RENDER VIDEO
// TODO: NOTIFY USER
const firstAvatarId = avatars[0]?.id;
const url = avatars[0]?.url;
generatedAvatars,
}: {
videoRecordId: string;
generatedAvatars: GeneratedAvatarData[];
}): Promise<void> {
const scenes = generatedAvatarToRemotionScene(generatedAvatars);
const scenesWithSavedAvatars = await this.saveGeneratedAvatar(scenes);

const renderId = await this.remotionService.renderVideo({
scenes: scenesWithSavedAvatars,
totalDurationInFrames: getTotalDuration(scenesWithSavedAvatars),
});

const url =
await this.remotionService.getRemotionRenderProgress(renderId);

if (!firstAvatarId || !url) {
await this.removeGeneratedAvatars(generatedAvatars);
await this.removeAvatarsFromBucket(generatedAvatars);

if (!url) {
return;
}
// TODO: NOTIFY USER
await this.updateVideoRecord(videoRecordId, url);
}

const savedUrl = await this.saveAvatarVideo(url, firstAvatarId);

private async updateVideoRecord(
videoRecordId: string,
videoUrl: string,
): Promise<void> {
const videoData = await this.videoService.update(videoRecordId, {
url: savedUrl,
url: videoUrl,
});

if (!videoData) {
Expand All @@ -193,13 +221,41 @@ class AvatarVideoService {
status: HTTPCode.BAD_REQUEST,
});
}
}

await Promise.all(
avatars.map((avatar) => {
private async removeGeneratedAvatars(
generatedAvatars: GeneratedAvatarData[],
): Promise<unknown> {
return Promise.all(
generatedAvatars.map((avatar) => {
return this.azureAIService.removeAvatarVideo(avatar.id);
}),
);
}

private async saveGeneratedAvatar(
generatedAvatars: RemotionAvatarScene[],
): Promise<RemotionAvatarScene[]> {
return Promise.all(
generatedAvatars.map(async (avatar) => {
return {
durationInFrames: avatar.durationInFrames,
id: avatar.id,
url: await this.saveAvatarVideo(avatar.url, avatar.id),
};
}),
);
}

private async removeAvatarsFromBucket(
generatedAvatars: GeneratedAvatarData[],
): Promise<unknown> {
return Promise.all(
generatedAvatars.map((avatar) => {
return this.fileService.deleteFile(getFileName(avatar.id));
}),
);
}
}

export { AvatarVideoService };
11 changes: 8 additions & 3 deletions backend/src/bundles/avatar-videos/avatar-videos.ts
Original file line number Diff line number Diff line change
@@ -1,15 +1,20 @@
import { logger } from '~/common/logger/logger.js';
import { azureAIService, fileService } from '~/common/services/services.js';
import {
azureAIService,
fileService,
remotionService,
} from '~/common/services/services.js';

import { videoService } from '../videos/videos.js';
import { AvatarVideoController } from './avatar-videos.controller.js';
import { AvatarVideoService } from './avatar-videos.service.js';

const avatarVideoService = new AvatarVideoService(
const avatarVideoService = new AvatarVideoService({
azureAIService,
fileService,
videoService,
);
remotionService,
});

const avatarVideoController = new AvatarVideoController(
logger,
Expand Down
1 change: 1 addition & 0 deletions backend/src/bundles/avatar-videos/constants/constnats.ts
Original file line number Diff line number Diff line change
@@ -1 +1,2 @@
export { FPS } from './fps.js';
export { REQUEST_DELAY } from './request-delay.constant.js';
3 changes: 3 additions & 0 deletions backend/src/bundles/avatar-videos/constants/fps.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
const FPS = 30;

export { FPS };
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
import { type RemotionAvatarScene } from '~/common/services/remotion/type/types.js';

import { FPS } from '../constants/fps.js';
import { type GeneratedAvatarData } from '../types/types.js';

const generatedAvatarToRemotionScene = (
generatedAvatars: GeneratedAvatarData[],
): RemotionAvatarScene[] => {
return generatedAvatars.map((avatar) => {
return {
id: avatar.id,
url: avatar.url,
durationInFrames: Math.round(
(avatar.durationInMilliseconds / 1000) * FPS,
),
};
});
};

export { generatedAvatarToRemotionScene };
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
import { type RemotionAvatarScene } from '~/common/services/remotion/type/types.js';

const getTotalDuration = (scenes: RemotionAvatarScene[]): number => {
return scenes.reduce((sum, scene) => sum + scene.durationInFrames, 0);
};

export { getTotalDuration };
1 change: 1 addition & 0 deletions backend/src/bundles/avatar-videos/helpers/helpers.ts
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
export { distributeScriptsToScenes } from './distribute-scripts-to-scenes.helper.js';
export { getFileName } from './get-file-name.helper.js';
export { getTotalDuration } from './get-total-duration.helper.js';
Loading

0 comments on commit c037d1c

Please sign in to comment.