From f428c7d8c6c26c350ba89e39a84eabb05784b0ed Mon Sep 17 00:00:00 2001 From: Gene Kogan Date: Mon, 15 Jan 2024 22:04:53 -0800 Subject: [PATCH 1/2] refactor and add stories --- README.md | 20 +- app/animations/__init__.py | 3 + app/animations/animation.py | 48 +++ app/animations/dialogue.py | 65 ++++ app/animations/monologue.py | 17 + app/animations/story.py | 70 ++++ app/character.py | 3 +- app/{routers => }/generator.py | 49 ++- app/models.py | 86 ++--- app/plugins/elevenlabs.py | 21 +- app/plugins/replicate.py | 12 +- .../cinema/screenwriter_system.txt | 6 +- app/prompt_templates/moderation.txt | 5 +- app/routers/dags.py | 111 ------- app/routers/story.py | 304 ------------------ app/scenarios/__init__.py | 4 + app/{routers => scenarios}/chat.py | 16 +- .../scenario.py => scenarios/dialogue.py} | 49 +-- app/scenarios/monologue.py | 25 ++ app/scenarios/story.py | 39 +++ app/{routers => scenarios}/tasks.py | 29 +- app/server.py | 43 ++- app/utils.py | 90 ++++-- tests/{test_dags.py => test_animation.py} | 12 +- tests/test_chat.py | 2 +- tests/test_generator.py | 61 ++++ tests/test_scenarios.py | 4 +- tests/test_stories.py | 22 +- 28 files changed, 570 insertions(+), 646 deletions(-) create mode 100644 app/animations/__init__.py create mode 100644 app/animations/animation.py create mode 100644 app/animations/dialogue.py create mode 100644 app/animations/monologue.py create mode 100644 app/animations/story.py rename app/{routers => }/generator.py (60%) delete mode 100644 app/routers/dags.py delete mode 100644 app/routers/story.py create mode 100644 app/scenarios/__init__.py rename app/{routers => scenarios}/chat.py (76%) rename app/{routers/scenario.py => scenarios/dialogue.py} (51%) create mode 100644 app/scenarios/monologue.py create mode 100644 app/scenarios/story.py rename app/{routers => scenarios}/tasks.py (59%) rename tests/{test_dags.py => test_animation.py} (60%) create mode 100644 tests/test_generator.py diff --git a/README.md b/README.md index 09cd548..8265ed2 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,24 @@ Run `rye run uvicorn app.server:app --reload` +Send a command to the server + + + curl -X POST 'http://localhost:5050/tasks/create' \ + -H 'x-api-key: YOUR_API_KEY' \ + -H 'x-api-secret: YOUR_API_SECRET' \ + -H 'Content-Type: application/json' \ + -d '{ + "generatorName": "monologue", + "config": { + "characterId": "6577e5d5c77b37642c252423", + "prompt": "who are you?" + } + }' + + + Tests - `rye run pytest -s tests` \ No newline at end of file + `rye run pytest -s tests` + diff --git a/app/animations/__init__.py b/app/animations/__init__.py new file mode 100644 index 0000000..b2c3517 --- /dev/null +++ b/app/animations/__init__.py @@ -0,0 +1,3 @@ +from .monologue import animated_monologue +from .dialogue import animated_dialogue +from .story import animated_story \ No newline at end of file diff --git a/app/animations/animation.py b/app/animations/animation.py new file mode 100644 index 0000000..f93dee3 --- /dev/null +++ b/app/animations/animation.py @@ -0,0 +1,48 @@ +from typing import Optional + +from ..plugins import replicate, elevenlabs, s3 +from ..character import EdenCharacter +from ..utils import combine_speech_video + + +def talking_head( + character: EdenCharacter, + text: str, + width: Optional[int] = None, + height: Optional[int] = None +) -> str: + audio_bytes = elevenlabs.tts( + text, + voice=character.voice + ) + audio_url = s3.upload(audio_bytes, "mp3") + output_url, thumbnail_url = replicate.wav2lip( + face_url=character.image, + speech_url=audio_url, + gfpgan=False, + gfpgan_upscale=1, + width=width, + height=height, + ) + return output_url, thumbnail_url + + +def screenplay_clip( + character: EdenCharacter, + speech: str, + image_text: str, + width: Optional[int] = None, + height: Optional[int] = None +) -> str: + audio_bytes = elevenlabs.tts( + speech, + voice=character.voice + ) + audio_url = s3.upload(audio_bytes, "mp3") + video_url, thumbnail_url = replicate.txt2vid( + interpolation_texts=[image_text], + width=width, + height=height, + ) + output_filename = combine_speech_video(audio_url, video_url) + return output_filename, thumbnail_url \ No newline at end of file diff --git a/app/animations/dialogue.py b/app/animations/dialogue.py new file mode 100644 index 0000000..1762474 --- /dev/null +++ b/app/animations/dialogue.py @@ -0,0 +1,65 @@ +import os +import requests +import tempfile +from concurrent.futures import ThreadPoolExecutor, as_completed + +from .. import utils +from .animation import talking_head +from ..plugins import replicate, elevenlabs, s3 +from ..character import EdenCharacter +from ..scenarios import dialogue +from ..models import DialogueRequest + +MAX_PIXELS = 1024 * 1024 +MAX_WORKERS = 3 + + +def animated_dialogue(request: DialogueRequest): + result = dialogue(request) + characters = { + character_id: EdenCharacter(character_id) + for character_id in request.character_ids + } + images = [ + characters[character_id].image + for character_id in request.character_ids + ] + width, height = utils.calculate_target_dimensions(images, MAX_PIXELS) + + def run_talking_head_segment(message): + character = characters[message["character_id"]] + output, _ = talking_head( + character, + message["message"], + width, + height + ) + with tempfile.NamedTemporaryFile(delete=False, suffix=".mp4") as temp_file: + response = requests.get(output, stream=True) + response.raise_for_status() + for chunk in response.iter_content(chunk_size=8192): + temp_file.write(chunk) + temp_file.flush() + return temp_file.name + + video_files = utils.process_in_parallel( + result.dialogue, + run_talking_head_segment, + max_workers=MAX_WORKERS + ) + + # concatenate the final video clips + with tempfile.NamedTemporaryFile(delete=False, suffix=".mp4") as temp_output_file: + utils.concatenate_videos(video_files, temp_output_file.name) + with open(temp_output_file.name, 'rb') as f: + video_bytes = f.read() + output_url = s3.upload(video_bytes, "mp4") + os.remove(temp_output_file.name) + for video_file in video_files: + os.remove(video_file) + + # generate thumbnail + thumbnail = utils.create_dialogue_thumbnail(*images, width, height) + thumbnail_url = s3.upload(thumbnail, "webp") + + return output_url, thumbnail_url \ No newline at end of file diff --git a/app/animations/monologue.py b/app/animations/monologue.py new file mode 100644 index 0000000..ee9eb23 --- /dev/null +++ b/app/animations/monologue.py @@ -0,0 +1,17 @@ +import requests + +from .animation import talking_head +from ..plugins import replicate, elevenlabs, s3 +from ..character import EdenCharacter +from ..scenarios import monologue +from ..models import MonologueRequest + + +def animated_monologue(request: MonologueRequest): + character = EdenCharacter(request.character_id) + #thumbnail_url = character.image + result = monologue(request) + output, thumbnail_url = talking_head(character, result.monologue) + output_bytes = requests.get(output).content + output_url = s3.upload(output_bytes, "mp4") + return output_url, thumbnail_url diff --git a/app/animations/story.py b/app/animations/story.py new file mode 100644 index 0000000..0c64cb9 --- /dev/null +++ b/app/animations/story.py @@ -0,0 +1,70 @@ +import os +import requests +import tempfile + +from .. import utils +from ..plugins import replicate, elevenlabs, s3 +from ..character import EdenCharacter +from ..scenarios import story +from ..models import StoryRequest, StoryResult +from .animation import screenplay_clip + +MAX_PIXELS = 1024 * 1024 +MAX_WORKERS = 3 + + +def animated_story(request: StoryRequest): + screenplay = story(request) + + characters = { + character_id: EdenCharacter(character_id) + for character_id in request.character_ids + [request.narrator_id] + } + + character_name_lookup = { + character.name: character_id + for character_id, character in characters.items() + } + + images = [ + characters[character_id].image + for character_id in request.character_ids + ] + + width, height = utils.calculate_target_dimensions(images, MAX_PIXELS) + + def run_story_segment(clip): + if clip['voiceover'] == 'character': + character_id = character_name_lookup[clip['character']] + character = characters[character_id] + else: + character = characters[request.narrator_id] + output_filename, thumbnail_url = screenplay_clip( + character, + clip['speech'], + clip['image_description'], + width, + height + ) + return output_filename, thumbnail_url + + results = utils.process_in_parallel( + screenplay['clips'], + run_story_segment, + max_workers=MAX_WORKERS + ) + + video_files = [video_file for video_file, thumbnail in results] + thumbnail_url = results[0][1] + + with tempfile.NamedTemporaryFile(delete=True, suffix=".mp4") as temp_output_file: + utils.concatenate_videos(video_files, temp_output_file.name) + with open(temp_output_file.name, 'rb') as f: + video_bytes = f.read() + output_url = s3.upload(video_bytes, "mp4") + + # clean up clips + for video_file in video_files: + os.remove(video_file) + + return output_url, thumbnail_url diff --git a/app/character.py b/app/character.py index 6fa487b..adb8a40 100644 --- a/app/character.py +++ b/app/character.py @@ -7,9 +7,10 @@ from pydantic import Field, BaseModel, ValidationError from .mongo import get_character_data -from .routers.tasks import summary, SummaryRequest +from .scenarios.tasks import summary from .llm import LLM from .llm.models import ChatMessage +from .models import SummaryRequest from .prompt_templates.assistant import ( identity_template, reply_template, diff --git a/app/routers/generator.py b/app/generator.py similarity index 60% rename from app/routers/generator.py rename to app/generator.py index 084a703..cf1a0ef 100644 --- a/app/routers/generator.py +++ b/app/generator.py @@ -1,38 +1,28 @@ -from typing import Optional, List -from fastapi import APIRouter, Request, BackgroundTasks -from fastapi.responses import JSONResponse -from pydantic import BaseModel +from fastapi import BackgroundTasks import uuid -import traceback - import requests -from .dags import monologue_dag, dialogue_dag -from .story import cinema -from ..mongo import get_character_data -from ..llm import LLM -from ..prompt_templates import monologue_template, dialogue_template - -from ..models import MonologueRequest, MonologueOutput -from ..models import DialogueRequest, DialogueOutput, CinemaRequest -from ..models import TaskRequest, TaskUpdate, TaskOutput - -router = APIRouter() +from .animations import animated_monologue, animated_dialogue, animated_story +from .models import MonologueRequest, MonologueResult +from .models import DialogueRequest, DialogueResult, StoryRequest +from .models import TaskRequest, TaskUpdate, TaskResult -def process_task(task_id: str, request: TaskRequest, task_type: str): +def process_task(task_id: str, request: TaskRequest): print("config", request.config) + task_type = request.generatorName webhook_url = request.webhookUrl update = TaskUpdate( id=task_id, - output=TaskOutput(progress=0), + output=TaskResult(progress=0), status="processing", error=None, ) - requests.post(webhook_url, json=update.dict()) + if webhook_url: + requests.post(webhook_url, json=update.dict()) try: if task_type == "monologue": @@ -42,7 +32,7 @@ def process_task(task_id: str, request: TaskRequest, task_type: str): character_id=character_id, prompt=prompt, ) - output_url, thumbnail_url = monologue_dag(task_req) + output_url, thumbnail_url = animated_monologue(task_req) elif task_type == "dialogue": character_ids = request.config.get("characterIds") @@ -51,19 +41,18 @@ def process_task(task_id: str, request: TaskRequest, task_type: str): character_ids=character_ids, prompt=prompt, ) - output_url, thumbnail_url = dialogue_dag(task_req) + output_url, thumbnail_url = animated_dialogue(task_req) elif task_type == "story": character_ids = request.config.get("characterIds") prompt = request.config.get("prompt") - task_req = CinemaRequest( + task_req = StoryRequest( character_ids=character_ids, prompt=prompt, ) - output_url = cinema(task_req) - thumbnail_url = "https://edenartlab-prod-data.s3.us-east-1.amazonaws.com/e745b8c200bb10efe744caa800c7c7f89c3ae05c39fa4aa0595bdd138117c592.png" + output_url, thumbnail_url = animated_story(task_req) - output = TaskOutput( + output = TaskResult( files=[output_url], thumbnails=[thumbnail_url], name=prompt, @@ -75,7 +64,7 @@ def process_task(task_id: str, request: TaskRequest, task_type: str): error = None except Exception as e: - output = TaskOutput( + output = TaskResult( files=[], thumbnails=[], name=prompt, @@ -94,12 +83,12 @@ def process_task(task_id: str, request: TaskRequest, task_type: str): ) print("update", update.dict()) - requests.post(webhook_url, json=update.dict()) + if webhook_url: + requests.post(webhook_url, json=update.dict()) -@router.post("/tasks/create") async def generate_task(background_tasks: BackgroundTasks, request: TaskRequest): task_id = str(uuid.uuid4()) if request.generatorName in ["monologue", "dialogue", "story"]: - background_tasks.add_task(process_task, task_id, request, request.generatorName) + background_tasks.add_task(process_task, task_id, request) return {"id": task_id} diff --git a/app/models.py b/app/models.py index e077e23..d183971 100644 --- a/app/models.py +++ b/app/models.py @@ -1,16 +1,14 @@ +from enum import Enum from typing import Optional, List from pydantic import BaseModel, Field -from .character import Character - class TaskRequest(BaseModel): generatorName: str config: dict = {} - webhookUrl: str - + webhookUrl: Optional[str] = None -class TaskOutput(BaseModel): +class TaskResult(BaseModel): files: List[str] = [] thumbnails: List[str] = [] name: str = "" @@ -18,11 +16,10 @@ class TaskOutput(BaseModel): progress: int = 0 isFinal: bool = False - class TaskUpdate(BaseModel): id: str status: str - output: TaskOutput + output: TaskResult error: Optional[str] = None @@ -32,8 +29,7 @@ class MonologueRequest(BaseModel): model: str = "gpt-4-1106-preview" params: dict = {} - -class MonologueOutput(BaseModel): +class MonologueResult(BaseModel): monologue: str @@ -43,37 +39,51 @@ class DialogueRequest(BaseModel): model: str = "gpt-4-1106-preview" params: dict = {} - -class DialogueOutput(BaseModel): +class DialogueResult(BaseModel): dialogue: List[dict] -class CinemaRequest(BaseModel): +class StoryRequest(BaseModel): character_ids: List[str] prompt: str + narrator_id: str = "6596129023f1c4b471dbb94a" model: str = "gpt-4-1106-preview" params: dict = {} -class CinemaResult(BaseModel): - stills: List[str] +class StoryVoiceoverMode(Enum): + character = 'character' + narrator = 'narrator' + #none = 'none' + +class StoryClip(BaseModel): + """ + A single clip in a screenplay sequence + """ + voiceover: StoryVoiceoverMode = Field(description="Voiceover mode for clip") + character: Optional[str] = Field(description="Character name if voiceover mode is character, otherwise null") + speech: str = Field(description="Spoken text for clip") + image_description: str = Field(description="Image content for clip") + +class StoryResult(BaseModel): + """ + A screenplay consisting of a sequence of clips + """ + clips: List[StoryClip] = Field(description="Clips in the sequence") class ChatRequest(BaseModel): """ A chat request to an EdenCharacter """ - character_id: str session_id: str message: str attachments: Optional[List[str]] = Field(None) - class ChatTestRequest(BaseModel): """ A chat request to a Character """ - name: str identity: str knowledge_summary: Optional[str] = Field(None) @@ -81,39 +91,33 @@ class ChatTestRequest(BaseModel): message: str attachments: Optional[List[str]] = Field(None) - class CharacterOutput(BaseModel): """ Output of chat message from a Character """ - message: str = Field(description="Text response from Eden") config: Optional[dict] = Field(description="Config for Eden generator") -# class Character(BaseModel): -# name: str -# description: str -# knowledge_summary: Optional[str] = None -# knowledge: Optional[str] = None -# voice: Optional[str] = None -# image: Optional[str] = None - - -# export interface CharacterSchema extends VisibilitySchema { -# user: UserDocument -# name: string -# slug: string -# greeting?: string -# dialogue?: ChatSchema[] -# logosData?: LogosData -# image?: string -# voice?: string -# creationCount?: number -# createdAt?: Date -# updatedAt?: Date -# } +class SummaryRequest(BaseModel): + text: str + model: str = "gpt-4-1106-preview" + +class SummaryResult(BaseModel): + summary: str + +class ModerationRequest(BaseModel): + text: str + model: str = "gpt-3.5-turbo" +class ModerationResult(BaseModel): + """ + Moderation scores for each category + """ + nsfw: int = Field(description="Sexually explicit or nudity") + gore: int = Field(description="Violence or gore") + hate: int = Field(description="Hate, abusive or toxic speech") + spam: int = Field(description="Spam, scam, or deceptive content") # class CharacterChatMessage(BaseModel): # character: Character diff --git a/app/plugins/elevenlabs.py b/app/plugins/elevenlabs.py index 6d999ac..edabaa4 100644 --- a/app/plugins/elevenlabs.py +++ b/app/plugins/elevenlabs.py @@ -1,6 +1,25 @@ import os -from elevenlabs import * +from elevenlabs import generate, set_api_key +from ..utils import exponential_backoff ELEVENLABS_API_KEY = os.environ.get("ELEVENLABS_API_KEY") set_api_key(ELEVENLABS_API_KEY) + + +def tts( + text: str, + voice: str, + max_attempts: int = 6, + initial_delay: int = 5, +): + def generate_with_params(): + return generate(text, voice=voice) + + audio_bytes = exponential_backoff( + generate_with_params, + max_attempts=max_attempts, + initial_delay=initial_delay + ) + + return audio_bytes \ No newline at end of file diff --git a/app/plugins/replicate.py b/app/plugins/replicate.py index f656174..1c71c94 100644 --- a/app/plugins/replicate.py +++ b/app/plugins/replicate.py @@ -49,6 +49,7 @@ def submit_task( ) return prediction + def wav2lip( face_url: str, speech_url: str, @@ -76,8 +77,9 @@ def wav2lip( output = list(output) output_url = output[0]["files"][0] + thumbnail_url = output[0]["thumbnails"][0] - return output_url + return output_url, thumbnail_url def sdxl( @@ -99,8 +101,9 @@ def sdxl( output = list(output) output_url = output[0]["files"][0] + thumbnail_url = output[0]["thumbnails"][0] - return output_url + return output_url, thumbnail_url def txt2vid( @@ -118,8 +121,6 @@ def txt2vid( "n_frames": 100, } - print("LETS RUN!!!", config) - output = run_task( config, model_name="abraham-ai/eden-comfyui" @@ -127,5 +128,6 @@ def txt2vid( output = list(output) output_url = output[0]["files"][0] + thumbnail_url = output[0]["thumbnails"][0] - return output_url \ No newline at end of file + return output_url, thumbnail_url \ No newline at end of file diff --git a/app/prompt_templates/cinema/screenwriter_system.txt b/app/prompt_templates/cinema/screenwriter_system.txt index d63e763..5706d25 100644 --- a/app/prompt_templates/cinema/screenwriter_system.txt +++ b/app/prompt_templates/cinema/screenwriter_system.txt @@ -2,9 +2,9 @@ You are a critically acclaimed screenwriter who writes incredibly captivating an Users will give you a cast of characters, including their names and biographies, as well as a premise or synopsis for the story. -You will then write a screenplay for a film based on the information provided. Intersperse the screenplay with descriptions of events, dialogues, and character monologues. It will come as a sequence of clips. A clip contains the following: - -voiceover: whether the voiceover is the narrator, the character speaking, or none at all (just sound effects) +You will then write a screenplay for a film based on the information provided. Intersperse the screenplay with descriptions of events and character dialogues. Stucture the screenplay as sequence of clips. A clip contains the following: + +voiceover: whether the voiceover is the narrator or a character speaking character: If voiceover is in character mode, the name of the speaking character speech: If voiceover is in character or narrator mode, the text of the speech image_description: a description of the image content for the clip diff --git a/app/prompt_templates/moderation.txt b/app/prompt_templates/moderation.txt index 4880982..73ee2b0 100644 --- a/app/prompt_templates/moderation.txt +++ b/app/prompt_templates/moderation.txt @@ -1,9 +1,10 @@ You are an assistant who helps in the moderation of content on a generative AI social media platform. You will read texts and prompts of various lengths made by users, and your goal is to assess them for various safety considerations. -When a user gives you a message, you will respond by grading it on a scale of 0 to 10 along 3 dimensions: nsfw, gore/violence, and hate/toxicity. 0 means most safe (not containing any nsfw/gore/hate), and 10 means most unsafe. +When a user gives you a message, you will respond by grading it on a scale of 0 to 10 along 4 dimensions: nsfw, gore/violence, hate/toxicity, and spam/scam. 0 means most safe (not containing any nsfw/gore/hate/spam), and 10 means most unsafe. To help you benchmark, here are some guidelines for each: nsfw: 10 means sexually explicit, 5 means suggestive or containing nudity, 0 means no nsfw content at all gore/violence: 10 means extremely violent or gory, the kind that would upset most people, 5 means somewhat violent or suggestive, and may upset some people, 0 means no violence or gore at all. -hate/toxicity: 10 means racism, sexism, homophobia, ethnocentrism, or other forms of objectionable speech, 5 refers to speech that is rude, mean, or offensive, but not necessarily hateful, 0 means no hate or toxicity at all. \ No newline at end of file +hate/toxicity: 10 means racism, sexism, homophobia, ethnocentrism, or other forms of objectionable speech, 5 refers to speech that is rude, mean, or offensive, but not necessarily hateful, 0 means no hate or toxicity at all. +spam: 10 means the message is deceptive, advertising, or scam posting, 0 means the message is not spam. \ No newline at end of file diff --git a/app/routers/dags.py b/app/routers/dags.py deleted file mode 100644 index a6c0692..0000000 --- a/app/routers/dags.py +++ /dev/null @@ -1,111 +0,0 @@ -import os -import uuid -import requests -import elevenlabs -import tempfile -import subprocess -from concurrent.futures import ThreadPoolExecutor, as_completed -from io import BytesIO -from botocore.exceptions import NoCredentialsError -from typing import Optional, List -from fastapi import APIRouter -from pydantic import BaseModel -from concurrent.futures import ThreadPoolExecutor, as_completed - -from .scenario import monologue, dialogue -from ..plugins import replicate, elevenlabs, s3 -from ..character import EdenCharacter -from ..models import MonologueRequest, DialogueRequest -from .. import utils - -MAX_PIXELS = 1024 * 1024 - -router = APIRouter() - -def talking_head( - character: EdenCharacter, - text: str, - width: Optional[int] = None, - height: Optional[int] = None -) -> str: - audio_bytes = elevenlabs.generate( - text, - voice=character.voice - ) - audio_url = s3.upload(audio_bytes, "mp3") - output = replicate.wav2lip( - face_url=character.image, - speech_url=audio_url, - gfpgan=False, - gfpgan_upscale=1, - width=width, - height=height, - ) - return output - - -@router.post("/dags/monologue") -def monologue_dag(request: MonologueRequest): - character = EdenCharacter(request.character_id) - thumbnail_url = character.image - result = monologue(request) - output = talking_head(character, result.monologue) - output_bytes = requests.get(output).content - output_url = s3.upload(output_bytes, "mp4") - return output_url, thumbnail_url - - -@router.post("/dags/dialogue") -def dialogue_dag(request: DialogueRequest): - result = dialogue(request) - characters = { - character_id: EdenCharacter(character_id) - for character_id in request.character_ids - } - images = [ - characters[character_id].image - for character_id in request.character_ids - ] - width, height = utils.calculate_target_dimensions(images, MAX_PIXELS) - - def run_talking_head_segment(message): - character = characters[message["character_id"]] - output = talking_head( - character, - message["message"], - width, - height - ) - with tempfile.NamedTemporaryFile(delete=False, suffix=".mp4") as temp_file: - response = requests.get(output, stream=True) - response.raise_for_status() - for chunk in response.iter_content(chunk_size=8192): - temp_file.write(chunk) - temp_file.flush() - return temp_file.name - - # process talking head segments in parallel - video_files = [] - with ThreadPoolExecutor(max_workers=3) as executor: - future_to_message = { - executor.submit(run_talking_head_segment, message): message - for message in result.dialogue - } - for future in as_completed(future_to_message): - video_files.append(future.result()) - - # concatenate the final video clips - with tempfile.NamedTemporaryFile(delete=False, suffix=".mp4") as temp_output_file: - utils.concatenate_videos(video_files, temp_output_file.name) - with open(temp_output_file.name, 'rb') as f: - video_bytes = f.read() - output_url = s3.upload(video_bytes, "mp4") - os.remove(temp_output_file.name) - for video_file in video_files: - os.remove(video_file) - - # generate thumbnail - thumbnail = utils.create_dialogue_thumbnail(*images, width, height) - thumbnail_url = s3.upload(thumbnail, "webm") - - return output_url, thumbnail_url \ No newline at end of file diff --git a/app/routers/story.py b/app/routers/story.py deleted file mode 100644 index 94316ce..0000000 --- a/app/routers/story.py +++ /dev/null @@ -1,304 +0,0 @@ -from enum import Enum -from typing import Optional, List -from fastapi import APIRouter -from typing import List, Optional -from pydantic import Field, BaseModel, ValidationError - -import requests -import tempfile - -from ..mongo import get_character_data -from ..llm import LLM -from ..utils import calculate_target_dimensions, concatenate_videos, combine_speech_video -from ..models import CinemaRequest, CinemaResult -from ..prompt_templates.cinema import ( - screenwriter_system_template, - screenwriter_prompt_template, - director_template, - cinematographer_template, -) -from ..plugins import replicate, elevenlabs, s3, eden -from ..character import EdenCharacter -from .dags import talking_head - -print("ok 1") -print(talking_head) -MAX_PIXELS = 1024 * 1024 - - -router = APIRouter() - - -import subprocess - -def combine_audio_video22(audio_url: str, video_url: str, output_filename: str): - command = f'ffmpeg -stream_loop -1 -i "{video_url}" -i "{audio_url}" -shortest -y "{output_filename}"' - subprocess.run(command, shell=True, check=True) - -import subprocess -import os -import tempfile - -def combine_audio_video(audio_url: str, video_url: str): - # Create temporary files - audio_file = tempfile.NamedTemporaryFile(suffix=".mp3", delete=True) - video_file = tempfile.NamedTemporaryFile(suffix=".mp4", delete=True) - output_file = tempfile.NamedTemporaryFile(suffix=".mp4", delete=False) - - # Download the files - os.system(f"wget -O {audio_file.name} {audio_url}") - os.system(f"wget -O {video_file.name} {video_url}") - - # Get the duration of the audio file - cmd = f"ffprobe -v error -show_entries format=duration -of default=noprint_wrappers=1:nokey=1 {audio_file.name}" - audio_duration = subprocess.check_output(cmd, shell=True).decode().strip() - - # Loop the video - looped_video = tempfile.NamedTemporaryFile(suffix=".mp4", delete=True) - cmd = f"ffmpeg -y -stream_loop -1 -i {video_file.name} -c copy -t {audio_duration} {looped_video.name}" - subprocess.run(cmd, shell=True) - - # Merge the audio and the looped video - cmd = f"ffmpeg -y -i {looped_video.name} -i {audio_file.name} -c:v copy -c:a aac -strict experimental -shortest {output_file.name}" - subprocess.run(cmd, shell=True) - - # Return the name of the output file - return output_file.name - -class VoiceoverMode(Enum): - character = 'character' - narrator = 'narrator' - none = 'none' - -class Clip(BaseModel): - """ - A single clip in a screenplay sequence - """ - voiceover: VoiceoverMode = Field(description="Voiceover mode for clip") - character: Optional[str] = Field(description="Character name if voiceover mode is character, otherwise null") - speech: str = Field(description="Spoken text for clip") - image_description: str = Field(description="Image for clip, or possibly just of narrator in some scene") - -class Screenplay(BaseModel): - """ - A screenplay consisting of a sequence of clips - """ - clips: List[Clip] = Field(description="Clips in the sequence") - - -@router.post("/story/cinema") -def cinema(request: CinemaRequest): - params = {"temperature": 1.0, "max_tokens": 1000, **request.params} - - print("ok") - -# files = ['/Users/genekogan/eden/edenartlab/logos-svc/thisisthevideo2.mp4', '/var/folders/kj/0_ly06kx2_1cq24q67mnns_00000gn/T/tmpw7jwu6a8.mp4', '/var/folders/kj/0_ly06kx2_1cq24q67mnns_00000gn/T/tmpiei1qihj.mp4', '/var/folders/kj/0_ly06kx2_1cq24q67mnns_00000gn/T/tmpf58ym23z.mp4'] - -# print(files) -# # concatenate_videos(files, "myoutput234.mp4") - - - -# audio_url = "https://edenartlab-dev.s3.amazonaws.com/077bb727a4d0cdb3c8789f2b208e1c36b68858e3a62e6ac8a779adc68b1a52a5.mp3" -# video_url = "https://replicate.delivery/pbxt/0ccL45e9IiW1HaItJZq5mw4dyfXwhWJUWUJegVIMfcV7QOpIB/txt2vid_00001.mp4" - -# output_filename = "thisisthevideo21.mp4" -# output_filename = combine_audio_video(audio_url, video_url) -# print(output_filename) - -# return - - characters = { - character_id: EdenCharacter(character_id) - for character_id in request.character_ids - } - - # hack: add narrator - characters["657926f90a0f725740a93b77"] = EdenCharacter("657926f90a0f725740a93b77") - - - character_name_lookup = { - character.name: character_id - for character_id, character in characters.items() - } - - images = [ - characters[character_id].image - for character_id in request.character_ids - ] - - width, height = calculate_target_dimensions(images, MAX_PIXELS) - - screenwriter = LLM( - model=request.model, - system_message=str(screenwriter_system_template), - params=params, - ) - - character_details = "" - for character_id in request.character_ids: - character = characters[character_id] - character_detail = f"""--- - Name: {character.name} - Description: {character.identity} - - """ - character_details += character_detail - - prompt = screenwriter_prompt_template.substitute( - character_details=character_details, - story=request.prompt, - ) - print("prompt", prompt) - - - story = screenwriter(prompt, output_schema=Screenplay) - # story = {'clips': [{'voiceover': 'narrator', 'character': None, 'speech': 'In the depths of an enigmatic cube-shaped chamber, cloaked in shadows, four strangers awaken. Confusion and urgency are etched upon their faces, for not one recalls how they came to be ensnared within this perplexing contraption.', 'image_description': 'A shadowy, mysterious cube-shaped chamber with eerie lighting.'}, {'voiceover': 'character', 'character': 'Orion Blackwood', 'speech': "As the echoes of our disoriented murmurs fade, I can't help but wonder... Is this yet another grand illusion, or a dire predicament we must escape from?", 'image_description': 'Orion Blackwood looking around, his face a mixture of curiosity and concern.'}, {'voiceover': 'character', 'character': 'Elara Vexley', 'speech': "Illusion or not, we must rely on more than just tricks to navigate our way out. Let's assess our surroundings for any clues. The stars have always guided me; perhaps they hold answers here too.", 'image_description': 'Elara Vexley examining the walls of the chamber with a resolute gaze.'}, {'voiceover': 'character', 'character': 'Dr. Silas Quill', 'speech': 'A curious puzzle indeed. If we are to unlock the secrets of this place, we must think like the cryptographer, deciphering the hidden within the apparent. Let us search for patterns.', 'image_description': 'Dr. Silas Quill scrutinizing the chamber with a contemplative look.'}, {'voiceover': 'character', 'character': 'Kaelin', 'speech': 'The walls may not speak in whispers like the forest, but I trust they conceal vital truths. Our escape may lie in understanding the nature of our confinement.', 'image_description': "Kaelin running her fingers along the chamber's walls, as if communicating with them."}]} - print(story) - - - all_clips = [] - - for clip in story['clips']: - - if clip['voiceover'] == 'character': - character_id = character_name_lookup[clip['character']] - character = characters[character_id] - output = talking_head( - character, - clip['speech'], - width, - height, - ) - - with tempfile.NamedTemporaryFile(delete=False, suffix=".mp4") as temp_file: - response = requests.get(output, stream=True) - response.raise_for_status() - for chunk in response.iter_content(chunk_size=8192): - temp_file.write(chunk) - temp_file.flush() - - #all_clips.append(output) - all_clips.append(temp_file.name) - - elif clip['voiceover'] == 'narrator': - character_id = "657926f90a0f725740a93b77" #character_name_lookup['Orion Blackwood'] - character = characters[character_id] - # output = talking_head( - # character, - # clip['speech'], - # width, - # height, - # ) - # all_clips.append(output) - - - audio_bytes = elevenlabs.generate( - clip['speech'], - voice=character.voice - ) - audio_url = s3.upload(audio_bytes, "mp3") - video_url = replicate.txt2vid( - interpolation_texts=[clip['image_description']], - width=width, - height=height, - ) - print("AUD", audio_url, video_url) - - # audio_url = "https://edenartlab-dev.s3.amazonaws.com/077bb727a4d0cdb3c8789f2b208e1c36b68858e3a62e6ac8a779adc68b1a52a5.mp3" - # video_url = "https://replicate.delivery/pbxt/0ccL45e9IiW1HaItJZq5mw4dyfXwhWJUWUJegVIMfcV7QOpIB/txt2vid_00001.mp4" - - #output_filename = "thisisthevideo.mp4" - output_filename = combine_speech_video(audio_url, video_url) - print(output_filename) - - #output_filename = temp_file.name - print(output_filename) - - all_clips.append(output_filename) - - else: - output = replicate.txt2vid( - interpolation_texts=[clip['image_description']], - width=width, - height=height, - ) - - with tempfile.NamedTemporaryFile(delete=False, suffix=".mp4") as temp_file: - response = requests.get(output, stream=True) - response.raise_for_status() - for chunk in response.iter_content(chunk_size=8192): - temp_file.write(chunk) - temp_file.flush() - - #all_clips.append(output) - all_clips.append(temp_file.name) - - - # print("THE URLS") - # all_clips = ['/Users/genekogan/eden/edenartlab/logos-svc/thisisthevideo.mp4', '/var/folders/kj/0_ly06kx2_1cq24q67mnns_00000gn/T/tmpw7jwu6a8.mp4', '/var/folders/kj/0_ly06kx2_1cq24q67mnns_00000gn/T/tmpiei1qihj.mp4', '/var/folders/kj/0_ly06kx2_1cq24q67mnns_00000gn/T/tmpf58ym23z.mp4'] - - print(all_clips) - with tempfile.NamedTemporaryFile(delete=False, suffix=".mp4") as temp_output_file: - concatenate_videos(all_clips, temp_output_file.name) - with open(temp_output_file.name, 'rb') as f: - video_bytes = f.read() - output_url = s3.upload(video_bytes, "mp4") - print("OUTPUT", output_url) - - # copy temp_output_file.name to output file in root dir - filename = temp_output_file.name.split("/")[-1] - print("FILENAME", filename) - os.system(f"cp {temp_output_file.name} new_{filename}") - - - - - os.remove(temp_output_file.name) - for clip in all_clips: - os.remove(clip) - # screenwriter_message = str(screenwriter_template) - # director_message = str(director_template) - # cinematographer_message = str(cinematographer_template) - - # screenwriter = LLM( - # model=request.model, - # system_message=screenwriter_message, - # params=params, - # ) - # director = LLM( - # model=request.model, - # system_message=director_message, - # params=params, - # ) - # cinematographer = LLM( - # model=request.model, - # system_message=cinematographer_message, - # params=params, - # ) - - # story = screenwriter(request.prompt) - # stills = director(story) - # design = cinematographer(stills) - - # stills = text_to_lines(stills) - - # result = CinemaResult(stills=stills) - - #return CinemaResult(stills=["TBD"]) - return output_url - - -class ComicRequest(BaseModel): - prompt: str - model: str = "gpt-4-1106-preview" - params: dict = {} - -class ComicResult(BaseModel): - comic: List[str] - -@router.post("/story/comic") -def comic(request: ComicRequest): - print("TBD comic") - result = ComicResult(comic=["TBD"]) - return result diff --git a/app/scenarios/__init__.py b/app/scenarios/__init__.py new file mode 100644 index 0000000..aef9181 --- /dev/null +++ b/app/scenarios/__init__.py @@ -0,0 +1,4 @@ +from .monologue import monologue +from .dialogue import dialogue +from .story import story +from . import chat \ No newline at end of file diff --git a/app/routers/chat.py b/app/scenarios/chat.py similarity index 76% rename from app/routers/chat.py rename to app/scenarios/chat.py index 19915c2..6d3860b 100644 --- a/app/routers/chat.py +++ b/app/scenarios/chat.py @@ -1,14 +1,9 @@ -from fastapi import APIRouter -from typing import Optional, List -from pydantic import BaseModel, Field - from ..character import Character, EdenCharacter from ..models import ChatRequest, ChatTestRequest, CharacterOutput -router = APIRouter() - characters = {} +print("do chat") def get_character(character_id: str): if character_id not in characters: @@ -17,8 +12,7 @@ def get_character(character_id: str): return character -@router.post("/chat/think") -async def think(request: ChatRequest) -> bool: +def think(request: ChatRequest) -> bool: character = get_character(request.character_id) character.sync() message = { @@ -29,8 +23,7 @@ async def think(request: ChatRequest) -> bool: return response -@router.post("/chat/speak") -async def speak(request: ChatRequest) -> CharacterOutput: +def speak(request: ChatRequest) -> CharacterOutput: character = get_character(request.character_id) message = { "message": request.message, @@ -40,8 +33,7 @@ async def speak(request: ChatRequest) -> CharacterOutput: return response -@router.post("/chat/test") -async def test(request: ChatTestRequest): +def test(request: ChatTestRequest): character = Character( name=request.name, identity=request.identity, diff --git a/app/routers/scenario.py b/app/scenarios/dialogue.py similarity index 51% rename from app/routers/scenario.py rename to app/scenarios/dialogue.py index 5f55d14..d240d9a 100644 --- a/app/routers/scenario.py +++ b/app/scenarios/dialogue.py @@ -1,58 +1,15 @@ -from typing import Optional, List -from fastapi import APIRouter -from pydantic import BaseModel - from ..mongo import get_character_data from ..llm import LLM from ..prompt_templates import monologue_template, dialogue_template +from ..models import DialogueRequest, DialogueResult -router = APIRouter() - -class MonologueRequest(BaseModel): - character_id: str - prompt: str - model: str = "gpt-4-1106-preview" - params: dict = {} - -class MonologueResult(BaseModel): - monologue: str - -@router.post("/scenarios/monologue") -def monologue(request: MonologueRequest): - params = {"temperature": 1.0, "max_tokens": 1000, **request.params} - - character_data = get_character_data(request.character_id) - name = character_data.get("name") - description = character_data.get("logosData").get("identity") - - system_message = monologue_template.substitute( - name=name, - description=description - ) - - llm = LLM(model=request.model, system_message=system_message, params=params) - monologue_text = llm(request.prompt) - - result = MonologueResult(monologue=monologue_text) - - return result - - -class DialogueRequest(BaseModel): - character_ids: List[str] - prompt: str - model: str = "gpt-4-1106-preview" - params: dict = {} - -class DialogueResult(BaseModel): - dialogue: List[dict] -@router.post("/scenarios/dialogue") def dialogue(request: DialogueRequest): params = {"temperature": 1.0, "max_tokens": 1000, **request.params} characters = [ - get_character_data(character_id) for character_id in request.character_ids + get_character_data(character_id) + for character_id in request.character_ids ] llms = [] diff --git a/app/scenarios/monologue.py b/app/scenarios/monologue.py new file mode 100644 index 0000000..b6ca362 --- /dev/null +++ b/app/scenarios/monologue.py @@ -0,0 +1,25 @@ +from ..mongo import get_character_data +from ..llm import LLM +from ..prompt_templates import monologue_template, dialogue_template +from ..models import MonologueRequest, MonologueResult + + +def monologue(request: MonologueRequest): + params = {"temperature": 1.0, "max_tokens": 1000, **request.params} + + character_data = get_character_data(request.character_id) + name = character_data.get("name") + description = character_data.get("logosData").get("identity") + + system_message = monologue_template.substitute( + name=name, + description=description + ) + + llm = LLM(model=request.model, system_message=system_message, params=params) + monologue_text = llm(request.prompt) + + result = MonologueResult(monologue=monologue_text) + + return result + diff --git a/app/scenarios/story.py b/app/scenarios/story.py new file mode 100644 index 0000000..3deea2d --- /dev/null +++ b/app/scenarios/story.py @@ -0,0 +1,39 @@ +from ..mongo import get_character_data +from ..llm import LLM +from ..models import StoryRequest, StoryResult +from ..character import EdenCharacter +from ..prompt_templates.cinema import ( + screenwriter_system_template, + screenwriter_prompt_template, + #director_template, + #cinematographer_template, +) + + +def story(request: StoryRequest) -> StoryResult: + params = {"temperature": 1.0, "max_tokens": 1000, **request.params} + + screenwriter = LLM( + model=request.model, + system_message=str(screenwriter_system_template), + params=params, + ) + + character_details = "" + for character_id in request.character_ids: + character = EdenCharacter(character_id) + character_detail = f"""--- + Name: {character.name} + Description: {character.identity} + + """ + character_details += character_detail + + prompt = screenwriter_prompt_template.substitute( + character_details=character_details, + story=request.prompt, + ) + + story = screenwriter(prompt, output_schema=StoryResult) + + return story diff --git a/app/routers/tasks.py b/app/scenarios/tasks.py similarity index 59% rename from app/routers/tasks.py rename to app/scenarios/tasks.py index 709ed83..603870f 100644 --- a/app/routers/tasks.py +++ b/app/scenarios/tasks.py @@ -3,18 +3,14 @@ from ..llm import LLM from ..prompt_templates import summary_template, moderation_template +from ..models import ( + SummaryRequest, + SummaryResult, + ModerationRequest, + ModerationResult +) -router = APIRouter() - -class SummaryRequest(BaseModel): - text: str - model: str = "gpt-4-1106-preview" - -class SummaryResult(BaseModel): - summary: str - -@router.post("/tasks/summary") def summary(request: SummaryRequest) -> SummaryResult: params = {"temperature": 0.0, "max_tokens": 1000} @@ -28,19 +24,6 @@ def summary(request: SummaryRequest) -> SummaryResult: return result -class ModerationRequest(BaseModel): - text: str - model: str = "gpt-3.5-turbo" - -class ModerationResult(BaseModel): - """ - Moderation scores for each category - """ - nsfw: int = Field(description="Sexually explicit or nudity") - gore: int = Field(description="Violence or gore") - hate: int = Field(description="Hate, abusive or toxic speech") - -@router.post("/tasks/moderation") def moderation(request: ModerationRequest) -> ModerationResult: params = {"temperature": 0.0, "max_tokens": 1000} diff --git a/app/server.py b/app/server.py index 116521b..9ac0986 100644 --- a/app/server.py +++ b/app/server.py @@ -1,29 +1,50 @@ -from fastapi import FastAPI, Request, BackgroundTasks +from fastapi import FastAPI, Request, APIRouter from fastapi.responses import JSONResponse import traceback import logging -from .routers import scenario, chat, story, dags, generator, tasks -app = FastAPI() +from .scenarios import monologue, dialogue, story, chat, tasks +from .animations import animated_monologue, animated_dialogue, animated_story +from .generator import generate_task + +app = FastAPI() +router = APIRouter() @app.exception_handler(Exception) def exception_handler(request: Request, exc: Exception): - logging.error(f"An error occurred: {exc}\n{traceback.format_exc()}") + logging.error(f"Error: {exc}\n{traceback.format_exc()}") return JSONResponse( status_code=400, - content={"message": f"An error occurred: {exc}"}, + content={"message": f"Error: {exc}"}, ) +# Scenarios +router.add_api_route(path="/scenarios/monologue", endpoint=monologue, methods=["POST"]) +router.add_api_route(path="/scenarios/dialogue", endpoint=dialogue, methods=["POST"]) +router.add_api_route(path="/scenarios/story", endpoint=story, methods=["POST"]) + +# Animations/DAGs +router.add_api_route(path="/animation/monologue", endpoint=animated_monologue, methods=["POST"]) +router.add_api_route(path="/animation/dialogue", endpoint=animated_dialogue, methods=["POST"]) +router.add_api_route(path="/animation/story", endpoint=animated_story, methods=["POST"]) -app.include_router(scenario.router) -app.include_router(chat.router) -app.include_router(story.router) -app.include_router(dags.router) -app.include_router(tasks.router) -app.include_router(generator.router) +# Chat +router.add_api_route(path="/chat/test", endpoint=chat.test, methods=["POST"]) +router.add_api_route(path="/chat/speak", endpoint=chat.speak, methods=["POST"]) +router.add_api_route(path="/chat/think", endpoint=chat.think, methods=["POST"]) +# Generator +router.add_api_route(path="/tasks/create", endpoint=generate_task, methods=["POST"]) + +# Tasks +router.add_api_route(path="/tasks/summary", endpoint=tasks.summary, methods=["POST"]) +router.add_api_route(path="/tasks/moderation", endpoint=tasks.moderation, methods=["POST"]) + + +app.include_router(router) @app.get("/") def main(): return {"status": "running"} + diff --git a/app/utils.py b/app/utils.py index 249a235..259b130 100644 --- a/app/utils.py +++ b/app/utils.py @@ -1,4 +1,5 @@ import tempfile +import time import os import re import traceback @@ -9,15 +10,7 @@ from PIL import Image from io import BytesIO from fastapi import HTTPException - - -def handle_error(e): - error_detail = { - "error_type": type(e).__name__, - "error_message": str(e), - "traceback": traceback.format_exc(), - } - raise HTTPException(status_code=400, detail=error_detail) +from concurrent.futures import ThreadPoolExecutor, as_completed def clean_text(text): @@ -135,7 +128,7 @@ def concatenate_videos(video_files, output_file): for i, video in enumerate(video_files): with tempfile.NamedTemporaryFile(suffix=".mp4", delete=False) as temp: output_video = temp.name - convert_command = ['ffmpeg', '-y', '-i', video, '-r', standard_fps, '-c:a', 'copy', output_video] + convert_command = ['ffmpeg', '-y', '-loglevel', 'panic', '-i', video, '-r', standard_fps, '-c:a', 'copy', output_video] subprocess.run(convert_command) converted_videos.append(output_video) @@ -149,7 +142,7 @@ def concatenate_videos(video_files, output_file): concat_command = ['ffmpeg'] for video in converted_videos: concat_command.extend(['-i', video]) - concat_command.extend(['-y', '-filter_complex', filter_complex, '-map', '[v]', '-map', '[a]', output_file]) + concat_command.extend(['-y', '-loglevel', 'panic','-filter_complex', filter_complex, '-map', '[v]', '-map', '[a]', output_file]) subprocess.run(concat_command) # Step 4: Delete temporary files @@ -162,25 +155,66 @@ def combine_speech_video(audio_url: str, video_url: str): video_file = tempfile.NamedTemporaryFile(suffix=".mp4", delete=True) output_file = tempfile.NamedTemporaryFile(suffix=".mp4", delete=False) - os.system(f"wget -O {audio_file.name} {audio_url}") - os.system(f"wget -O {video_file.name} {video_url}") + subprocess.run(['wget', '-nv', '-O', audio_file.name, audio_url]) + subprocess.run(['wget', '-nv', '-O', video_file.name, video_url]) - # Get the duration of the audio file - cmd = f"ffprobe -v error -show_entries format=duration -of default=noprint_wrappers=1:nokey=1 {audio_file.name}" - audio_duration = subprocess.check_output(cmd, shell=True).decode().strip() + # get the duration of the audio file + cmd = ['ffprobe', '-v', 'error', '-show_entries', 'format=duration', '-of', 'default=noprint_wrappers=1:nokey=1', audio_file.name] + audio_duration = subprocess.check_output(cmd).decode().strip() - # Loop the video + # loop the video looped_video = tempfile.NamedTemporaryFile(suffix=".mp4", delete=True) - cmd = f"ffmpeg -y -stream_loop -1 -i {video_file.name} -c copy -t {audio_duration} {looped_video.name}" - subprocess.run(cmd, shell=True) + cmd = ['ffmpeg', '-y', '-loglevel', 'panic', '-stream_loop', '-1', '-i', video_file.name, '-c', 'copy', '-t', audio_duration, looped_video.name] + subprocess.run(cmd) + + # merge the audio and the looped video + cmd = ['ffmpeg', '-y', '-loglevel', 'panic', '-i', looped_video.name, '-i', audio_file.name, '-c:v', 'copy', '-c:a', 'aac', '-strict', 'experimental', '-shortest', output_file.name] + subprocess.run(cmd) + + return output_file.name + + +def exponential_backoff( + func, + max_attempts=5, + initial_delay=1, +): + delay = initial_delay + for attempt in range(1, max_attempts + 1): + try: + return func() + except Exception as e: + if attempt == max_attempts: + raise e + print(f"Attempt {attempt} failed. Retrying in {delay} seconds...") + time.sleep(delay) + delay = delay * 2 + + +def process_in_parallel( + array, + func, + max_workers=3 +): + with ThreadPoolExecutor(max_workers=max_workers) as executor: + futures = {executor.submit(func, item): index for index, item in enumerate(array)} + results = [None] * len(array) + for future in as_completed(futures): + try: + index = futures[future] + results[index] = future.result() + except Exception as e: + print(f"Task error: {e}") + for f in futures: + f.cancel() + raise e + return results - # Merge the audio and the looped video - cmd = f"ffmpeg -y -i {looped_video.name} -i {audio_file.name} -c:v copy -c:a aac -strict experimental -shortest {output_file.name}" - subprocess.run(cmd, shell=True) - os.remove(audio_file.name) - os.remove(video_file.name) - os.remove(looped_video.name) - - # Return the name of the output file - return output_file.name \ No newline at end of file +def handle_error(e): + error_detail = { + "error_type": type(e).__name__, + "error_message": str(e), + "traceback": traceback.format_exc(), + } + raise HTTPException(status_code=400, detail=error_detail) diff --git a/tests/test_dags.py b/tests/test_animation.py similarity index 60% rename from tests/test_dags.py rename to tests/test_animation.py index a04f2dd..a89ad0c 100644 --- a/tests/test_dags.py +++ b/tests/test_animation.py @@ -4,31 +4,31 @@ client = TestClient(app) -def test_monologue_dag(): +def test_monologue_animation(): """ Test monologue on static character and prompt """ request = { - "character_id": "6577e5d5c77b37642c252423", + "character_id": "6596129023f1c4b471dbb94a", "prompt": "Tell me a story about pizza" } - response = client.post("/dags/monologue", json=request) + response = client.post("/animation/monologue", json=request) print(response.json()) assert response.status_code == 200 -def test_dialogue_dag(): +def test_dialogue_animation(): """ Test monologue on static character and prompt """ request = { - "character_ids": ["658fddadf0e5f5c4a0638a37", "6577e5d5c77b37642c252423"], + "character_ids": ["6596129023f1c4b471dbb94a", "6598e117dd06d165264f2277"], "prompt": "Debate panspermia vs. abiogenesis" } - response = client.post("/dags/dialogue", json=request) + response = client.post("/animation/dialogue", json=request) print(response.json()) assert response.status_code == 200 diff --git a/tests/test_chat.py b/tests/test_chat.py index 07bca20..37c4f41 100644 --- a/tests/test_chat.py +++ b/tests/test_chat.py @@ -9,7 +9,7 @@ def test_edencharacter_chat(): Test chat function on static character and prompt """ request = { - "character_id": "6577e5d5c77b37642c252423", + "character_id": "6596129023f1c4b471dbb94a", "session_id": "default", "message": "I want to make a video which morphs between these two picture ideas I have. I want the video to start like a lush tropical forest with birds and nature and fireflies and stuff. And then it should evolve into a sketchy mountain scene with two moons.", "attachments": [], diff --git a/tests/test_generator.py b/tests/test_generator.py new file mode 100644 index 0000000..518fc3f --- /dev/null +++ b/tests/test_generator.py @@ -0,0 +1,61 @@ +from fastapi.testclient import TestClient +from app.server import app + +client = TestClient(app) + + +def test_monologue(): + """ + Test making a generator request for a monologue + """ + + request = { + "generatorName": "monologue", + "config": { + "characterId": "6596129023f1c4b471dbb94a", + "prompt": "Who are you?", + } + } + + response = client.post("/tasks/create", json=request) + print(response.json()) + + assert response.status_code == 200 + + +def test_dialogue(): + """ + Test making a generator request for a monologue + """ + + request = { + "generatorName": "dialogue", + "config": { + "characterIds": ["6596129023f1c4b471dbb94a", "6598e117dd06d165264f2277"], + "prompt": "Debate whether or not pizza is a vegetable", + } + } + + response = client.post("/tasks/create", json=request) + print(response.json()) + + assert response.status_code == 200 + + +def test_story(): + """ + Test making a generator request for stories + """ + + request = { + "generatorName": "story", + "config": { + "characterIds": ["6596129023f1c4b471dbb94a", "6598e117dd06d165264f2277", "6598e103dd06d165264f2247", "6598ee16dd06d16526503ce7"], + "prompt": "The four protagonists wake up suddenly, inside of a cube-shaped chamber with 6 hatches on each wall. They have no memory of how they got there, and no idea how to escape.", + } + } + + response = client.post("/tasks/create", json=request) + print(response.json()) + + assert response.status_code == 200 diff --git a/tests/test_scenarios.py b/tests/test_scenarios.py index 4406f71..ef3da79 100644 --- a/tests/test_scenarios.py +++ b/tests/test_scenarios.py @@ -9,7 +9,7 @@ def test_monologue(): Test monologue on static character and prompt """ request = { - "character_id": "6577e5d5c77b37642c252423", + "character_id": "6596129023f1c4b471dbb94a", "prompt": "Tell me a story about pizza" } @@ -24,7 +24,7 @@ def test_dialogue(): Test dialogue function on static characters and prompt """ request = { - "character_ids": ["6577e5d5c77b37642c252423", "658fddadf0e5f5c4a0638a37"], + "character_ids": ["6596129023f1c4b471dbb94a", "6598e117dd06d165264f2277"], "prompt": "Debate whether or not pizza is a vegetable" } diff --git a/tests/test_stories.py b/tests/test_stories.py index 77f73e3..1146df3 100644 --- a/tests/test_stories.py +++ b/tests/test_stories.py @@ -4,31 +4,17 @@ client = TestClient(app) -def test_cinema(): +def test_story(): """ - Test cinema story prompt + Test story prompt """ request = { "character_ids": ["6596129023f1c4b471dbb94a", "6598e117dd06d165264f2277", "6598e103dd06d165264f2247", "6598ee16dd06d16526503ce7"], - "prompt": "The protagonists wake up suddenly, inside of a cube-shaped chamber with 6 hatches on each wall. They have no memory of how they got there, and no idea how to escape." + "prompt": "You are members of an elite space exploration team, encountering and interpreting alien forms of art and communication." } - response = client.post("/story/cinema", json=request) + response = client.post("/animation/story", json=request) print(response.json()) assert response.status_code == 200 - - -# def test_comic(): -# """ -# Test comic book story prompt -# """ -# request = { -# "prompt": "Make a comic book." -# } - -# response = client.post("/story/comic", json=request) -# print(response.json()) - -# assert response.status_code == 200 From 826f89de03b5e845217e89832bfa496ca77b2e49 Mon Sep 17 00:00:00 2001 From: Gene Kogan Date: Mon, 15 Jan 2024 22:37:59 -0800 Subject: [PATCH 2/2] more printing --- app/animations/dialogue.py | 2 ++ app/animations/monologue.py | 2 +- app/animations/story.py | 1 + app/scenarios/chat.py | 1 - 4 files changed, 4 insertions(+), 2 deletions(-) diff --git a/app/animations/dialogue.py b/app/animations/dialogue.py index 1762474..a557169 100644 --- a/app/animations/dialogue.py +++ b/app/animations/dialogue.py @@ -16,6 +16,8 @@ def animated_dialogue(request: DialogueRequest): result = dialogue(request) + print(result) + characters = { character_id: EdenCharacter(character_id) for character_id in request.character_ids diff --git a/app/animations/monologue.py b/app/animations/monologue.py index ee9eb23..d8f231c 100644 --- a/app/animations/monologue.py +++ b/app/animations/monologue.py @@ -9,8 +9,8 @@ def animated_monologue(request: MonologueRequest): character = EdenCharacter(request.character_id) - #thumbnail_url = character.image result = monologue(request) + print(result) output, thumbnail_url = talking_head(character, result.monologue) output_bytes = requests.get(output).content output_url = s3.upload(output_bytes, "mp4") diff --git a/app/animations/story.py b/app/animations/story.py index 0c64cb9..cbd55a7 100644 --- a/app/animations/story.py +++ b/app/animations/story.py @@ -15,6 +15,7 @@ def animated_story(request: StoryRequest): screenplay = story(request) + print(screenplay) characters = { character_id: EdenCharacter(character_id) diff --git a/app/scenarios/chat.py b/app/scenarios/chat.py index 6d3860b..9b2aa3f 100644 --- a/app/scenarios/chat.py +++ b/app/scenarios/chat.py @@ -3,7 +3,6 @@ characters = {} -print("do chat") def get_character(character_id: str): if character_id not in characters: