Merge pull request #35 from edenartlab/stage

a lot

app/animations/animation.py

@@ -41,6 +41,7 @@ def talking_head(
     gfpgan: bool = False,
     gfpgan_upscale: int = 1
 ) -> str:
+    print("* talking head: {character.name} says {text}")
     if character.voice:
         voice_id = character.voice
     else:
@@ -52,6 +53,9 @@ def talking_head(
     )
 
     audio_url = s3.upload(audio_bytes, "mp3")
+
+    print(f"run wav2lip on {character.image} and {audio_url}")
+
     output_url, thumbnail_url = replicate.wav2lip(
         face_url=character.image,
         speech_url=audio_url,
@@ -60,6 +64,9 @@ def talking_head(
         width=width,
         height=height,
     )
+
+    print(f"output: {output_url}")
+
     return output_url, thumbnail_url
 
 

app/animations/dialogue.py

@@ -15,8 +15,14 @@
 
 
 def animated_dialogue(request: DialogueRequest, callback=None):
+    print("===== animated_dialogue =====")
+    print(request)
+
     result = dialogue(request)
 
+    print("---")
+    print(result)
+
     if callback:
         callback(progress=0.1)
 
@@ -37,14 +43,17 @@ def animated_dialogue(request: DialogueRequest, callback=None):
     def run_talking_head_segment(message, idx):
         nonlocal progress, progress_increment
         character = characters[message["character_id"]]
+        print(f'run talking head: {message["message"]}')
         output, _ = talking_head(
             character, 
             message["message"], 
             width, 
             height,
             gfpgan=request.gfpgan
         )
+        print(f'output: {output}')
         with tempfile.NamedTemporaryFile(delete=False, suffix=".mp4") as temp_file:
+            print("download:", output)
             response = requests.get(output, stream=True)
             response.raise_for_status()
             for chunk in response.iter_content(chunk_size=8192):
@@ -53,15 +62,22 @@ def run_talking_head_segment(message, idx):
         progress += progress_increment
         if callback:
             callback(progress=progress)
+        print("return temp_file.name:", temp_file.name)
         return temp_file.name
 
+    print("--- run video file tasks ----")
+
     video_files = utils.process_in_parallel(
         result.dialogue,
         run_talking_head_segment,
         max_workers=MAX_WORKERS
     )
 
-    if request.dual_view: 
+    print("--- end video file tasks ----")
+    print(video_files)
+
+    if request.dual_view:
+        print(" -> dual view")
         cropped_images = {}       
         for character in characters:
             temp_file = tempfile.NamedTemporaryFile(suffix=".webp", delete=False)
@@ -79,15 +95,14 @@ def run_talking_head_segment(message, idx):
             left = idx % 2 == 0
             video_file = utils.stitch_image_video(image, video_file, left)
             dual_video_files.append(video_file)
-
         for character in characters:
             os.remove(cropped_images[character])
         for video_file in video_files:
             os.remove(video_file)
-
         video_files = dual_video_files
 
     if request.intro_screen:
+        print(" -> intro screen")
         character_names = [characters[character_id].name for character_id in request.character_ids]
         character_name_str = " and ".join(character_names)
         paragraphs = [
@@ -101,14 +116,15 @@ def run_talking_head_segment(message, idx):
             duration = 8,
             fade_in = 1.5,
             margin_left = 25,
-            margin_right = width + 25
+            margin_right = 25 #width + 25
         )
         video_files = [intro_screen] + video_files
 
     print(video_files)
 
     # concatenate the final video clips
     with tempfile.NamedTemporaryFile(delete=True, suffix=".mp4") as temp_output_file:
+        print("concatenate videos")
         utils.concatenate_videos(video_files, temp_output_file.name)
         with open(temp_output_file.name, 'rb') as f:
             video_bytes = f.read()
@@ -119,8 +135,10 @@ def run_talking_head_segment(message, idx):
         os.remove(video_file)
 
     # generate thumbnail
+    print("make thumbnail")
     thumbnail = utils.create_dialogue_thumbnail(*images, 2*width, height)
     thumbnail_url = s3.upload(thumbnail, "webp")
+    print("finished thumbnail", thumbnail_url)
 
     if callback:
         callback(progress=0.99)

app/animations/monologue.py

@@ -7,6 +7,8 @@
 from ..models import MonologueRequest
 from ..utils import *
 
+MAX_PIXELS = 1024 * 1024
+
 
 def animated_monologue(request: MonologueRequest, callback=None):
     result = monologue(request)
@@ -16,15 +18,19 @@ def animated_monologue(request: MonologueRequest, callback=None):
 
     character = EdenCharacter(request.character_id)
 
+    width, height = calculate_target_dimensions([character.image], MAX_PIXELS)
+
     output, thumbnail_url = talking_head(
         character, 
         result.monologue, 
+        width, 
+        height,
         gfpgan=request.gfpgan
     )
 
     if request.intro_screen:
-        image = download_image(character.image)
-        width, height = image.size
+        #image = download_image(character.image)
+        #width, height = image.size
 
         text = [
             f"{character.name}: {request.prompt}"
@@ -47,12 +53,15 @@ def animated_monologue(request: MonologueRequest, callback=None):
             temp_file.flush()
 
         video_files = [intro_screen, temp_file.name]
-
+        print(video_files)
+        print("cat")
         with tempfile.NamedTemporaryFile(delete=True, suffix=".mp4") as temp_output_file:
+            print(video_files, temp_output_file.name)
             concatenate_videos(video_files, temp_output_file.name)
             with open(temp_output_file.name, 'rb') as f:
                 video_bytes = f.read()
             output_url = s3.upload(video_bytes, "mp4")
+            print(output_url)
 
     else:
         output_bytes = requests.get(output).content

app/animations/reel.py

@@ -8,6 +8,7 @@
 from ..plugins import replicate, elevenlabs, s3
 from ..character import Character, EdenCharacter
 from ..scenarios import reel
+from ..animations.animation import select_random_voice
 from ..models import ReelRequest
 
 
@@ -41,11 +42,11 @@ def animated_reel(request: ReelRequest, callback=None):
     if request.aspect_ratio == "portrait":
         width, height = 1280, 1920
     elif request.aspect_ratio == "landscape":
-        width, height = 1920, 1280
+        width, height = 1920, 1088
     else:
         width, height = 1600, 1600
 
-    min_duration = 20
+    min_duration = 25
     speech_audio = None
     duration = min_duration
 

app/animations/story.py

@@ -1,6 +1,7 @@
 import os
 import requests
 import tempfile
+from pydub import AudioSegment
 
 from .. import utils
 from ..plugins import replicate, elevenlabs, s3
@@ -10,10 +11,15 @@
 from .animation import screenplay_clip
 
 MAX_WORKERS = 3
+INTRO_SCREEN_DURATION = 10
 
 
 def animated_story(request: StoryRequest, callback=None):
     screenplay = story(request)
+
+    # screenplay["clips"] = screenplay["clips"][:3]
+    music_prompt = screenplay.get("music_prompt")
+
     if callback:
         callback(progress=0.1)
 
@@ -66,6 +72,10 @@ def run_story_segment(clip, idx):
         screenplay["clips"], run_story_segment, max_workers=MAX_WORKERS
     )
 
+
+    print(":TH RESULTS")
+    print(results)
+
     video_files = [video_file for video_file, thumbnail in results]
     thumbnail_url = results[0][1]
 
@@ -80,23 +90,63 @@ def run_story_segment(clip, idx):
             paragraphs, 
             width, 
             height, 
-            duration = 10,
+            duration = INTRO_SCREEN_DURATION,
             fade_in = 2,
             margin_left = 25,
             margin_right = 25
         )
         video_files = [intro_screen] + video_files
 
+
+    print("T?he VIDEO FILES")
+    print(video_files)
+
+    audio_file = None
+    if music_prompt:
+        print("get audio")
+        duration = sum([utils.get_video_duration(video_file) for video_file in video_files])
+
+        print("full dur", duration)
+        print("the music prompt", music_prompt)
+        music_url, _ = replicate.audiocraft(
+            prompt=music_prompt,
+            seconds=duration
+        )
+        print(music_url)
+
+        response = requests.get(music_url)
+        response.raise_for_status()
+        audio_file = tempfile.NamedTemporaryFile(delete=False, suffix=".mp3")
+        audio_file.write(response.content)
+        audio_file.flush()
+
+        if request.intro_screen:
+            print("intro screen silence")
+            silence = AudioSegment.silent(duration=INTRO_SCREEN_DURATION * 1000)
+            music = AudioSegment.from_mp3(audio_file.name)
+            music = music - 8
+            music_with_silence = silence + music
+            music_with_silence.export(audio_file.name, format="mp3")
+
     with tempfile.NamedTemporaryFile(delete=True, suffix=".mp4") as temp_output_file:
         utils.concatenate_videos(video_files, temp_output_file.name)
-        with open(temp_output_file.name, "rb") as f:
-            video_bytes = f.read()
+        if audio_file:
+            with tempfile.NamedTemporaryFile(delete=True, suffix=".mp4") as temp_output_file2:
+                utils.mix_video_audio(temp_output_file.name, audio_file.name, temp_output_file2.name)                
+                with open(temp_output_file2.name, "rb") as f:
+                    video_bytes = f.read()
+        else:
+            with open(temp_output_file.name, "rb") as f:
+                video_bytes = f.read()
         output_url = s3.upload(video_bytes, "mp4")
 
     # clean up clips
     for video_file in video_files:
         os.remove(video_file)
 
+    if audio_file:
+        os.remove(audio_file.name)
+
     if callback:
         callback(progress=0.99)
 

app/character.py

@@ -113,12 +113,12 @@ def update(
         self.creation_enabled = creation_enabled
         self.story_creation_enabled = story_creation_enabled
         self.concept = concept
-        self.smart_reply = smart_reply
+        self.smart_reply = False # smart_reply  # disabled until ready
         self.chat_model = chat_model
         self.image = image
         self.voice = voice
         self.function_map = {"1": self._chat_}
-        options = ["Regular conversation, chat, humor, or small talk"]
+        options = ["Regular conversation, chat, humor, small talk, or a asking for a question or comment about an attached image"]
 
         if knowledge:
             if not self.knowledge_summary.strip():
@@ -264,11 +264,12 @@ def _chat_(
         session_id=None,
     ) -> dict:
         response = self.chat(
-            prompt=message.message, 
+            prompt=message.message,
+            image=message.attachments[0] if message.attachments else None,
             id=session_id, 
             save_messages=False,
             model=self.chat_model,
-        )
+        )        
         user_message = ChatMessage(role="user", content=message.message)
         assistant_message = ChatMessage(role="assistant", content=response)
         output = {"message": response, "config": None}
@@ -490,22 +491,26 @@ def __call__(
                     system=self.chat_prompt,
                     params=self.chat_params,
                 )
+
         function = None
         if self.router_prompt:
             index = self._route_(message, session_id=session_id)
             function = self.function_map.get(index)
 
         if not function:
             function = self.function_map.get("1")
+
         output, user_message, assistant_message = function(
             message, session_id=session_id
         )
+
         self.router.add_messages(user_message, assistant_message, id=session_id)
         self.creator.add_messages(user_message, assistant_message, id=session_id)
         self.story_editor.add_messages(user_message, assistant_message, id=session_id)
         self.story_context.add_messages(user_message, assistant_message, id=session_id)
         self.qa.add_messages(user_message, assistant_message, id=session_id)
         self.chat.add_messages(user_message, assistant_message, id=session_id)
+
         return output