Merge pull request #6 from edenartlab/stage

stories

.gitignore

@@ -5,7 +5,10 @@ build/
 dist/
 wheels/
 *.egg-info
+.DS_Store
+*.mp4
+*.mp3
 
 # venv
 .venv
-.env
+.env

app/models.py

@@ -48,6 +48,16 @@ class DialogueOutput(BaseModel):
     dialogue: List[dict]
 
 
+class CinemaRequest(BaseModel):
+    character_ids: List[str]
+    prompt: str
+    model: str = "gpt-4-1106-preview"
+    params: dict = {}
+
+class CinemaResult(BaseModel):
+    stills: List[str]
+
+
 class ChatRequest(BaseModel):
     """
     A chat request to an EdenCharacter

app/plugins/eden.py

@@ -0,0 +1,37 @@
+import os
+from io import BytesIO
+import requests
+import replicate
+from typing import Optional
+from pydantic import BaseModel, Field
+
+EDEN_API_URL = os.environ.get("EDEN_API_URL")
+EDEN_API_KEY = os.environ.get("EDEN_API_KEY")
+EDEN_API_SECRET = os.environ.get("EDEN_API_SECRET")
+
+from eden_sdk.EdenClient import EdenClient
+
+eden = EdenClient(
+    api_url=EDEN_API_URL,
+    api_key=EDEN_API_KEY,
+    api_secret=EDEN_API_SECRET,
+)
+
+create = eden.create
+
+# def run_task(
+#     config: dict[any], 
+# ):
+#     config = {
+#         "text_input": "someone here",
+#     }
+#     print("GO", config)
+#     print("ok 2")
+#     urls = eden.create(generator_name='create', config=config)
+#     print("ok 3")
+
+#     print(urls[0])
+#     print("ok 4")
+
+#     return urls[0]
+

app/plugins/replicate.py

@@ -2,7 +2,7 @@
 from io import BytesIO
 import requests
 import replicate
-from typing import Optional
+from typing import Optional, List
 from pydantic import BaseModel, Field
 
 REPLICATE_API_KEY = os.environ.get("REPLICATE_API_KEY")
@@ -76,5 +76,56 @@ def wav2lip(
 
     output = list(output)
     output_url = output[0]["files"][0]
-    print("output_url", output_url)
+
+    return output_url
+
+
+def sdxl(
+    text_input: str,
+    width: int,
+    height: int,
+):
+    config = {
+        "text_input": text_input,
+        "width": width,
+        "height": height,
+        "n_samples": 1,
+    }
+
+    output = run_task(
+        config, 
+        model_name="abraham-ai/eden-sd-pipelines-sdxl"
+    )
+
+    output = list(output)
+    output_url = output[0]["files"][0]
+
+    return output_url
+
+
+def txt2vid(
+    interpolation_texts: List[str],
+    width: int,
+    height: int,
+):
+    interpolation_texts = "|".join(interpolation_texts)
+
+    config = {
+        "mode": "comfy_txt2vid",
+        "interpolation_texts": interpolation_texts,
+        "width": width/2,
+        "height": height/2,
+        "n_frames": 100,
+    }
+
+    print("LETS RUN!!!", config)
+
+    output = run_task(
+        config, 
+        model_name="abraham-ai/eden-comfyui"
+    )
+
+    output = list(output)
+    output_url = output[0]["files"][0]
+
     return output_url

app/prompt_templates/cinema/__init__.py

@@ -3,8 +3,11 @@
 
 dir_path = Path(__file__).parent
 
-with open(dir_path / 'screenwriter.txt', 'r') as file:
-    screenwriter_template = Template(file.read())
+with open(dir_path / 'screenwriter_system.txt', 'r') as file:
+    screenwriter_system_template = Template(file.read())
+
+with open(dir_path / 'screenwriter_prompt.txt', 'r') as file:
+    screenwriter_prompt_template = Template(file.read())
 
 with open(dir_path / 'director.txt', 'r') as file:
     director_template = Template(file.read())

app/prompt_templates/cinema/screenwriter_prompt.txt

@@ -0,0 +1,5 @@
+Characters:
+$character_details
+
+The premise of the story is:
+$story

app/prompt_templates/cinema/screenwriter_system.txt

@@ -0,0 +1,12 @@
+You are a critically acclaimed screenwriter who writes incredibly captivating and original scripts.
+
+Users will give you a cast of characters, including their names and biographies, as well as a premise or synopsis for the story.
+
+You will then write a screenplay for a film based on the information provided. Intersperse the screenplay with descriptions of events, dialogues, and character monologues. It will come as a sequence of clips. A clip contains the following:
+
+voiceover: whether the voiceover is the narrator, the character speaking, or none at all (just sound effects)
+character: If voiceover is in character mode, the name of the speaking character
+speech: If voiceover is in character or narrator mode, the text of the speech
+image_description: a description of the image content for the clip
+
+Do not include an introduction or restatement of the prompt, just go straight into the screenplay.

app/routers/generator.py

@@ -8,12 +8,13 @@
 import requests
 
 from .dags import monologue_dag, dialogue_dag
+from .story import cinema
 from ..mongo import get_character_data
 from ..llm import LLM
 from ..prompt_templates import monologue_template, dialogue_template
 
 from ..models import MonologueRequest, MonologueOutput
-from ..models import DialogueRequest, DialogueOutput
+from ..models import DialogueRequest, DialogueOutput, CinemaRequest
 from ..models import TaskRequest, TaskUpdate, TaskOutput
 
 router = APIRouter()
@@ -52,6 +53,15 @@ def process_task(task_id: str, request: TaskRequest, task_type: str):
             )
             output_url, thumbnail_url = dialogue_dag(task_req)
 
+        elif task_type == "story":
+            character_ids = request.config.get("characterIds")
+            prompt = request.config.get("prompt")
+            task_req = CinemaRequest(
+                character_ids=character_ids,
+                prompt=prompt,
+            )
+            output_url, thumbnail_url = cinema(task_req)
+
         output = TaskOutput(
             files=[output_url],
             thumbnails=[thumbnail_url],
@@ -89,6 +99,6 @@ def process_task(task_id: str, request: TaskRequest, task_type: str):
 @router.post("/tasks/create")
 async def generate_task(background_tasks: BackgroundTasks, request: TaskRequest):
     task_id = str(uuid.uuid4())
-    if request.generatorName in ["monologue", "dialogue"]:
+    if request.generatorName in ["monologue", "dialogue", "story"]:
         background_tasks.add_task(process_task, task_id, request, request.generatorName)
     return {"id": task_id}