videos?

App/Generate/database/Model.py

@@ -6,6 +6,7 @@ from pydub import AudioSegment
 from .DescriptAPI import Speak
 from .ElevenLab import ElevenLab
 from .Vercel import AsyncImageGenerator
+from .Video3d import VideoGenerator
 import aiohttp
 from typing import List
 from pydantic import BaseModel
@@ -56,7 +57,7 @@ class Project(orm.Model):
     }
 
     async def get_all_scenes(self):
-        return await Scene.objects.filter(project=self).order_by("id").all()
+        return await Scene.objects.filter(project=self).all()
 
     async def generate_json(self):
         project_scenes: List[Scene] = await self.get_all_scenes()
@@ -113,34 +114,67 @@ class Project(orm.Model):
             )
             text_stream.extend(temp[:-1])
 
-            ## images and transitions
-            for image in scene.images:
-                file_name = str(uuid.uuid4()) + ".png"
-                self.links.append({"file_name": file_name, "link": image})
-                image_assets.append(
-                    {
-                        "type": "image",
-                        "name": file_name,
-                        "start": self.start,
-                        "end": self.start + scene.image_duration,
-                    }
-                )
-                self.start = self.start + scene.image_duration
-
-                # transitions between images
-                video_assets.append(
-                    {
-                        "type": "video",
-                        "name": "Effects/" + random.choice(transitions),
-                        "start": self.start - 1,
-                        "end": self.start + 2,
-                        "props": {
-                            "startFrom": 1 * 30,
-                            "endAt": 3 * 30,
-                            "volume": 0,
-                        },
-                    }
-                )
+            sample_image_extension = scene.images[0].split(".")[-1]
+
+            if sample_image_extension == "mp4":
+                ## moving images
+                for image in scene.images:
+                    file_name = str(uuid.uuid4()) + ".mp4"
+                    self.links.append({"file_name": file_name, "link": image})
+                    video_assets.append(
+                        {
+                            "type": "video",
+                            "name": file_name,
+                            "start": self.start,
+                            "end": self.start + scene.image_duration,
+                            "props": {
+                                "volume": 0,
+                                "loop": "true",
+                                "style": {
+                                    {
+                                        "transform": "translate(-50%, -50%)",
+                                        "position": "absolute",
+                                        "top": "50%",
+                                        "left": "50%",
+                                        "width": 1080,
+                                        "height": 1920,
+                                        "objectFit": "cover",
+                                    }
+                                },
+                            },
+                        }
+                    )
+                    self.start = self.start + scene.image_duration
+
+            else:
+                ## images and transitions
+                for image in scene.images:
+                    file_name = str(uuid.uuid4()) + ".png"
+                    self.links.append({"file_name": file_name, "link": image})
+                    image_assets.append(
+                        {
+                            "type": "image",
+                            "name": file_name,
+                            "start": self.start,
+                            "end": self.start + scene.image_duration,
+                        }
+                    )
+                    self.start = self.start + scene.image_duration
+
+                ## transitions between images
+                # video_assets.append(
+                #     {
+                #         "type": "video",
+                #         "name": "Effects/" + random.choice(transitions),
+                #         "start": self.start - 1,
+                #         "end": self.start + 2,
+                #         "props": {
+                #             "startFrom": 1 * 30,
+                #             "endAt": 3 * 30,
+                #             "volume": 0,
+                #         },
+                #     }
+                # )
 
         self.assets.append({"type": "audio", "sequence": audio_assets})
         ## add the images to assets
@@ -197,7 +231,6 @@ class Scene(orm.Model):
         self.narration_link = link
 
     async def retry_narration_generation(self):
-        print(self.narration)
         retry_count = 0
         while retry_count < 3:
             try:

App/Generate/database/Video3d.py

@@ -0,0 +1,120 @@
+import aiohttp
+import asyncio
+from itertools import chain
+
+
+class VideoGenerator:
+    def __init__(self):
+        self.base_urls = [f"https://yakova-depthflow-{i}.hf.space" for i in range(10)]
+        self.headers = {"accept": "application/json"}
+        self.default_params = {
+            "frame_rate": 30,
+            "duration": 3,
+            "quality": 1,
+            "ssaa": 0.8,
+            "raw": "false",
+        }
+
+    async def generate_video(self, base_url, params):
+        url = f"{base_url}/generate_video"
+
+        async with aiohttp.ClientSession() as session:
+            async with session.post(
+                url, params=params, headers=self.headers
+            ) as response:
+                if response.status == 200:
+                    data = await response.json()
+                    output_file = data.get("output_file")
+                    return output_file
+                else:
+                    print(f"Request to {url} failed with status: {response.status}")
+                    return None
+
+    async def check_video_ready(self, base_url, output_file):
+        url = f"{base_url}/download/{output_file}"
+
+        async with aiohttp.ClientSession() as session:
+            while True:
+                async with session.get(url, headers=self.headers) as response:
+                    if response.status == 200:
+                        video_content = await response.read()
+                        if len(video_content) > 0:
+                            return url
+                        else:
+                            print(
+                                f"Video {output_file} is ready but the file size is zero, retrying in 10 seconds..."
+                            )
+                            await asyncio.sleep(120)
+                    elif response.status == 404:
+                        data = await response.json()
+                        if data.get("detail") == "Video not found":
+                            print(
+                                f"Video {output_file} not ready yet, retrying in 10 seconds..."
+                            )
+                            await asyncio.sleep(10)
+                        else:
+                            print(f"Unexpected response for {output_file}: {data}")
+                            return None
+                    else:
+                        print(f"Request to {url} failed with status: {response.status}")
+                        return None
+
+    async def process_image(self, base_url, image_link):
+        params = self.default_params.copy()
+        params["image_link"] = image_link
+
+        output_file = await self.generate_video(base_url, params)
+        if output_file:
+            print(f"Generated video file id: {output_file} for image {image_link}")
+            video_url = await self.check_video_ready(base_url, output_file)
+            if video_url:
+                print(
+                    f"Video for {image_link} is ready and can be downloaded from: {video_url}"
+                )
+                return video_url
+            else:
+                print(f"Failed to get the video URL for {image_link}")
+                return None
+        else:
+            print(f"Failed to generate the video for {image_link}")
+            return None
+
+    def flatten(self, nested_list):
+        return list(chain.from_iterable(nested_list))
+
+    def nest(self, flat_list, nested_dims):
+        it = iter(flat_list)
+        return [[next(it) for _ in inner_list] for inner_list in nested_dims]
+
+    async def run(self, nested_image_links):
+        flat_image_links = self.flatten(nested_image_links)
+        tasks = []
+        base_index = 0
+
+        for image_link in flat_image_links:
+            base_url = self.base_urls[base_index % len(self.base_urls)]
+            tasks.append(self.process_image(base_url, image_link))
+            base_index += 1
+
+        flat_video_urls = await asyncio.gather(*tasks)
+        nested_video_urls = self.nest(flat_video_urls, nested_image_links)
+        return nested_video_urls
+
+
+# # Example usage
+# nested_image_links = [
+#     [
+#         "https://replicate.delivery/yhqm/mQId1rdf4Z3odCyB7cPsx1KwhHfdRc3w44eYAGNG9AQfV0dMB/out-0.png"
+#     ],
+#     [
+#         "https://replicate.delivery/yhqm/mQId1rdf4Z3odCyB7cPsx1KwhHfdRc3w44eYAGNG9AQfV0dMB/out-1.png",
+#         "https://replicate.delivery/yhqm/mQId1rdf4Z3odCyB7cPsx1KwhHfdRc3w44eYAGNG9AQfV0dMB/out-2.png",
+#     ],
+#     # Add more nested image links here
+# ]
+
+# loop = asyncio.get_event_loop()
+# video_generator = VideoGenerator()
+# nested_video_urls = loop.run_until_complete(video_generator.run(nested_image_links))
+
+# print("Generated video URLs:", nested_video_urls)

App/Generate/generatorRoutes.py

@@ -6,7 +6,14 @@ from .utils.HuggingChat import Hugging
 from .Story.Story import Story
 import asyncio, pprint, json
 from tqdm import tqdm
-from .database.Model import models, database_url, Scene, Project, database
+from .database.Model import (
+    models,
+    database_url,
+    Scene,
+    Project,
+    database,
+    VideoGenerator,
+)
 from .utils.RenderVideo import RenderVideo
 from .Prompts.StoryGen import Prompt
 from App.Editor.editorRoutes import celery_task, EditorRequest
@@ -23,18 +30,21 @@ async def from_dict_generate(data: Story):
     await generate_assets(generated_story=generated_strory)
 
 
-async def generate_assets(generated_story: Story, batch_size=4):
+async def generate_assets(generated_story: Story, batch_size=4, threeD=True):
     x = await Project.objects.create(name=str(uuid.uuid4()))
 
     # Assuming generated_story.scenes is a list of scenes
-    scene_updates = []
     with tqdm(total=len(generated_story.scenes)) as pbar:
+
+        all_scenes: list[Scene] = []
+        # create the batches
         for i in range(0, len(generated_story.scenes), batch_size):
             batch = generated_story.scenes[
                 i : i + batch_size
             ]  # Get a batch of two story scenes
             batch_updates = []
 
+            # generate pictures or narration per batch
             for story_scene in batch:
                 model_scene = await Scene.objects.create(project=x)
                 model_scene.image_prompts = story_scene.image_prompts
@@ -43,12 +53,27 @@ async def generate_assets(generated_story: Story, batch_size=4):
                 batch_updates.append(
                     update_scene(model_scene)
                 )  # Append update coroutine to batch_updates
-            scene_updates.extend(batch_updates)  # Accumulate updates for later awaiting
+            # pause per batch
             await asyncio.gather(
                 *batch_updates
             )  # Await update coroutines for this batch
+            all_scenes.append(model_scene)
             pbar.update(len(batch))  # Increment progress bar by the size of the batch
 
+    ###### Here we generate the videos
+
+    if threeD:
+        vid_gen = VideoGenerator
+        nested_images = []
+        for scene in all_scenes:
+            nested_images.append(scene.images)
+
+        results = await vid_gen.run(nested_image_links=nested_images)
+
+        for result, _scene in zip(results, all_scenes):
+            _scene.images = result
+            await _scene.update(**_scene.__dict__)
+
     temp = await x.generate_json()
     # print(temp)
 

App/Generate/utils/Cohere.py

@@ -22,7 +22,11 @@ class VideoOutput(BaseModel):
 
 # Patching the Cohere client with the instructor for enhanced capabilities
 client = instructor.from_cohere(
-    cohere.Client(os.environ.get("COHERE_API", "RANDOM_STRING")),
+    cohere.Client(
+        os.environ.get(
+            "COHERE_API",
+        )
+    ),
     # max_tokens=5000,
     model="command-r-plus",
 )
@@ -36,7 +40,7 @@ def chatbot(prompt: str, model: str = "command-r-plus"):
 
     response: VideoOutput = client.chat.completions.create(
         model=model,
-        max_tokens=5000,
+        # max_tokens=5000,
         response_model=VideoOutput,
         messages=[
             {
@@ -46,3 +50,6 @@ def chatbot(prompt: str, model: str = "command-r-plus"):
         ],
     )
     return response.dict()
+
+
+# print(chatbot("A horror story"))

Remotion-app/package.json

@@ -17,8 +17,6 @@
 		"@remotion/transitions": "4.0.147",
 		"@remotion/zod-types": "4.0.147",
 		"@remotion/tailwind": "4.0.147",
-		"class-variance-authority": "^0.7.0",
-		"clsx": "^2.1.0",
 		"react": "^18.0.0",
 		"react-dom": "^18.0.0",
 		"remotion": "4.0.147",

Remotion-app/remotion.config.js

@@ -12,4 +12,4 @@ Config.overrideWebpackConfig((currentConfiguration) => {
 
 //Config.setBrowserExecutable("/usr/bin/chrome-headless-shell");
 Config.setVideoImageFormat('jpeg');
-// Config.setConcurrency(2);
+Config.setConcurrency(1);