allow prompt only

app.py

@@ -47,7 +47,7 @@ os.environ["PROCESSED_RESULTS"] = f"{os.getcwd()}/processed_results"
 import gradio as gr
 import argparse
 from ovi.ovi_fusion_engine import OviFusionEngine, DEFAULT_CONFIG
-from diffusers import FluxPipeline
+from diffusers import DiffusionPipeline
 import tempfile
 from ovi.utils.io_utils import save_video
 from ovi.utils.processing_utils import clean_text, scale_hw_to_area_divisible
@@ -104,6 +104,7 @@ print(f"loading model...")
 DEFAULT_CONFIG['cpu_offload'] = enable_cpu_offload # always use cpu offload if image generation is enabled
 DEFAULT_CONFIG['mode'] = "t2v"  # hardcoded since it is always cpu offloaded
 ovi_engine = OviFusionEngine()
+flux_model = DiffusionPipeline.from_pretrained("black-forest-labs/FLUX.1-Krea-dev", torch_dtype=torch.bfloat16)
 print("loaded model")
 
 
@@ -133,10 +134,39 @@ def resize_for_model(image_path):
     )
     return new_img, target_size
 
+@spaces.GPU()
+def generate_image(text_prompt, session_id, image_height = 1024, image_width = 1024):
+
+    if flux_model is None:
+        return None
+    text_prompt = clean_text(text_prompt)
+
+    image_h, image_w = scale_hw_to_area_divisible(image_height, image_width, area=1024 * 1024)
+
+    flux_model.to("cuda")
+
+    image = flux_model(
+        text_prompt,
+        height=image_h,
+        width=image_w,
+        num_inference_steps = 28,
+        guidance_scale=4.5,
+        generator=torch.Generator().manual_seed(int(1234))
+    ).images[0] 
+
+    flux_model.to("cpu")
+
+    output_dir = os.path.join(os.environ["PROCESSED_RESULTS"], session_id)
+    os.makedirs(output_dir, exist_ok=True)
+    output_path = os.path.join(output_dir, f"generate_image.png")
+
+    image.save(output_path)
+    return output_path
+
 def generate_scene(
     text_prompt,
-    image,
     sample_steps = 50,
+    image = None,
     session_id = None,
     video_seed = 100,
     solver_name = "unipc",
@@ -150,17 +180,15 @@ def generate_scene(
 ):
     text_prompt_processed = (text_prompt or "").strip()
 
-    if not image:
-        raise gr.Error("Please provide an image")
-    
-
     if not text_prompt_processed:
         raise gr.Error("Please enter a prompt.")
     
+    if session_id is None:
+        session_id = uuid.uuid4().hex
     
     return generate_video(text_prompt, 
-                        image, 
                         sample_steps, 
+                        image, 
                         session_id,
                         video_seed,
                         solver_name,
@@ -174,8 +202,8 @@ def generate_scene(
 
 def get_duration(
     text_prompt,
-    image,
     sample_steps,
+    image,
     session_id,
     video_seed,
     solver_name,
@@ -187,16 +215,21 @@ def get_duration(
     audio_negative_prompt,
     progress,
 ):
+    image_generation_s = 0
+
+    if not image:
+        image_generation_s = 30
+
     warmup = 20
 
-    return int(sample_steps * 3 + warmup)
+    return int(sample_steps * 3 + warmup + image_generation_s)
     
 
 @spaces.GPU(duration=get_duration)
 def generate_video(
     text_prompt,
-    image,
     sample_steps = 50,
+    image = None,
     session_id = None,
     video_seed = 100,
     solver_name = "unipc",
@@ -208,47 +241,48 @@ def generate_video(
     audio_negative_prompt = "",
     progress=gr.Progress(track_tqdm=True)
 ):
-    try:
-        image_path = None
+
+    if session_id is None:
+        session_id = uuid.uuid4().hex
+
+    image_path = None
+
+    if not image:
+        image = generate_image(text_prompt, session_id)
+
+    if image is not None:
+        image_path = image
+
         
-        if image is not None:
-            image_path = image
-
-        if session_id is None:
-            session_id = uuid.uuid4().hex
-            
-
-        output_dir = os.path.join(os.environ["PROCESSED_RESULTS"], session_id)
-        os.makedirs(output_dir, exist_ok=True)
-        output_path = os.path.join(output_dir, f"generated_video.mp4")
-
-
-        _, target_size = resize_for_model(image_path)
-
-        video_frame_width = target_size[0]
-        video_frame_height = target_size[1]
-
-        generated_video, generated_audio, _ = ovi_engine.generate(
-            text_prompt=text_prompt,
-            image_path=image_path,
-            video_frame_height_width=[video_frame_height, video_frame_width],
-            seed=video_seed,
-            solver_name=solver_name,
-            sample_steps=sample_steps,
-            shift=shift,
-            video_guidance_scale=video_guidance_scale,
-            audio_guidance_scale=audio_guidance_scale,
-            slg_layer=slg_layer,
-            video_negative_prompt=video_negative_prompt,
-            audio_negative_prompt=audio_negative_prompt,
-        )
 
-        save_video(output_path, generated_video, generated_audio, fps=24, sample_rate=16000)
+    output_dir = os.path.join(os.environ["PROCESSED_RESULTS"], session_id)
+    os.makedirs(output_dir, exist_ok=True)
+    output_path = os.path.join(output_dir, f"generated_video.mp4")
+
+
+    _, target_size = resize_for_model(image_path)
+
+    video_frame_width = target_size[0]
+    video_frame_height = target_size[1]
+
+    generated_video, generated_audio, _ = ovi_engine.generate(
+        text_prompt=text_prompt,
+        image_path=image_path,
+        video_frame_height_width=[video_frame_height, video_frame_width],
+        seed=video_seed,
+        solver_name=solver_name,
+        sample_steps=sample_steps,
+        shift=shift,
+        video_guidance_scale=video_guidance_scale,
+        audio_guidance_scale=audio_guidance_scale,
+        slg_layer=slg_layer,
+        video_negative_prompt=video_negative_prompt,
+        audio_negative_prompt=audio_negative_prompt,
+    )
 
-        return output_path
-    except Exception as e:
-        print(f"Error during video generation: {e}")
-        return None
+    save_video(output_path, generated_video, generated_audio, fps=24, sample_rate=16000)
+
+    return output_path
 
 
 def cleanup(request: gr.Request):
@@ -268,7 +302,6 @@ css = """
         max-width: 1024px;
     }
     """
-
 theme = gr.themes.Ocean()
 
 with gr.Blocks(css=css, theme=theme) as demo:
@@ -298,21 +331,21 @@ with gr.Blocks(css=css, theme=theme) as demo:
         with gr.Row():
             with gr.Column():
                 # Image section
-                image = gr.Image(type="filepath", label="Image", height=360)
 
                 video_text_prompt = gr.Textbox(label="Scene Prompt", 
                                                lines=5,
-                                               value="A person in a scene that their mouth is slightly open as they speak, <S>Enjoy this moment<E> and as they roll their eyes. <AUDCAP>Clear voice, faint ambient outdoor sounds.<ENDAUDCAP>",
-                                               placeholder="Describe your video...")
+                                               placeholder="Describe your scene...")
                 sample_steps = gr.Slider(
                     value=50,
-                    label="Generation Steps",
+                    label="Sample Steps",
                     minimum=20,
                     maximum=100,
                     step=1.0
                 )
                 run_btn = gr.Button("Action 🎬", variant="primary")
 
+                image = gr.Image(type="filepath", label="Image Ref", height=360)
+
                 gr.Markdown(
                     """
                     💡 **Prompt Guidelines**  
@@ -350,35 +383,41 @@ with gr.Blocks(css=css, theme=theme) as demo:
 
                         [
                             "The video opens with a close-up of a woman with vibrant reddish-orange, shoulder-length hair and heavy dark eye makeup. She is wearing a dark brown leather jacket over a grey hooded top. She looks intently to her right, her mouth slightly agape, and her expression is serious and focused. The background shows a room with light green walls and dark wooden cabinets on the left, and a green plant on the right. She speaks, her voice clear and direct, saying, <S>doing<E>. She then pauses briefly, her gaze unwavering, and continues, <S>And I need you to trust them.<E>. Her mouth remains slightly open, indicating she is either about to speak more or has just finished a sentence, with a look of intense sincerity.. <AUDCAP>Tense, dramatic background music, clear female voice.<ENDAUDCAP>", 
-                            "example_prompts/pngs/8.png",
                             50,
+                            None,
                         ],
 
                         [
                             "A young woman with long, wavy blonde hair and light-colored eyes is shown in a medium shot against a blurred backdrop of lush green foliage. She wears a denim jacket over a striped top. Initially, her eyes are closed and her mouth is slightly open as she speaks, <S>Enjoy this moment<E>. Her eyes then slowly open, looking slightly upwards and to the right, as her expression shifts to one of thoughtful contemplation. She continues to speak, <S>No matter where it's taking<E>, her gaze then settling with a serious and focused look towards someone off-screen to her right.. <AUDCAP>Clear female voice, faint ambient outdoor sounds.<ENDAUDCAP>",
-                            "example_prompts/pngs/2.png",
                             50,
+                            "example_prompts/pngs/2.png",
                         ],
 
                         [
                             "A bearded man wearing large dark sunglasses and a blue patterned cardigan sits in a studio, actively speaking into a large, suspended microphone. He has headphones on and gestures with his hands, displaying rings on his fingers. Behind him, a wall is covered with red, textured sound-dampening foam on the left, and a white banner on the right features the ""CHOICE FM"" logo and various social media handles like ""@ilovechoicefm"" with ""RALEIGH"" below it. The man intently addresses the microphone, articulating, <S>is talent. It's all about authenticity. You gotta be who you really are, especially if you're working<E>. He leans forward slightly as he speaks, maintaining a serious expression behind his sunglasses.. <AUDCAP>Clear male voice speaking into a microphone, a low background hum.<ENDAUDCAP>",
-                            "example_prompts/pngs/5.png",
                             50,
+                            "example_prompts/pngs/5.png",
                         ],
 
                         [
                             "The video opens with a close-up on an older man with long, grey hair and a short, grey beard, wearing dark sunglasses. He is clad in a dark coat, possibly with fur trim, and black gloves. His face is angled slightly upwards and to the right, as he begins to speak, his mouth slightly open. In the immediate foreground, out of focus, is the dark-clad shoulder and the back of the head of another person. The man articulates, <S>labbra. Ti ci vorrebbe...<E> His expression remains contemplative, and he continues, seemingly completing his thought, <S>Un ego solare.<E> The background behind him is a textured, grey stone wall, suggesting an outdoor setting. The man's gaze remains fixed upwards, his expression thoughtful.. <AUDCAP>A clear, slightly low-pitched male voice speaking Italian. The overall soundscape is quiet, with no prominent background noises or music.<ENDAUDCAP>",
-                            "example_prompts/pngs/7.png",
                             50,
+                            "example_prompts/pngs/7.png",
                         ],
 
                         [
                             "The scene is set outdoors with a blurry, bright green background, suggesting grass and a sunny environment. On the left, a woman with long, dark hair, wearing a red top and a necklace with a white pendant, faces towards the right. Her expression is serious and slightly perturbed as she speaks, with her lips slightly pursed. She says, <S>UFO, UFC thing.<E> On the right, the back of a man's head and his right ear are visible, indicating he is facing away from the camera, listening to the woman. He has short, dark hair. The woman continues speaking, her expression remaining serious, <S>And if you're not watching that, it's one of those ancient movies from an era that's<E> as the frame holds steady on the two figures.. <AUDCAP>Clear female speech, distant low-frequency hum.<ENDAUDCAP>",
+                            50,
                             "example_prompts/pngs/9.png",
+                        ],
+
+                        [
+                            "The scene is set in a dimly lit, hazy room, creating a somber atmosphere. An woman with light, slightly disheveled hair is visible in the foreground, her face mostly obscured by deep shadows, but her mouth is visible as she speaks. She wears a work-style shirt, and her hands are clasped together. In the background, to the right and slightly out of focus, a man with a mustache and beard is seated, facing forward, also largely in shadow, appearing to listen intently. The woman looks directly forward as she slowly enunciates, <S>Only through death will the third door be<E>. The scene ends abruptly.. <AUDCAP>Clear, deliberate female voice speaking, low ambient hum and subtle atmospheric sounds creating a tense mood.<ENDAUDCAP>",
                             50,
+                            None,
                         ],
                     ],
-                    inputs=[video_text_prompt, image, sample_steps],
+                    inputs=[video_text_prompt, sample_steps, image],
                     outputs=[output_path],
                     fn=generate_video,
                     cache_examples=True,
@@ -386,7 +425,7 @@ with gr.Blocks(css=css, theme=theme) as demo:
 
     run_btn.click(
         fn=generate_scene,
-        inputs=[video_text_prompt, image, sample_steps, session_state],
+        inputs=[video_text_prompt, sample_steps, image, session_state],
         outputs=[output_path],
     )
 

ovi/utils/fm_solvers_unipc.py

@@ -16,6 +16,21 @@ from diffusers.utils import deprecate, is_scipy_available
 if is_scipy_available():
     import scipy.stats
 
+def _safe_solve(R, b, device, out_dtype):
+    """
+    Try GPU solve (MAGMA/cuSOLVER) in float32; on failure, fall back to CPU solve.
+    """
+    R32 = R.float().contiguous()
+    b32 = b.float().contiguous()
+    try:
+        return torch.linalg.solve(R32, b32).to(device=device, dtype=out_dtype)
+    except RuntimeError as e:
+        # cuSOLVER creation / internal errors are common here on certain GPUs
+        if "cusolver" in str(e).lower() or "CUSOLVER" in str(e):
+            R_cpu, b_cpu = R32.cpu(), b32.cpu()
+            sol = torch.linalg.solve(R_cpu, b_cpu)
+            return sol.to(device=device, dtype=out_dtype)
+        raise
 
 class FlowUniPCMultistepScheduler(SchedulerMixin, ConfigMixin):
     """
@@ -604,7 +619,9 @@ class FlowUniPCMultistepScheduler(SchedulerMixin, ConfigMixin):
         if order == 1:
             rhos_c = torch.tensor([0.5], dtype=x.dtype, device=device)
         else:
-            rhos_c = torch.linalg.solve(R, b).to(device).to(x.dtype)
+            rhos_c = _safe_solve(R, b, device, x.dtype)
+
+
 
         if self.predict_x0:
             x_t_ = sigma_t / sigma_s0 * x - alpha_t * h_phi_1 * m0