src/generation.py

import os

import sys
sys.path.insert(1, os.path.join(sys.path[0], '..'))

import warnings

import cv2
import numpy as np
import tqdm
import torch
import torch.nn.functional as F
import torchvision.io as vision_io



from models.pipelines import TextToVideoSDPipelineSpatialAware
from diffusers.utils import export_to_video
from PIL import Image
import torchvision



import warnings
warnings.filterwarnings("ignore")

OUTPUT_PATH = "/scr/demo"

def generate_video(pipe, overall_prompt, latents, get_latents=False, num_frames=24, num_inference_steps=50, fg_masks=None, 
        fg_masked_latents=None, frozen_steps=0, frozen_prompt=None, custom_attention_mask=None, fg_prompt=None):
    
    video_frames = pipe(overall_prompt, num_frames=num_frames, latents=latents, num_inference_steps=num_inference_steps, frozen_mask=fg_masks, 
    frozen_steps=frozen_steps, latents_all_input=fg_masked_latents, frozen_prompt=frozen_prompt, custom_attention_mask=custom_attention_mask, fg_prompt=fg_prompt,
    make_attention_mask_2d=True, attention_mask_block_diagonal=True, height=320, width=576 ).frames
    if get_latents:
        video_latents = pipe(overall_prompt, num_frames=num_frames, latents=latents, num_inference_steps=num_inference_steps, output_type="latent").frames
        return video_frames, video_latents
    
    return video_frames

def save_frames(path):
    video, audio, video_info = vision_io.read_video(f"{path}.mp4", pts_unit='sec')

    # Number of frames
    num_frames = video.size(0)

    # Save each frame
    os.makedirs(f"{path}", exist_ok=True)
    for i in range(num_frames):
        frame = video[i, :, :, :].numpy()
        # Convert from C x H x W to H x W x C and from torch tensor to PIL Image
        # frame = frame.permute(1, 2, 0).numpy()
        img = Image.fromarray(frame.astype('uint8'))
        img.save(f"{path}/frame_{i:04d}.png")

if __name__ == "__main__":
    # Example usage
    num_frames = 24
    save_path = "video"
    torch_device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    random_latents = torch.randn([1, 4, num_frames, 40, 72], generator=torch.Generator().manual_seed(2)).to(torch_device)
    
    try:
        pipe = TextToVideoSDPipelineSpatialAware.from_pretrained(
            "cerspense/zeroscope_v2_576w", torch_dtype=torch.float, variant="fp32").to(torch_device)
    except:
        pipe = TextToVideoSDPipelineSpatialAware.from_pretrained(
            "cerspense/zeroscope_v2_576w", torch_dtype=torch.float, variant="fp32").to(torch_device)
        
    # Generate video


    bbox_mask = torch.zeros([24, 1, 40, 72], device=torch_device)
    bbox_mask_2 = torch.zeros([24, 1, 40, 72], device=torch_device)

    
    x_start = [10 + (i % 3) for i in range(num_frames)]  # Simulating slight movement in x
    x_end = [30 + (i % 3) for i in range(num_frames)]    # Simulating slight movement in x
    y_start = [10 for _ in range(num_frames)]            # Static y start as the bear is seated/standing
    y_end = [25 for _ in range(num_frames)]              # Static y end, considering the size of the guitar

    # Populate the bbox_mask tensor with ones where the bounding box is located
    for i in range(num_frames):
        bbox_mask[i, :, x_start[i]:x_end[i], y_start[i]:y_end[i]] = 1
        bbox_mask_2[i, :, x_start[i]:x_end[i], 72-y_end[i]:72-y_start[i]] = 1

    # fg_masks = bbox_mask
    fg_masks = [bbox_mask, bbox_mask_2]
    
       

    frozen_prompt = None
    fg_masked_latents = None
    fg_objects = []
    prompts = []
    prompts = [
        (["cat", "goldfish bowl"], "A cat curiously staring at a goldfish bowl on a sunny windowsill."),
        (["Superman", "Batman"], "Superman and Batman standing side by side in a heroic pose against a city skyline."),
        (["rose", "daisy"], "A rose and a daisy in a small vase on a rustic wooden table."),
        (["Harry Potter", "Hermione Granger"], "Harry Potter and Hermione Granger studying a magical map."),
        (["butterfly", "dragonfly"], "A butterfly and a dragonfly resting on a leaf in a vibrant garden."),
        (["teddy bear", "toy train"], "A teddy bear and a toy train on a child's playmat in a brightly lit room."),
        (["frog", "turtle"], "A frog and a turtle sitting on a lily pad in a serene pond."),
        (["Mickey Mouse", "Donald Duck"], "Mickey Mouse and Donald Duck enjoying a day at the beach, building a sandcastle."),
        (["penguin", "seal"], "A penguin and a seal lounging on an iceberg in the Antarctic."),
        (["lion", "zebra"], "A lion and a zebra peacefully drinking water from the same pond in the savannah.")
    ]

    for fg_object, overall_prompt in prompts:
        os.makedirs(f"{OUTPUT_PATH}/{save_path}/{overall_prompt}-mask", exist_ok=True)
        try:
            for i in range(num_frames):
                torchvision.utils.save_image(fg_masks[0][i,0], f"{OUTPUT_PATH}/{save_path}/{overall_prompt}-mask/frame_{i:04d}_0.png")
                torchvision.utils.save_image(fg_masks[1][i,0], f"{OUTPUT_PATH}/{save_path}/{overall_prompt}-mask/frame_{i:04d}_1.png")
        except:
            pass
        print(fg_object, overall_prompt)
        seed = 2
        random_latents = torch.randn([1, 4, num_frames, 40, 72], generator=torch.Generator().manual_seed(seed)).to(torch_device)
        for num_inference_steps in range(40,50,10):
            for frozen_steps in [0, 1, 2]:
                video_frames = generate_video(pipe, overall_prompt, random_latents, get_latents=False, num_frames=num_frames, num_inference_steps=num_inference_steps, 
                    fg_masks=fg_masks, fg_masked_latents=fg_masked_latents, frozen_steps=frozen_steps, frozen_prompt=frozen_prompt, fg_prompt=fg_object)
                # Save video frames
                os.makedirs(f"{OUTPUT_PATH}/{save_path}/{overall_prompt}", exist_ok=True)
                video_path = export_to_video(video_frames, f"{OUTPUT_PATH}/{save_path}/{overall_prompt}/{frozen_steps}_of_{num_inference_steps}_{seed}_masked.mp4")
                save_frames(f"{OUTPUT_PATH}/{save_path}/{overall_prompt}/{frozen_steps}_of_{num_inference_steps}_{seed}_masked")