app.py

import argparse
from pathlib import Path

import gradio as gr
from icecream import ic
import torch
from diffusers import DiffusionPipeline

from visual_anagrams.views import get_views, VIEW_MAP_NAMES
from visual_anagrams.samplers import sample_stage_1, sample_stage_2
from visual_anagrams.utils import add_args, save_illusion, save_metadata
from visual_anagrams.animate import animate_two_view

stage_1 = DiffusionPipeline.from_pretrained(
                "DeepFloyd/IF-I-M-v1.0",
                variant="fp16",
                torch_dtype=torch.float16)
stage_2 = DiffusionPipeline.from_pretrained(
                "DeepFloyd/IF-II-M-v1.0",
                text_encoder=None,
                variant="fp16",
                torch_dtype=torch.float16,
            )
stage_1.enable_model_cpu_offload()
stage_2.enable_model_cpu_offload()


def generate_content(
    style,
    prompt_for_original,
    prompt_for_transformed,
    transformation,
    num_inference_steps,
    seed,
):
    prompts = [f'{style} {p}'.strip() for p in [prompt_for_original, prompt_for_transformed]]
    prompt_embeds = [stage_1.encode_prompt(p) for p in prompts]
    prompt_embeds, negative_prompt_embeds = zip(*prompt_embeds)
    prompt_embeds = torch.cat(prompt_embeds)
    negative_prompt_embeds = torch.cat(negative_prompt_embeds)

    views = ['identity', VIEW_MAP_NAMES[transformation]]
    views = get_views(views)

    generator = torch.manual_seed(seed + 42)

    print("Sample stage 1")
    image = sample_stage_1(stage_1,
                           prompt_embeds,
                           negative_prompt_embeds,
                           views,
                           num_inference_steps=num_inference_steps,
                           generator=generator)

    print("Sample stage 2")
    image = sample_stage_2(stage_2,
                           image,
                           prompt_embeds,
                           negative_prompt_embeds,
                           views,
                           num_inference_steps=num_inference_steps,
                           generator=generator)
    save_illusion(image, views, Path(""))
    
    output_name = f"illusion.mp4"
    size = image.shape[-1]
    animate_two_view(
        f"sample_{size}.png",
        views[1],
        prompts[0],
        prompts[1],
        save_video_path=output_name,
    )
    return output_name, f"sample_{size}.views.png"


with open("description.txt") as f:
    description = f.read()

choices = list(VIEW_MAP_NAMES.keys())
gradio_app = gr.Interface(
    fn=generate_content,
    title="Multi-View Illusion Diffusion",
    inputs=[
        gr.Textbox(label="Style", placeholder="an oil painting of"),
        gr.Textbox(label="Prompt for original view", placeholder="a dress"),
        gr.Textbox(label="Prompt for transformed view", placeholder="an old man"),
        gr.Dropdown(label="View transformation", choices=choices, value=choices[0]),
        gr.Number(label="Number of diffusion steps", value=50, step=1, minimum=1, maximum=300),
        gr.Number(label="Random seed", value=0, step=1, minimum=0, maximum=100000),
    ],
    outputs=[gr.Video(label="Illusion"), gr.Image(label="Before and After")],
    description=description,
)


if __name__ == "__main__":
    gradio_app.launch() # server_name="0.0.0.0"