app.py

import gradio as gr
import torch

from diffuserslocal.src.diffusers import UNet2DConditionModel
from share_btn import community_icon_html, loading_icon_html, share_js
from diffuserslocal.src.diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_ldm3d_inpaint import StableDiffusionLDM3DInpaintPipeline
from PIL import Image
import numpy as np
import cv2

from functools import partial
import tempfile

from mesh import get_mesh

device = "cuda" if torch.cuda.is_available() else "cpu"
model_arch = "zoe"
# Inpainting pipeline


unet = UNet2DConditionModel.from_pretrained("pablodawson/ldm3d-inpainting", cache_dir="cache", subfolder="unet")
pipe = StableDiffusionLDM3DInpaintPipeline.from_pretrained("Intel/ldm3d-4c", cache_dir="cache" ).to(device)

# Depth estimation
model_type = "DPT_Large"     # MiDaS v3 - Large     (highest accuracy, slowest inference speed)
#model_type = "DPT_Hybrid"   # MiDaS v3 - Hybrid    (medium accuracy, medium inference speed)
#model_type = "MiDaS_small"  # MiDaS v2.1 - Small   (lowest accuracy, highest inference speed)

if model_arch == "midas":
    midas = torch.hub.load("intel-isl/MiDaS", model_type)

    midas.to(device)
    midas.eval()

    midas_transforms = torch.hub.load("intel-isl/MiDaS", "transforms")

    if model_type == "DPT_Large" or model_type == "DPT_Hybrid":
        transform = midas_transforms.dpt_transform
    else:
        transform = midas_transforms.small_transform
    
    def estimate_depth(image):
        input_batch = transform(image).to(device)

        with torch.no_grad():
            prediction = midas(input_batch)

            prediction = torch.nn.functional.interpolate(
                prediction.unsqueeze(1),
                size=image.shape[:2],
                mode="bicubic",
                align_corners=False,
            ).squeeze()

        output = prediction.cpu().numpy()

        output= 65535 * (output - np.min(output))/(np.max(output) - np.min(output))
        
        return Image.fromarray(output.astype("int32")), output.min(), output.max()


elif model_arch == "zoe":
    # Zoe_N
    repo = "isl-org/ZoeDepth"
    model_zoe_n = torch.hub.load(repo, "ZoeD_N", pretrained=True)
    zoe = model_zoe_n.to(device)

    def estimate_depth(image):

        depth_tensor = zoe.infer_pil(image, output_type="tensor")
        output = depth_tensor.cpu().numpy()

        output_ = 65535 * (1 - (output - np.min(output))/(np.max(output) - np.min(output)))
        
        return Image.fromarray(output_.astype("int32")), output.min(), output.max()

def denormalize(image, min, max):
    image =  (image / 65535 - 1 ) * (min - max) + min
    return image

def read_content(file_path: str) -> str:
    """read the content of target file
    """
    with open(file_path, 'r', encoding='utf-8') as f:
        content = f.read()

    return content

def predict_images(dict, depth, prompt="", negative_prompt="", guidance_scale=7.5, steps=20, strength=1.0, scheduler="EulerDiscreteScheduler"):

    if negative_prompt == "":
        negative_prompt = None

    og_size = (dict["image"].shape[1], dict["image"].shape[0])

    init_image = cv2.resize(dict["image"], (512, 512))

    mask = Image.fromarray(cv2.resize(dict["mask"], (512, 512))[:,:,0])

    if (depth is None):
        depth_image, _, _ = estimate_depth(init_image)
        
    else:
        d_i = depth[:,:,0]
        depth_image = 65535 * (d_i - np.min(d_i))/(np.max(d_i) - np.min(d_i))
        depth_image = depth_image.astype("int32")
        depth_image = Image.fromarray(depth_image)

    init_image = Image.fromarray(init_image.astype("uint8"))    
    
    depth_image = depth_image.resize((512, 512))
    
    output = pipe(prompt = prompt, negative_prompt=negative_prompt, image=init_image, mask_image=mask, depth_image=depth_image, guidance_scale=guidance_scale, num_inference_steps=int(steps), strength=strength)
    
    depth_out = np.array(output.depth[0])

    output_depth_vis = (depth_out - np.min(depth_out)) / (np.max(depth_out) - np.min(depth_out)) * 255
    output_depth_vis = output_depth_vis.astype("uint8")
    output_depth = Image.fromarray(output_depth_vis)

    return output.rgb[0].resize(og_size), output_depth.resize(og_size), gr.update(visible=True)


css = '''
.gradio-container{max-width: 1100px !important}
#image_upload{min-height:400px}
#image_upload [data-testid="image"], #image_upload [data-testid="image"] > div{min-height: 400px}
#mask_radio .gr-form{background:transparent; border: none}
#word_mask{margin-top: .75em !important}
#word_mask textarea:disabled{opacity: 0.3}
.footer {margin-bottom: 45px;margin-top: 35px;text-align: center;border-bottom: 1px solid #e5e5e5}
.footer>p {font-size: .8rem; display: inline-block; padding: 0 10px;transform: translateY(10px);background: white}
.dark .footer {border-color: #303030}
.dark .footer>p {background: #0b0f19}
.acknowledgments h4{margin: 1.25em 0 .25em 0;font-weight: bold;font-size: 115%}
#image_upload .touch-none{display: flex}
@keyframes spin {
    from {
        transform: rotate(0deg);
    }
    to {
        transform: rotate(360deg);
    }
}
#share-btn-container {padding-left: 0.5rem !important; padding-right: 0.5rem !important; background-color: #000000; justify-content: center; align-items: center; border-radius: 9999px !important; max-width: 13rem; margin-left: auto;}
div#share-btn-container > div {flex-direction: row;background: black;align-items: center}
#share-btn-container:hover {background-color: #060606}
#share-btn {all: initial; color: #ffffff;font-weight: 600; cursor:pointer; font-family: 'IBM Plex Sans', sans-serif; margin-left: 0.5rem !important; padding-top: 0.5rem !important; padding-bottom: 0.5rem !important;right:0;}
#share-btn * {all: unset}
#share-btn-container div:nth-child(-n+2){width: auto !important;min-height: 0px !important;}
#share-btn-container .wrap {display: none !important}
#share-btn-container.hidden {display: none!important}
#prompt input{width: calc(100% - 160px);border-top-right-radius: 0px;border-bottom-right-radius: 0px;}
#run_button{position:absolute;margin-top: 11px;right: 0;margin-right: 0.8em;border-bottom-left-radius: 0px;
    border-top-left-radius: 0px;}
#prompt-container{margin-top:-18px;}
#prompt-container .form{border-top-left-radius: 0;border-top-right-radius: 0}
#image_upload{border-bottom-left-radius: 0px;border-bottom-right-radius: 0px}
'''

image_blocks = gr.Blocks(css=css, elem_id="total-container")

def create_vis_demo():
    with gr.Row():
        with gr.Column():
            image = gr.Image(source='upload', tool='sketch', elem_id="image_upload", type="numpy", label="Upload",height=400)
            depth = gr.Image(source='upload', elem_id="depth_upload", type="numpy", label="Upload",height=400)
            
            with gr.Row(elem_id="prompt-container", mobile_collapse=False, equal_height=True):
                with gr.Row():
                    prompt = gr.Textbox(placeholder="Your prompt (what you want in place of what is erased)", show_label=False, elem_id="prompt")
                    btn = gr.Button("Inpaint!", elem_id="run_button")
            
            with gr.Accordion(label="Advanced Settings", open=False):
                with gr.Row(mobile_collapse=False, equal_height=True):
                    guidance_scale = gr.Number(value=7.5, minimum=1.0, maximum=20.0, step=0.1, label="guidance_scale")
                    steps = gr.Number(value=20, minimum=10, maximum=30, step=1, label="steps")
                    strength = gr.Number(value=0.99, minimum=0.01, maximum=0.99, step=0.01, label="strength")
                    negative_prompt = gr.Textbox(label="negative_prompt", placeholder="Your negative prompt", info="what you don't want to see in the image")
                with gr.Row(mobile_collapse=False, equal_height=True):
                    schedulers = ["DEISMultistepScheduler", "HeunDiscreteScheduler", "EulerDiscreteScheduler", "DPMSolverMultistepScheduler", "DPMSolverMultistepScheduler-Karras", "DPMSolverMultistepScheduler-Karras-SDE"]
                    scheduler = gr.Dropdown(label="Schedulers", choices=schedulers, value="EulerDiscreteScheduler")
                
        with gr.Column():
            image_out = gr.Image(label="Output", elem_id="output-img", height=400)
            depth_out = gr.Image(label="Depth", elem_id="depth-img", height=400)

            with gr.Group(elem_id="share-btn-container", visible=False) as share_btn_container:
                community_icon = gr.HTML(community_icon_html)
                loading_icon = gr.HTML(loading_icon_html)
                share_button = gr.Button("Share to community", elem_id="share-btn",visible=True)

    btn.click(fn=predict_images, inputs=[image, depth, prompt, negative_prompt, guidance_scale, steps, strength, scheduler], outputs=[image_out, depth_out, share_btn_container], api_name='run')
    prompt.submit(fn=predict_images, inputs=[image, depth, prompt, negative_prompt, guidance_scale, steps, strength, scheduler], outputs=[image_out, depth_out, share_btn_container])
    share_button.click(None, [], [], _js=share_js)




def predict_images_3d(dict, depth, prompt="", negative_prompt="", guidance_scale=7.5, steps=20, strength=1.0, scheduler="EulerDiscreteScheduler", keep_edges=False):
    if negative_prompt == "":
        negative_prompt = None

    og_size = (dict["image"].shape[1], dict["image"].shape[0])
    init_image = cv2.resize(dict["image"], (512, 512))

    mask = Image.fromarray(cv2.resize(dict["mask"], (512, 512))[:,:,0])
    mask.save("temp_mask.jpg")

    if (depth is None):
        depth_image, min, max = estimate_depth(init_image)
        
    else:
        d_i = depth[:,:,0]
        depth_image = 65535 * (d_i - np.min(d_i))/(np.max(d_i) - np.min(d_i))
        depth_image = depth_image.astype("int32")
        depth_image = Image.fromarray(depth_image)

    init_image = Image.fromarray(init_image.astype("uint8"))    
    
    depth_image = depth_image.resize((512, 512))
    
    output = pipe(prompt = prompt, negative_prompt=negative_prompt, image=init_image, mask_image=mask, depth_image=depth_image, guidance_scale=guidance_scale, num_inference_steps=int(steps), strength=strength)
    
    # resize to original size
    #depth_image = depth_image.resize(og_size)
    #output_depth = output.depth[0].resize(og_size)
    
    depth_in = denormalize(np.array(depth_image), min, max)
    depth_out = denormalize(np.array(output.depth[0]), min, max)
    
    output_image = output.rgb[0]

    input_mesh = get_mesh(depth_in,init_image, keep_edges=keep_edges)
    output_mesh = get_mesh(depth_out, output_image, keep_edges=keep_edges)
    
    return input_mesh, output_mesh, gr.update(visible=True)

def create_3d_demo():

    gr.Markdown("### Image to 3D mesh")

    with gr.Row():
        with gr.Row():
            with gr.Column():
                image = gr.Image(source='upload', tool='sketch', elem_id="image_upload", type="numpy", label="Upload",height=400, shape=(512,512))
                depth = gr.Image(source='upload', elem_id="depth_upload", type="numpy", label="Upload",height=400, shape=(512,512))
                checkbox = gr.Checkbox(label="Keep occlusion edges", value=False)
                prompt = gr.Textbox(placeholder="Your prompt (what you want in place of what is erased)", show_label=False, elem_id="prompt")
                
                with gr.Accordion(label="Advanced Settings", open=False):
                    with gr.Row(mobile_collapse=False, equal_height=True):
                        guidance_scale = gr.Number(value=7.5, minimum=1.0, maximum=20.0, step=0.1, label="guidance_scale")
                        steps = gr.Number(value=20, minimum=10, maximum=30, step=1, label="steps")
                        strength = gr.Number(value=0.99, minimum=0.01, maximum=0.99, step=0.01, label="strength")
                        negative_prompt = gr.Textbox(label="negative_prompt", placeholder="Your negative prompt", info="what you don't want to see in the image")
                    with gr.Row(mobile_collapse=False, equal_height=True):
                        schedulers = ["DEISMultistepScheduler", "HeunDiscreteScheduler", "EulerDiscreteScheduler", "DPMSolverMultistepScheduler", "DPMSolverMultistepScheduler-Karras", "DPMSolverMultistepScheduler-Karras-SDE"]
                        scheduler = gr.Dropdown(label="Schedulers", choices=schedulers, value="EulerDiscreteScheduler")
        
        with gr.Row() as share_btn_container:
            with gr.Column():
                result_og = gr.Model3D(label="original 3d reconstruction", clear_color=[
                                                        1.0, 1.0, 1.0, 1.0])
                
                result_new = gr.Model3D(label="inpainted 3d reconstruction", clear_color=[
                                                        1.0, 1.0, 1.0, 1.0])
            
    
    submit = gr.Button("Submit")
    submit.click(fn=predict_images_3d, inputs=[image, depth, prompt, negative_prompt, guidance_scale, steps, strength, scheduler, checkbox], outputs=[result_og, result_new, share_btn_container], api_name='run')


with image_blocks as demo:
    with gr.Tab("Image", default=True):
        create_vis_demo()
    with gr.Tab("3D"):
        create_3d_demo()
    
    gr.HTML(read_content("header.html"))
    

image_blocks.queue(max_size=25).launch()