import gradio as gr import torch from diffuserslocal.src.diffusers import UNet2DConditionModel from share_btn import community_icon_html, loading_icon_html, share_js from diffuserslocal.src.diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_ldm3d_inpaint import StableDiffusionLDM3DInpaintPipeline from PIL import Image import numpy as np import cv2 from functools import partial import tempfile from mesh import get_mesh device = "cuda" if torch.cuda.is_available() else "cpu" model_arch = "zoe" # Inpainting pipeline unet = UNet2DConditionModel.from_pretrained("pablodawson/ldm3d-inpainting", cache_dir="cache", subfolder="unet") pipe = StableDiffusionLDM3DInpaintPipeline.from_pretrained("Intel/ldm3d-4c", cache_dir="cache" ).to(device) # Depth estimation model_type = "DPT_Large" # MiDaS v3 - Large (highest accuracy, slowest inference speed) #model_type = "DPT_Hybrid" # MiDaS v3 - Hybrid (medium accuracy, medium inference speed) #model_type = "MiDaS_small" # MiDaS v2.1 - Small (lowest accuracy, highest inference speed) if model_arch == "midas": midas = torch.hub.load("intel-isl/MiDaS", model_type) midas.to(device) midas.eval() midas_transforms = torch.hub.load("intel-isl/MiDaS", "transforms") if model_type == "DPT_Large" or model_type == "DPT_Hybrid": transform = midas_transforms.dpt_transform else: transform = midas_transforms.small_transform def estimate_depth(image): input_batch = transform(image).to(device) with torch.no_grad(): prediction = midas(input_batch) prediction = torch.nn.functional.interpolate( prediction.unsqueeze(1), size=image.shape[:2], mode="bicubic", align_corners=False, ).squeeze() output = prediction.cpu().numpy() output= 65535 * (output - np.min(output))/(np.max(output) - np.min(output)) return Image.fromarray(output.astype("int32")), output.min(), output.max() elif model_arch == "zoe": # Zoe_N repo = "isl-org/ZoeDepth" model_zoe_n = torch.hub.load(repo, "ZoeD_N", pretrained=True) zoe = model_zoe_n.to(device) def estimate_depth(image): depth_tensor = zoe.infer_pil(image, output_type="tensor") output = depth_tensor.cpu().numpy() output_ = 65535 * (1 - (output - np.min(output))/(np.max(output) - np.min(output))) return Image.fromarray(output_.astype("int32")), output.min(), output.max() def denormalize(image, min, max): image = (image / 65535 - 1 ) * (min - max) + min return image def read_content(file_path: str) -> str: """read the content of target file """ with open(file_path, 'r', encoding='utf-8') as f: content = f.read() return content def predict_images(dict, depth, prompt="", negative_prompt="", guidance_scale=7.5, steps=20, strength=1.0, scheduler="EulerDiscreteScheduler"): if negative_prompt == "": negative_prompt = None og_size = (dict["image"].shape[1], dict["image"].shape[0]) init_image = cv2.resize(dict["image"], (512, 512)) mask = Image.fromarray(cv2.resize(dict["mask"], (512, 512))[:,:,0]) if (depth is None): depth_image, _, _ = estimate_depth(init_image) else: d_i = depth[:,:,0] depth_image = 65535 * (d_i - np.min(d_i))/(np.max(d_i) - np.min(d_i)) depth_image = depth_image.astype("int32") depth_image = Image.fromarray(depth_image) init_image = Image.fromarray(init_image.astype("uint8")) depth_image = depth_image.resize((512, 512)) output = pipe(prompt = prompt, negative_prompt=negative_prompt, image=init_image, mask_image=mask, depth_image=depth_image, guidance_scale=guidance_scale, num_inference_steps=int(steps), strength=strength) depth_out = np.array(output.depth[0]) output_depth_vis = (depth_out - np.min(depth_out)) / (np.max(depth_out) - np.min(depth_out)) * 255 output_depth_vis = output_depth_vis.astype("uint8") output_depth = Image.fromarray(output_depth_vis) return output.rgb[0].resize(og_size), output_depth.resize(og_size), gr.update(visible=True) css = ''' .gradio-container{max-width: 1100px !important} #image_upload{min-height:400px} #image_upload [data-testid="image"], #image_upload [data-testid="image"] > div{min-height: 400px} #mask_radio .gr-form{background:transparent; border: none} #word_mask{margin-top: .75em !important} #word_mask textarea:disabled{opacity: 0.3} .footer {margin-bottom: 45px;margin-top: 35px;text-align: center;border-bottom: 1px solid #e5e5e5} .footer>p {font-size: .8rem; display: inline-block; padding: 0 10px;transform: translateY(10px);background: white} .dark .footer {border-color: #303030} .dark .footer>p {background: #0b0f19} .acknowledgments h4{margin: 1.25em 0 .25em 0;font-weight: bold;font-size: 115%} #image_upload .touch-none{display: flex} @keyframes spin { from { transform: rotate(0deg); } to { transform: rotate(360deg); } } #share-btn-container {padding-left: 0.5rem !important; padding-right: 0.5rem !important; background-color: #000000; justify-content: center; align-items: center; border-radius: 9999px !important; max-width: 13rem; margin-left: auto;} div#share-btn-container > div {flex-direction: row;background: black;align-items: center} #share-btn-container:hover {background-color: #060606} #share-btn {all: initial; color: #ffffff;font-weight: 600; cursor:pointer; font-family: 'IBM Plex Sans', sans-serif; margin-left: 0.5rem !important; padding-top: 0.5rem !important; padding-bottom: 0.5rem !important;right:0;} #share-btn * {all: unset} #share-btn-container div:nth-child(-n+2){width: auto !important;min-height: 0px !important;} #share-btn-container .wrap {display: none !important} #share-btn-container.hidden {display: none!important} #prompt input{width: calc(100% - 160px);border-top-right-radius: 0px;border-bottom-right-radius: 0px;} #run_button{position:absolute;margin-top: 11px;right: 0;margin-right: 0.8em;border-bottom-left-radius: 0px; border-top-left-radius: 0px;} #prompt-container{margin-top:-18px;} #prompt-container .form{border-top-left-radius: 0;border-top-right-radius: 0} #image_upload{border-bottom-left-radius: 0px;border-bottom-right-radius: 0px} ''' image_blocks = gr.Blocks(css=css, elem_id="total-container") def create_vis_demo(): with gr.Row(): with gr.Column(): image = gr.Image(source='upload', tool='sketch', elem_id="image_upload", type="numpy", label="Upload",height=400) depth = gr.Image(source='upload', elem_id="depth_upload", type="numpy", label="Upload",height=400) with gr.Row(elem_id="prompt-container", mobile_collapse=False, equal_height=True): with gr.Row(): prompt = gr.Textbox(placeholder="Your prompt (what you want in place of what is erased)", show_label=False, elem_id="prompt") btn = gr.Button("Inpaint!", elem_id="run_button") with gr.Accordion(label="Advanced Settings", open=False): with gr.Row(mobile_collapse=False, equal_height=True): guidance_scale = gr.Number(value=7.5, minimum=1.0, maximum=20.0, step=0.1, label="guidance_scale") steps = gr.Number(value=20, minimum=10, maximum=30, step=1, label="steps") strength = gr.Number(value=0.99, minimum=0.01, maximum=0.99, step=0.01, label="strength") negative_prompt = gr.Textbox(label="negative_prompt", placeholder="Your negative prompt", info="what you don't want to see in the image") with gr.Row(mobile_collapse=False, equal_height=True): schedulers = ["DEISMultistepScheduler", "HeunDiscreteScheduler", "EulerDiscreteScheduler", "DPMSolverMultistepScheduler", "DPMSolverMultistepScheduler-Karras", "DPMSolverMultistepScheduler-Karras-SDE"] scheduler = gr.Dropdown(label="Schedulers", choices=schedulers, value="EulerDiscreteScheduler") with gr.Column(): image_out = gr.Image(label="Output", elem_id="output-img", height=400) depth_out = gr.Image(label="Depth", elem_id="depth-img", height=400) with gr.Group(elem_id="share-btn-container", visible=False) as share_btn_container: community_icon = gr.HTML(community_icon_html) loading_icon = gr.HTML(loading_icon_html) share_button = gr.Button("Share to community", elem_id="share-btn",visible=True) btn.click(fn=predict_images, inputs=[image, depth, prompt, negative_prompt, guidance_scale, steps, strength, scheduler], outputs=[image_out, depth_out, share_btn_container], api_name='run') prompt.submit(fn=predict_images, inputs=[image, depth, prompt, negative_prompt, guidance_scale, steps, strength, scheduler], outputs=[image_out, depth_out, share_btn_container]) share_button.click(None, [], [], _js=share_js) def predict_images_3d(dict, depth, prompt="", negative_prompt="", guidance_scale=7.5, steps=20, strength=1.0, scheduler="EulerDiscreteScheduler", keep_edges=False): if negative_prompt == "": negative_prompt = None og_size = (dict["image"].shape[1], dict["image"].shape[0]) init_image = cv2.resize(dict["image"], (512, 512)) mask = Image.fromarray(cv2.resize(dict["mask"], (512, 512))[:,:,0]) mask.save("temp_mask.jpg") if (depth is None): depth_image, min, max = estimate_depth(init_image) else: d_i = depth[:,:,0] depth_image = 65535 * (d_i - np.min(d_i))/(np.max(d_i) - np.min(d_i)) depth_image = depth_image.astype("int32") depth_image = Image.fromarray(depth_image) init_image = Image.fromarray(init_image.astype("uint8")) depth_image = depth_image.resize((512, 512)) output = pipe(prompt = prompt, negative_prompt=negative_prompt, image=init_image, mask_image=mask, depth_image=depth_image, guidance_scale=guidance_scale, num_inference_steps=int(steps), strength=strength) # resize to original size #depth_image = depth_image.resize(og_size) #output_depth = output.depth[0].resize(og_size) depth_in = denormalize(np.array(depth_image), min, max) depth_out = denormalize(np.array(output.depth[0]), min, max) output_image = output.rgb[0] input_mesh = get_mesh(depth_in,init_image, keep_edges=keep_edges) output_mesh = get_mesh(depth_out, output_image, keep_edges=keep_edges) return input_mesh, output_mesh, gr.update(visible=True) def create_3d_demo(): gr.Markdown("### Image to 3D mesh") with gr.Row(): with gr.Row(): with gr.Column(): image = gr.Image(source='upload', tool='sketch', elem_id="image_upload", type="numpy", label="Upload",height=400, shape=(512,512)) depth = gr.Image(source='upload', elem_id="depth_upload", type="numpy", label="Upload",height=400, shape=(512,512)) checkbox = gr.Checkbox(label="Keep occlusion edges", value=False) prompt = gr.Textbox(placeholder="Your prompt (what you want in place of what is erased)", show_label=False, elem_id="prompt") with gr.Accordion(label="Advanced Settings", open=False): with gr.Row(mobile_collapse=False, equal_height=True): guidance_scale = gr.Number(value=7.5, minimum=1.0, maximum=20.0, step=0.1, label="guidance_scale") steps = gr.Number(value=20, minimum=10, maximum=30, step=1, label="steps") strength = gr.Number(value=0.99, minimum=0.01, maximum=0.99, step=0.01, label="strength") negative_prompt = gr.Textbox(label="negative_prompt", placeholder="Your negative prompt", info="what you don't want to see in the image") with gr.Row(mobile_collapse=False, equal_height=True): schedulers = ["DEISMultistepScheduler", "HeunDiscreteScheduler", "EulerDiscreteScheduler", "DPMSolverMultistepScheduler", "DPMSolverMultistepScheduler-Karras", "DPMSolverMultistepScheduler-Karras-SDE"] scheduler = gr.Dropdown(label="Schedulers", choices=schedulers, value="EulerDiscreteScheduler") with gr.Row() as share_btn_container: with gr.Column(): result_og = gr.Model3D(label="original 3d reconstruction", clear_color=[ 1.0, 1.0, 1.0, 1.0]) result_new = gr.Model3D(label="inpainted 3d reconstruction", clear_color=[ 1.0, 1.0, 1.0, 1.0]) submit = gr.Button("Submit") submit.click(fn=predict_images_3d, inputs=[image, depth, prompt, negative_prompt, guidance_scale, steps, strength, scheduler, checkbox], outputs=[result_og, result_new, share_btn_container], api_name='run') with image_blocks as demo: with gr.Tab("Image", default=True): create_vis_demo() with gr.Tab("3D"): create_3d_demo() gr.HTML(read_content("header.html")) image_blocks.queue(max_size=25).launch()