import spaces import os import numpy as np from PIL import Image from omegaconf import OmegaConf from functools import partial import gradio as gr from huggingface_hub import hf_hub_download import torch from torchvision import transforms import rembg import cv2 from pytorch_lightning import seed_everything from src.visualizer import CameraVisualizer from src.pose_estimation import load_model_from_config, estimate_poses, estimate_elevs from src.pose_funcs import find_optimal_poses from src.utils import spherical_to_cartesian, elu_to_c2w if torch.cuda.is_available(): _device_ = 'cuda:0' else: _device_ = 'cpu' _config_path_ = 'src/configs/sd-objaverse-finetune-c_concat-256.yaml' _ckpt_path_ = hf_hub_download(repo_id='tokenid/ID-Pose', filename='ckpts/zero123-xl.ckpt', repo_type='model') _matcher_ckpt_path_ = hf_hub_download(repo_id='tokenid/ID-Pose', filename='ckpts/indoor_ds_new.ckpt', repo_type='model') _config_ = OmegaConf.load(_config_path_) _model_ = load_model_from_config(_config_, _ckpt_path_, device='cpu') _model_ = _model_.to(_device_) _model_.eval() def rgba_to_rgb(img): assert img.mode == 'RGBA' img = np.asarray(img, dtype=np.float32) img[:, :, :3] = img[:, :, :3] * (img[..., 3:]/255.) + (255-img[..., 3:]) img = img.clip(0, 255).astype(np.uint8) return Image.fromarray(img[:, :, :3]) def remove_background(image, rembg_session = None, force = False, **rembg_kwargs): do_remove = True if image.mode == "RGBA" and image.getextrema()[3][0] < 255: do_remove = False do_remove = do_remove or force if do_remove: image = rembg.remove(image, session=rembg_session, **rembg_kwargs) return image def group_recenter(images, ratio=1.5, mask_thres=127, bkg_color=[255, 255, 255, 255]): ws = [] hs = [] images = [ np.asarray(img) for img in images ] for img in images: alpha = img[:, :, 3] yy, xx = np.where(alpha > mask_thres) y0, y1 = yy.min(), yy.max() x0, x1 = xx.min(), xx.max() ws.append(float(x1 - x0) / img.shape[0]) hs.append(float(y1 - y0) / img.shape[1]) sz_w = np.max(ws) sz_h = np.max(hs) sz = max(ratio*sz_w, ratio*sz_h) out_rgbs = [] for rgba in images: rgb = rgba[:, :, :3] alpha = rgba[:, :, 3] yy, xx = np.where(alpha > mask_thres) y0, y1 = yy.min(), yy.max() x0, x1 = xx.min(), xx.max() height, width, chn = rgb.shape cy = (y0 + y1) // 2 cx = (x0 + x1) // 2 y0 = cy - int(np.floor(sz * rgba.shape[0] / 2)) y1 = cy + int(np.ceil(sz * rgba.shape[0] / 2)) x0 = cx - int(np.floor(sz * rgba.shape[1] / 2)) x1 = cx + int(np.ceil(sz * rgba.shape[1] / 2)) out = rgba[ max(y0, 0) : min(y1, height) , max(x0, 0) : min(x1, width), : ].copy() pads = [(max(0-y0, 0), max(y1-height, 0)), (max(0-x0, 0), max(x1-width, 0)), (0, 0)] out = np.pad(out, pads, mode='constant', constant_values=0) out[:, :, :3] = out[:, :, :3] * (out[..., 3:]/255.) + np.array(bkg_color)[None, None, :3] * (1-out[..., 3:]/255.) out[:, :, -1] = bkg_color[-1] out = cv2.resize(out.astype(np.uint8), (256, 256)) out = out[:, :, :3] out_rgbs.append(out) return out_rgbs def run_preprocess(image1, image2, preprocess_chk, seed_value): seed_everything(seed_value) if preprocess_chk: rembg_session = rembg.new_session() image1 = remove_background(image1, force=True, rembg_session = rembg_session) image2 = remove_background(image2, force=True, rembg_session = rembg_session) rgbs = group_recenter([image1, image2]) image1 = Image.fromarray(rgbs[0]) image2 = Image.fromarray(rgbs[1]) return image1, image2 def image_to_tensor(img, width=256, height=256): img = transforms.ToTensor()(img).unsqueeze(0) img = img * 2 - 1 img = transforms.functional.resize(img, [height, width]) return img @spaces.GPU(duration=110) def run_pose_exploration(image1, image2, probe_bsz, adj_bsz, adj_iters, seed_value): seed_everything(seed_value) image1 = image_to_tensor(image1).to(_device_) image2 = image_to_tensor(image2).to(_device_) images = [image1, image2] elevs, elev_ranges = estimate_elevs( _model_, images, est_type='all', matcher_ckpt_path=_matcher_ckpt_path_ ) anchor_polar = elevs[0] if torch.mean(torch.abs(image1 - image2)) < 0.005: theta = azimuth = radius = 0 print('Identical images found!') else: noise = np.random.randn(probe_bsz, 4, 32, 32) result_poses, aux_data = estimate_poses( _model_, images, seed_cand_num=8, explore_type='triangular', refine_type='triangular', probe_ts_range=[0.2, 0.21], ts_range=[0.2, 0.21], probe_bsz=probe_bsz, adjust_factor=10.0, adjust_iters=adj_iters, adjust_bsz=adj_bsz, refine_factor=1.0, refine_iters=0, refine_bsz=4, noise=noise, elevs=elevs, elev_ranges=elev_ranges ) theta, azimuth, radius = result_poses[0] if anchor_polar is None: anchor_polar = np.pi/2 explored_sph = (theta, azimuth, radius) return anchor_polar, explored_sph @spaces.GPU(duration=110) def run_pose_refinement(image1, image2, est_result, refine_iters, seed_value): seed_everything(seed_value) anchor_polar = est_result[0] explored_sph = est_result[1] images = [image_to_tensor(image1).to(_device_), image_to_tensor(image2).to(_device_)] images = [ img.permute(0, 2, 3, 1) for img in images ] out_poses, _, loss = find_optimal_poses( _model_, images, 1.0, bsz=1, n_iter=refine_iters, init_poses={1: explored_sph}, ts_range=[0.2, 0.21], combinations=[(0, 1), (1, 0)], avg_last_n=20, print_n=100 ) final_sph = out_poses[0] theta, azimuth, radius = final_sph xyz0 = spherical_to_cartesian((anchor_polar, 0., 4.)) c2w0 = elu_to_c2w(xyz0, np.zeros(3), np.array([0., 0., 1.])) xyz1 = spherical_to_cartesian((theta + anchor_polar, 0. + azimuth, 4. + radius)) c2w1 = elu_to_c2w(xyz1, np.zeros(3), np.array([0., 0., 1.])) cam_vis = CameraVisualizer([c2w0, c2w1], ['Image 1', 'Image 2'], ['red', 'blue'], images=[np.asarray(image1, dtype=np.uint8), np.asarray(image2, dtype=np.uint8)]) fig = cam_vis.update_figure(5, base_radius=-1.2, font_size=16, show_background=True, show_grid=True, show_ticklabels=True) return (anchor_polar, final_sph), fig def run_example(image1, image2): image1, image2 = run_preprocess(image1, image2, True, 0) anchor_polar, explored_sph = run_pose_exploration(image1, image2, 16, 4, 10, 0) return (anchor_polar, explored_sph), image1, image2 def run_or_visualize(image1, image2, probe_bsz, adj_bsz, adj_iters, seed_value, est_result): if est_result is None: anchor_polar, explored_sph = run_pose_exploration(image1, image2, probe_bsz, adj_bsz, adj_iters, seed_value) else: anchor_polar = est_result[0] explored_sph = est_result[1] print('Using cache result.') xyz0 = spherical_to_cartesian((anchor_polar, 0., 4.)) c2w0 = elu_to_c2w(xyz0, np.zeros(3), np.array([0., 0., 1.])) xyz1 = spherical_to_cartesian((explored_sph[0] + anchor_polar, 0. + explored_sph[1], 4. + explored_sph[2])) c2w1 = elu_to_c2w(xyz1, np.zeros(3), np.array([0., 0., 1.])) cam_vis = CameraVisualizer([c2w0, c2w1], ['Image 1', 'Image 2'], ['red', 'blue'], images=[np.asarray(image1, dtype=np.uint8), np.asarray(image2, dtype=np.uint8)]) fig = cam_vis.update_figure(5, base_radius=-1.2, font_size=16, show_background=True, show_grid=True, show_ticklabels=True) return (anchor_polar, explored_sph), fig, gr.update(interactive=True) _HEADER_ = ''' # Official 🤗 Gradio Demo for [ID-Pose: Sparse-view Camera Pose Estimation By Inverting Diffusion Models](https://github.com/xt4d/id-pose) - ID-Pose accepts input images with NO overlapping appearance. - The estimation takes about 1 minute. ZeroGPU may be halted during processing due to quota restrictions. ''' _FOOTER_ = ''' [Project Page](https://xt4d.github.io/id-pose-web/) | ⭐ [Github](https://github.com/xt4d/id-pose) ⭐ [![GitHub Stars](https://img.shields.io/github/stars/xt4d/id-pose?style=social)](https://github.com/xt4d/id-pose) --- ''' _CITE_ = r""" ```bibtex @article{cheng2023id, title={ID-Pose: Sparse-view Camera Pose Estimation by Inverting Diffusion Models}, author={Cheng, Weihao and Cao, Yan-Pei and Shan, Ying}, journal={arXiv preprint arXiv:2306.17140}, year={2023} } ``` """ def run_demo(): demo = gr.Blocks(title='ID-Pose: Sparse-view Camera Pose Estimation By Inverting Diffusion Models') with demo: est_result = gr.JSON(visible=False) gr.Markdown(_HEADER_) with gr.Row(variant='panel'): with gr.Column(scale=1): with gr.Row(): with gr.Column(min_width=280): input_image1 = gr.Image(type='pil', image_mode='RGBA', label='Input Image 1', width=280) with gr.Column(min_width=280): input_image2 = gr.Image(type='pil', image_mode='RGBA', label='Input Image 2', width=280) with gr.Row(): with gr.Column(min_width=280): processed_image1 = gr.Image(type='numpy', image_mode='RGB', label='Processed Image 1', width=280, interactive=False) with gr.Column(min_width=280): processed_image2 = gr.Image(type='numpy', image_mode='RGB', label='Processed Image 2', width=280, interactive=False) with gr.Row(): preprocess_chk = gr.Checkbox(True, label='Remove background and recenter object') with gr.Accordion('Advanced options', open=False): probe_bsz = gr.Slider(4, 32, value=16, step=4, label='Probe Batch Size') adj_bsz = gr.Slider(1, 8, value=4, step=1, label='Adjust Batch Size') adj_iters = gr.Slider(1, 20, value=10, step=1, label='Adjust Iterations') seed_value = gr.Number(value=0, label="Seed Value", precision=0) with gr.Row(): run_btn = gr.Button('Estimate', variant='primary', interactive=True) with gr.Row(): refine_iters = gr.Slider(0, 1000, value=0, step=50, label='Refinement Iterations') with gr.Row(): refine_btn = gr.Button('Refine', variant='primary', interactive=False) with gr.Row(): gr.Markdown(_FOOTER_) with gr.Row(): gr.Markdown(_CITE_) with gr.Column(scale=1.4): with gr.Row(): vis_output = gr.Plot(label='Camera Pose Results: anchor (red) and target (blue)') with gr.Row(): with gr.Column(min_width=200): gr.Examples( examples = [ ['data/gradio_demo/duck_0.png', 'data/gradio_demo/duck_1.png'], ['data/gradio_demo/chair_0.png', 'data/gradio_demo/chair_1.png'], ['data/gradio_demo/foosball_0.png', 'data/gradio_demo/foosball_1.png'], ['data/gradio_demo/bunny_0.png', 'data/gradio_demo/bunny_1.png'], ['data/gradio_demo/circo_0.png', 'data/gradio_demo/circo_1.png'], ], inputs=[input_image1, input_image2], fn=run_example, outputs=[est_result, processed_image1, processed_image2], label='Examples (Captured)', cache_examples='lazy', examples_per_page=5 ) with gr.Column(min_width=200): gr.Examples( examples = [ ['data/gradio_demo/arc_0.png', 'data/gradio_demo/arc_1.png'], ['data/gradio_demo/husky_0.png', 'data/gradio_demo/husky_1.png'], ['data/gradio_demo/cybertruck_0.png', 'data/gradio_demo/cybertruck_1.png'], ['data/gradio_demo/plane_0.png', 'data/gradio_demo/plane_1.png'], ['data/gradio_demo/christ_0.png', 'data/gradio_demo/christ_1.png'], ], inputs=[input_image1, input_image2], fn=run_example, outputs=[est_result, processed_image1, processed_image2], label='Examples (Internet)', cache_examples='lazy', examples_per_page=5 ) with gr.Column(min_width=200): gr.Examples( examples = [ ['data/gradio_demo/status_0.png', 'data/gradio_demo/status_1.png'], ['data/gradio_demo/cat_0.png', 'data/gradio_demo/cat_1.png'], ['data/gradio_demo/ferrari_0.png', 'data/gradio_demo/ferrari_1.png'], ['data/gradio_demo/elon_0.png', 'data/gradio_demo/elon_1.png'], ['data/gradio_demo/ride_horse_0.png', 'data/gradio_demo/ride_horse_1.png'], ], inputs=[input_image1, input_image2], fn=run_example, outputs=[est_result, processed_image1, processed_image2], label='Examples (Generated)', cache_examples='lazy', examples_per_page=5 ) run_btn.click( fn=run_preprocess, inputs=[input_image1, input_image2, preprocess_chk, seed_value], outputs=[processed_image1, processed_image2], ).success( fn=run_or_visualize, inputs=[processed_image1, processed_image2, probe_bsz, adj_bsz, adj_iters, seed_value, est_result], outputs=[est_result, vis_output, refine_btn] ) refine_btn.click( fn=run_pose_refinement, inputs=[processed_image1, processed_image2, est_result, refine_iters, seed_value], outputs=[est_result, vis_output] ) input_image1.clear( fn=lambda: None, outputs=[est_result] ) input_image2.clear( fn=lambda: None, outputs=[est_result] ) demo.launch() if __name__ == '__main__': run_demo()