import torch import os import numpy as np import cv2 import PIL.Image as pil_img import subprocess subprocess.run( 'pip install networkx==2.5' .split() ) import gradio as gr import trimesh import plotly.graph_objects as go from models.deco import DECO from common import constants if torch.cuda.is_available(): device = torch.device('cuda') else: device = torch.device('cpu') description = ''' ### DECO: Dense Estimation of 3D Human-Scene Contact in the Wild (ICCV 2023, Oral)

  • Colab Notebook Google Colab

#### Citation ``` @InProceedings{tripathi2023deco, author = {Tripathi, Shashank and Chatterjee, Agniv and Passy, Jean-Claude and Yi, Hongwei and Tzionas, Dimitrios and Black, Michael J.}, title = {{DECO}: Dense Estimation of {3D} Human-Scene Contact In The Wild}, booktitle = {Proceedings of the IEEE/CVF International Conference on Computer Vision (ICCV)}, month = {October}, year = {2023}, pages = {8001-8013} } ```
More #### Acknowledgments: - [ECON](https://huggingface.co/spaces/Yuliang/ECON)
''' DEFAULT_LIGHTING = dict( ambient=0.6, diffuse=0.5, fresnel=0.01, specular=0.1, roughness=0.001) DEFAULT_LIGHT_POSITION = dict(x=6, y=0, z=10) def initiate_model(model_path): deco_model = DECO('hrnet', True, device) print(f'Loading weights from {model_path}') checkpoint = torch.load(model_path) deco_model.load_state_dict(checkpoint['deco'], strict=True) deco_model.eval() return deco_model def create_layout(dummy, camera=None): if camera is None: camera = dict( up=dict(x=0, y=1, z=0), center=dict(x=0, y=0, z=0), eye=dict(x=dummy.x.mean(), y=0, z=3), projection=dict(type='perspective')) layout = dict( scene={ "xaxis": { 'showgrid': False, 'zeroline': False, 'visible': False, "range": [dummy.x.min(), dummy.x.max()] }, "yaxis": { 'showgrid': False, 'zeroline': False, 'visible': False, "range": [dummy.y.min(), dummy.y.max()] }, "zaxis": { 'showgrid': False, 'zeroline': False, 'visible': False, "range": [dummy.z.min(), dummy.z.max()] }, }, autosize=False, width=750, height=1000, scene_camera=camera, scene_aspectmode="data", clickmode="event+select", margin={'l': 0, 't': 0} ) return layout def create_fig(dummy, camera=None): fig = go.Figure( data=dummy.mesh_3d(), layout=create_layout(dummy, camera)) return fig class Dummy: def __init__(self, mesh_path): """A simple polygonal dummy with colored patches.""" self._load_trimesh(mesh_path) def _load_trimesh(self, path): """Load a mesh given a path to a .PLY file.""" self._trimesh = trimesh.load(path, process=False) self._vertices = np.array(self._trimesh.vertices) self._faces = np.array(self._trimesh.faces) self.colors = self._trimesh.visual.vertex_colors @property def vertices(self): """All the mesh vertices.""" return self._vertices @property def faces(self): """All the mesh faces.""" return self._faces @property def n_vertices(self): """Number of vertices in a mesh.""" return self._vertices.shape[0] @property def n_faces(self): """Number of faces in a mesh.""" return self._faces.shape[0] @property def x(self): """An array of vertex x coordinates""" return self._vertices[:, 0] @property def y(self): """An array of vertex y coordinates""" return self._vertices[:, 1] @property def z(self): """An array of vertex z coordinates""" return self._vertices[:, 2] @property def i(self): """An array of the first face vertices""" return self._faces[:, 0] @property def j(self): """An array of the second face vertices""" return self._faces[:, 1] @property def k(self): """An array of the third face vertices""" return self._faces[:, 2] @property def default_selection(self): """Default patch selection mask.""" return dict(vertices=[]) def mesh_3d( self, lighting=DEFAULT_LIGHTING, light_position=DEFAULT_LIGHT_POSITION ): """Construct a Mesh3D object give a clickmask for patch coloring.""" return go.Mesh3d( x=self.x, y=self.y, z=self.z, i=self.i, j=self.j, k=self.k, vertexcolor=self.colors, lighting=lighting, lightposition=light_position, hoverinfo='none') def main(pil_img, out_dir='demo_out', model_path='checkpoint/deco_best.pth', mesh_colour=[130, 130, 130, 255], annot_colour=[0, 255, 0, 255]): deco_model = initiate_model(model_path) smpl_path = os.path.join(constants.SMPL_MODEL_DIR, 'smpl_neutral_tpose.ply') img = np.array(pil_img) img = cv2.resize(img, (256, 256), cv2.INTER_CUBIC) img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) img = img.transpose(2,0,1)/255.0 img = img[np.newaxis,:,:,:] img = torch.tensor(img, dtype = torch.float32).to(device) with torch.no_grad(): cont, _, _ = deco_model(img) cont = cont.detach().cpu().numpy().squeeze() cont_smpl = [] for indx, i in enumerate(cont): if i >= 0.5: cont_smpl.append(indx) img = img.detach().cpu().numpy() img = np.transpose(img[0], (1, 2, 0)) img = img * 255 img = img.astype(np.uint8) contact_smpl = np.zeros((1, 1, 6890)) contact_smpl[0][0][cont_smpl] = 1 body_model_smpl = trimesh.load(smpl_path, process=False) for vert in range(body_model_smpl.visual.vertex_colors.shape[0]): body_model_smpl.visual.vertex_colors[vert] = mesh_colour body_model_smpl.visual.vertex_colors[cont_smpl] = annot_colour mesh_out_dir = os.path.join(out_dir, 'Preds') os.makedirs(mesh_out_dir, exist_ok=True) print(f'Saving mesh to {mesh_out_dir}') body_model_smpl.export(os.path.join(mesh_out_dir, 'pred.obj')) dummy = Dummy(os.path.join(mesh_out_dir, 'pred.obj')) fig = create_fig(dummy) return fig, os.path.join(mesh_out_dir, 'pred.obj') with gr.Blocks(title="DECO", css=".gradio-container") as demo: gr.Markdown(description) gr.HTML("""

DECO Demo

""") with gr.Row(): with gr.Column(): input_image = gr.Image(label="Input image", type="pil") with gr.Column(): output_image = gr.Plot(label="Renders") output_meshes = gr.File(label="3D meshes") gr.HTML("""
""") with gr.Row(): send_btn = gr.Button("Infer") send_btn.click(fn=main, inputs=[input_image], outputs=[output_image, output_meshes]) example_images = gr.Examples([ ['/home/user/app/example_images/213.jpg'], ['/home/user/app/example_images/pexels-photo-207569.webp'], ['/home/user/app/example_images/pexels-photo-3622517.webp'], ['/home/user/app/example_images/pexels-photo-15732209.jpeg'], ], inputs=[input_image]) demo.launch(debug=True)