import torch
import os
import numpy as np
import cv2
import PIL.Image as pil_img
import subprocess
subprocess.run(
'pip install networkx==2.5'
.split()
)
import gradio as gr
import trimesh
import plotly.graph_objects as go
from models.deco import DECO
from common import constants
if torch.cuda.is_available():
device = torch.device('cuda')
else:
device = torch.device('cpu')
description = '''
### DECO: Dense Estimation of 3D Human-Scene Contact in the Wild (ICCV 2023, Oral)
- Colab Notebook
|
#### Citation
```
@InProceedings{tripathi2023deco,
author = {Tripathi, Shashank and Chatterjee, Agniv and Passy, Jean-Claude and Yi, Hongwei and Tzionas, Dimitrios and Black, Michael J.},
title = {{DECO}: Dense Estimation of {3D} Human-Scene Contact In The Wild},
booktitle = {Proceedings of the IEEE/CVF International Conference on Computer Vision (ICCV)},
month = {October},
year = {2023},
pages = {8001-8013}
}
```
More
#### Acknowledgments:
- [ECON](https://huggingface.co/spaces/Yuliang/ECON)
'''
DEFAULT_LIGHTING = dict(
ambient=0.6,
diffuse=0.5,
fresnel=0.01,
specular=0.1,
roughness=0.001)
DEFAULT_LIGHT_POSITION = dict(x=6, y=0, z=10)
def initiate_model(model_path):
deco_model = DECO('hrnet', True, device)
print(f'Loading weights from {model_path}')
checkpoint = torch.load(model_path)
deco_model.load_state_dict(checkpoint['deco'], strict=True)
deco_model.eval()
return deco_model
def create_layout(dummy, camera=None):
if camera is None:
camera = dict(
up=dict(x=0, y=1, z=0),
center=dict(x=0, y=0, z=0),
eye=dict(x=dummy.x.mean(), y=0, z=3),
projection=dict(type='perspective'))
layout = dict(
scene={
"xaxis": {
'showgrid': False,
'zeroline': False,
'visible': False,
"range": [dummy.x.min(), dummy.x.max()]
},
"yaxis": {
'showgrid': False,
'zeroline': False,
'visible': False,
"range": [dummy.y.min(), dummy.y.max()]
},
"zaxis": {
'showgrid': False,
'zeroline': False,
'visible': False,
"range": [dummy.z.min(), dummy.z.max()]
},
},
autosize=False,
width=750, height=1000,
scene_camera=camera,
scene_aspectmode="data",
clickmode="event+select",
margin={'l': 0, 't': 0}
)
return layout
def create_fig(dummy, camera=None):
fig = go.Figure(
data=dummy.mesh_3d(),
layout=create_layout(dummy, camera))
return fig
class Dummy:
def __init__(self, mesh_path):
"""A simple polygonal dummy with colored patches."""
self._load_trimesh(mesh_path)
def _load_trimesh(self, path):
"""Load a mesh given a path to a .PLY file."""
self._trimesh = trimesh.load(path, process=False)
self._vertices = np.array(self._trimesh.vertices)
self._faces = np.array(self._trimesh.faces)
self.colors = self._trimesh.visual.vertex_colors
@property
def vertices(self):
"""All the mesh vertices."""
return self._vertices
@property
def faces(self):
"""All the mesh faces."""
return self._faces
@property
def n_vertices(self):
"""Number of vertices in a mesh."""
return self._vertices.shape[0]
@property
def n_faces(self):
"""Number of faces in a mesh."""
return self._faces.shape[0]
@property
def x(self):
"""An array of vertex x coordinates"""
return self._vertices[:, 0]
@property
def y(self):
"""An array of vertex y coordinates"""
return self._vertices[:, 1]
@property
def z(self):
"""An array of vertex z coordinates"""
return self._vertices[:, 2]
@property
def i(self):
"""An array of the first face vertices"""
return self._faces[:, 0]
@property
def j(self):
"""An array of the second face vertices"""
return self._faces[:, 1]
@property
def k(self):
"""An array of the third face vertices"""
return self._faces[:, 2]
@property
def default_selection(self):
"""Default patch selection mask."""
return dict(vertices=[])
def mesh_3d(
self,
lighting=DEFAULT_LIGHTING,
light_position=DEFAULT_LIGHT_POSITION
):
"""Construct a Mesh3D object give a clickmask for patch coloring."""
return go.Mesh3d(
x=self.x, y=self.y, z=self.z,
i=self.i, j=self.j, k=self.k,
vertexcolor=self.colors,
lighting=lighting,
lightposition=light_position,
hoverinfo='none')
def main(pil_img, out_dir='demo_out', model_path='checkpoint/deco_best.pth', mesh_colour=[130, 130, 130, 255], annot_colour=[0, 255, 0, 255]):
deco_model = initiate_model(model_path)
smpl_path = os.path.join(constants.SMPL_MODEL_DIR, 'smpl_neutral_tpose.ply')
img = np.array(pil_img)
img = cv2.resize(img, (256, 256), cv2.INTER_CUBIC)
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
img = img.transpose(2,0,1)/255.0
img = img[np.newaxis,:,:,:]
img = torch.tensor(img, dtype = torch.float32).to(device)
with torch.no_grad():
cont, _, _ = deco_model(img)
cont = cont.detach().cpu().numpy().squeeze()
cont_smpl = []
for indx, i in enumerate(cont):
if i >= 0.5:
cont_smpl.append(indx)
img = img.detach().cpu().numpy()
img = np.transpose(img[0], (1, 2, 0))
img = img * 255
img = img.astype(np.uint8)
contact_smpl = np.zeros((1, 1, 6890))
contact_smpl[0][0][cont_smpl] = 1
body_model_smpl = trimesh.load(smpl_path, process=False)
for vert in range(body_model_smpl.visual.vertex_colors.shape[0]):
body_model_smpl.visual.vertex_colors[vert] = mesh_colour
body_model_smpl.visual.vertex_colors[cont_smpl] = annot_colour
mesh_out_dir = os.path.join(out_dir, 'Preds')
os.makedirs(mesh_out_dir, exist_ok=True)
print(f'Saving mesh to {mesh_out_dir}')
body_model_smpl.export(os.path.join(mesh_out_dir, 'pred.obj'))
dummy = Dummy(os.path.join(mesh_out_dir, 'pred.obj'))
fig = create_fig(dummy)
return fig, os.path.join(mesh_out_dir, 'pred.obj')
with gr.Blocks(title="DECO", css=".gradio-container") as demo:
gr.Markdown(description)
gr.HTML("""DECO Demo
""")
with gr.Row():
with gr.Column():
input_image = gr.Image(label="Input image", type="pil")
with gr.Column():
output_image = gr.Plot(label="Renders")
output_meshes = gr.File(label="3D meshes")
gr.HTML("""
""")
with gr.Row():
send_btn = gr.Button("Infer")
send_btn.click(fn=main, inputs=[input_image], outputs=[output_image, output_meshes])
example_images = gr.Examples([
['/home/user/app/example_images/213.jpg'],
['/home/user/app/example_images/pexels-photo-207569.webp'],
['/home/user/app/example_images/pexels-photo-3622517.webp'],
['/home/user/app/example_images/pexels-photo-15732209.jpeg'],
],
inputs=[input_image])
demo.launch(debug=True)