|
|
|
|
|
import gradio as gr |
|
import os |
|
|
|
import subprocess |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
from apps.infer import generate_model, generate_video |
|
|
|
|
|
|
|
title = ''' |
|
# Unconstrained & Detailed Clothed Human Digitization (ECON + ControlNet) |
|
### ECON: Explicit Clothed humans Optimized via Normal integration (CVPR 2023, Highlight) |
|
''' |
|
|
|
bottom = ''' |
|
|
|
|
|
#### Citation |
|
``` |
|
@inproceedings{xiu2023econ, |
|
title = {{ECON: Explicit Clothed humans Optimized via Normal integration}}, |
|
author = {Xiu, Yuliang and Yang, Jinlong and Cao, Xu and Tzionas, Dimitrios and Black, Michael J.}, |
|
booktitle = {Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)}, |
|
month = {June}, |
|
year = {2023}, |
|
} |
|
``` |
|
|
|
|
|
<details> |
|
|
|
<summary>More</summary> |
|
|
|
#### Acknowledgments: |
|
- [controlnet-openpose](https://huggingface.co/spaces/diffusers/controlnet-openpose) |
|
- [TEXTure](https://huggingface.co/spaces/TEXTurePaper/TEXTure) |
|
|
|
|
|
#### Image Credits |
|
|
|
* [Pinterest](https://www.pinterest.com/search/pins/?q=parkour&rs=sitelinks_searchbox) |
|
|
|
#### Related works |
|
|
|
* [ICON @ MPI-IS](https://icon.is.tue.mpg.de/) |
|
* [MonoPort @ USC](https://xiuyuliang.cn/monoport) |
|
* [Phorhum @ Google](https://phorhum.github.io/) |
|
* [PIFuHD @ Meta](https://shunsukesaito.github.io/PIFuHD/) |
|
* [PaMIR @ Tsinghua](http://www.liuyebin.com/pamir/pamir.html) |
|
|
|
</details> |
|
|
|
<center> |
|
<a href="https://huggingface.co/spaces/Yuliang/ECON?duplicate=true"><img src="https://huggingface.co/datasets/huggingface/badges/raw/main/duplicate-this-space-lg-dark.svg"/></a> |
|
<h2> Generate pose & prompt-guided images / Upload photos / Use examples → Submit Image (~3min) → Generate Video (~3min) </h2> |
|
<h2><span style="color:red">ECON is only suitable for "humanoid images" and will not work well on cartoons with non-human shapes.</span></h2> |
|
</center> |
|
''' |
|
|
|
description = ''' |
|
<table> |
|
<th width="20%"> |
|
<ul> |
|
<li><strong>Homepage</strong> <a href="https://econ.is.tue.mpg.de/">econ.is.tue.mpg.de</a></li> |
|
<li><strong>Code</strong> <a href="https://github.com/YuliangXiu/ECON">YuliangXiu/ECON</a></li> |
|
<li><strong>Paper</strong> <a href="https://arxiv.org/abs/2212.07422">arXiv</a>, <a href="https://readpaper.com/paper/4736821012688027649">ReadPaper</a></li> |
|
<li><strong>Chatroom</strong> <a href="https://discord.gg/Vqa7KBGRyk">Discord</a></li> |
|
</ul> |
|
<br> |
|
<ul> |
|
<li><strong>Colab Notebook</strong> <a href='https://colab.research.google.com/drive/1YRgwoRCZIrSB2e7auEWFyG10Xzjbrbno?usp=sharing'><img style="display: inline-block;" src='https://colab.research.google.com/assets/colab-badge.svg' alt='Google Colab'></a></li> |
|
<li><strong>Blender Plugin</strong> <a href='https://carlosedubarreto.gumroad.com/l/CEB_ECON'><img style="display: inline-block;" src='https://img.shields.io/badge/Blender-F6DDCC.svg?logo=Blender' alt='Blender'></a></li> |
|
<li><strong>Docker Image</strong> <a href='https://github.com/YuliangXiu/ECON/blob/master/docs/installation-docker.md'><img style="display: inline-block;" src='https://img.shields.io/badge/Docker-9cf.svg?logo=Docker' alt='Docker'></a></li> |
|
<li><strong>Windows Setup</strong> <a href="https://github.com/YuliangXiu/ECON/blob/master/docs/installation-windows.md"><img style="display: inline-block;" src='https://img.shields.io/badge/Windows-00a2ed.svg?logo=Windows' akt='Windows'></a></li> |
|
</ul> |
|
|
|
<br> |
|
<a href="https://twitter.com/yuliangxiu"><img alt="Twitter Follow" src="https://img.shields.io/twitter/follow/yuliangxiu?style=social"></a><br> |
|
<iframe src="https://ghbtns.com/github-btn.html?user=yuliangxiu&repo=ECON&type=star&count=true&v=2&size=small" frameborder="0" scrolling="0" width="100" height="20"></iframe> |
|
</th> |
|
<th width="40%"> |
|
<iframe width="560" height="315" src="https://www.youtube.com/embed/5PEd_p90kS0" title="YouTube video player" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe> |
|
</th> |
|
<th width="40%"> |
|
<iframe width="560" height="315" src="https://www.youtube.com/embed/sbWZbTf6ZYk" title="YouTube video player" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe> |
|
</th> |
|
</table> |
|
''' |
|
|
|
from controlnet_aux import OpenposeDetector |
|
from diffusers import StableDiffusionControlNetPipeline, ControlNetModel |
|
from diffusers import UniPCMultistepScheduler |
|
import gradio as gr |
|
import torch |
|
import base64 |
|
from io import BytesIO |
|
from PIL import Image |
|
|
|
|
|
canvas_html = "<pose-canvas id='canvas-root' style='display:flex;max-width: 500px;margin: 0 auto;'></pose-canvas>" |
|
load_js = """ |
|
async () => { |
|
const url = "https://huggingface.co/datasets/radames/gradio-components/raw/main/pose-gradio.js" |
|
fetch(url) |
|
.then(res => res.text()) |
|
.then(text => { |
|
const script = document.createElement('script'); |
|
script.type = "module" |
|
script.src = URL.createObjectURL(new Blob([text], { type: 'application/javascript' })); |
|
document.head.appendChild(script); |
|
}); |
|
} |
|
""" |
|
get_js_image = """ |
|
async (image_in_img, prompt, image_file_live_opt, live_conditioning) => { |
|
const canvasEl = document.getElementById("canvas-root"); |
|
const data = canvasEl? canvasEl._data : null; |
|
return [image_in_img, prompt, image_file_live_opt, data] |
|
} |
|
""" |
|
|
|
|
|
cached = False |
|
|
|
|
|
pose_model = OpenposeDetector.from_pretrained("lllyasviel/ControlNet") |
|
controlnet = ControlNetModel.from_pretrained( |
|
"lllyasviel/sd-controlnet-openpose", torch_dtype=torch.float16 |
|
) |
|
pipe = StableDiffusionControlNetPipeline.from_pretrained( |
|
"runwayml/stable-diffusion-v1-5", |
|
controlnet=controlnet, |
|
safety_checker=None, |
|
torch_dtype=torch.float16 |
|
) |
|
pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config) |
|
|
|
|
|
|
|
pipe.enable_model_cpu_offload() |
|
|
|
|
|
pipe.enable_xformers_memory_efficient_attention() |
|
|
|
|
|
generator = torch.manual_seed(0) |
|
|
|
hint_prompts = ''' |
|
<strong>Hints</strong>: <br> |
|
best quality, extremely detailed, solid color background, |
|
super detail, high detail, edge lighting, soft focus, |
|
light and dark contrast, 8k, edge lighting, 3d, c4d, |
|
blender, oc renderer, ultra high definition, 3d rendering |
|
''' |
|
|
|
|
|
def get_pose(image): |
|
return pose_model(image) |
|
|
|
import sys |
|
|
|
def read_logs(): |
|
sys.stdout.flush() |
|
with open("output.log", "r") as f: |
|
return f.read() |
|
|
|
|
|
def generate_images(image, prompt, image_file_live_opt='file', live_conditioning=None): |
|
if image is None and 'image' not in live_conditioning: |
|
raise gr.Error("Please provide an image") |
|
try: |
|
if image_file_live_opt == 'file': |
|
pose = get_pose(image) |
|
elif image_file_live_opt == 'webcam': |
|
base64_img = live_conditioning['image'] |
|
image_data = base64.b64decode(base64_img.split(',')[1]) |
|
pose = Image.open(BytesIO(image_data)).convert('RGB').resize((512, 512)) |
|
output = pipe( |
|
prompt, |
|
pose, |
|
generator=generator, |
|
num_images_per_prompt=3, |
|
num_inference_steps=50, |
|
) |
|
all_outputs = [] |
|
all_outputs.append(pose) |
|
for image in output.images: |
|
all_outputs.append(image) |
|
return all_outputs, all_outputs |
|
except Exception as e: |
|
raise gr.Error(str(e)) |
|
|
|
|
|
def toggle(choice): |
|
if choice == "file": |
|
return gr.update(visible=True, value=None), gr.update(visible=False, value=None) |
|
elif choice == "webcam": |
|
return gr.update(visible=False, value=None), gr.update(visible=True, value=canvas_html) |
|
|
|
|
|
examples_pose = 'examples/pose' |
|
examples_cloth = 'examples/cloth' |
|
|
|
def show_video(): |
|
return gr.update(visible=True), gr.update(visible=True) |
|
|
|
with gr.Blocks() as demo: |
|
|
|
gr.Markdown(title) |
|
gr.HTML(description) |
|
gr.Markdown(bottom) |
|
|
|
out_lst = [] |
|
with gr.Row(): |
|
with gr.Column(): |
|
with gr.Row(): |
|
|
|
live_conditioning = gr.JSON(value={}, visible=False) |
|
|
|
with gr.Column(): |
|
image_file_live_opt = gr.Radio(["file", "webcam"], |
|
value="file", |
|
label="How would you like to upload your image?") |
|
|
|
with gr.Row(): |
|
image_in_img = gr.Image( |
|
visible=True, type="pil", label="Image for Pose" |
|
) |
|
canvas = gr.HTML(None, elem_id="canvas_html", visible=False) |
|
|
|
image_file_live_opt.change( |
|
fn=toggle, |
|
inputs=[image_file_live_opt], |
|
outputs=[image_in_img, canvas], |
|
queue=False |
|
) |
|
prompt = gr.Textbox( |
|
label="Enter your prompt to synthesise the image", |
|
max_lines=10, |
|
placeholder="best quality, extremely detailed", |
|
) |
|
|
|
gr.Markdown(hint_prompts) |
|
|
|
with gr.Column(): |
|
gallery = gr.Gallery(label="Generated Images", columns=[2],rows=[2]) |
|
gallery_cache = gr.State() |
|
|
|
gr.Markdown( |
|
''' |
|
<center> |
|
<strong>Click the target generated image for Reconstruction.</strong> <br> |
|
↓ |
|
</center> |
|
''' |
|
) |
|
|
|
inp = gr.Image(type="filepath", label="Input Image for Reconstruction") |
|
fitting_step = gr.Slider( |
|
10, |
|
100, |
|
step=10, |
|
label='Fitting steps (Slower yet Better-aligned SMPL-X)', |
|
value=50 |
|
) |
|
|
|
with gr.Row(): |
|
btn_sample = gr.Button("Generate Image") |
|
btn_submit = gr.Button("Submit Image (~3min)") |
|
|
|
btn_sample.click( |
|
fn=generate_images, |
|
inputs=[image_in_img, prompt, image_file_live_opt, live_conditioning], |
|
outputs=[gallery, gallery_cache], |
|
js=get_js_image |
|
) |
|
|
|
def get_select_index(cache, evt: gr.SelectData): |
|
return cache[evt.index] |
|
|
|
gallery.select( |
|
fn=get_select_index, |
|
inputs=[gallery_cache], |
|
outputs=[inp], |
|
) |
|
|
|
with gr.Row(): |
|
|
|
gr.Examples( |
|
examples=examples_pose, |
|
inputs=[inp], |
|
cache_examples=cached, |
|
fn=generate_model, |
|
outputs=out_lst, |
|
label="Hard Pose Examples" |
|
) |
|
|
|
gr.Examples( |
|
examples=examples_cloth, |
|
inputs=[inp], |
|
cache_examples=cached, |
|
fn=generate_model, |
|
outputs=out_lst, |
|
label="Loose Cloth Examples" |
|
) |
|
|
|
|
|
with gr.Column(): |
|
overlap_inp = gr.Image(type="filepath", label="Image Normal Overlap") |
|
out_final = gr.Model3D( |
|
clear_color=[0.0, 0.0, 0.0, 0.0], label="Clothed human", elem_id="avatar" |
|
) |
|
out_smpl = gr.Model3D( |
|
clear_color=[0.0, 0.0, 0.0, 0.0], label="SMPL-X body (via PIXIE)", elem_id="avatar" |
|
) |
|
|
|
vis_tensor_path = gr.State() |
|
|
|
|
|
|
|
btn_video = gr.Button("Generate Video (~3min)", visible=False) |
|
out_vid = gr.Video(label="Shared on Twitter with #ECON", visible=False) |
|
|
|
out_lst = [out_smpl, out_final, overlap_inp, vis_tensor_path] |
|
|
|
btn_video.click( |
|
fn=generate_video, |
|
inputs=[vis_tensor_path], |
|
outputs=[out_vid], |
|
) |
|
|
|
btn_submit.click(fn=generate_model, inputs=[inp, fitting_step], outputs=out_lst) |
|
btn_submit.click(fn=show_video, outputs=[btn_video, out_vid]) |
|
|
|
demo.load(None, None, None, js=load_js) |
|
|
|
if __name__ == "__main__": |
|
|
|
demo.queue() |
|
demo.launch(max_threads=4) |
|
|
