app.py

# Copyright (c) 2023-2024, Zexin He
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


import os
from PIL import Image
import numpy as np
import gradio as gr


def assert_input_image(input_front_image, input_back_image):
    if input_front_image is None:
        raise gr.Error("No front image selected or uploaded!")
    if input_back_image is None:
        raise gr.Error("No back image selected or uploaded!")

def prepare_working_dir():
    import tempfile
    working_dir = tempfile.TemporaryDirectory()
    return working_dir

def init_preprocessor():
    from openlrm.utils.preprocess import Preprocessor
    global preprocessor
    preprocessor = Preprocessor()

def preprocess_fn(image_in_front: np.ndarray, image_in_back: np.ndarray, remove_bg: bool, recenter: bool, working_dir):
    # save front image first
    image_raw_front = os.path.join(working_dir.name, "raw_front.png")
    with Image.fromarray(image_in_front) as img:
        img.save(image_raw_front)
    image_out_front = os.path.join(working_dir.name, "front/rembg_front.png")

    # save back image first
    image_raw_back = os.path.join(working_dir.name, "raw_back.png")
    with Image.fromarray(image_in_back) as img:
        img.save(image_raw_back)
    image_out_back = os.path.join(working_dir.name, "back/rembg_back.png")

    # process the front and back image.
    success_front = preprocessor.preprocess(image_path=image_raw_front, save_path=image_out_front, rmbg=remove_bg, recenter=recenter)
    success_back = preprocessor.preprocess(image_path=image_raw_back, save_path=image_out_back, rmbg=remove_bg, recenter=recenter)
    assert success_front and success_back, f"Failed under preprocess_fn!"
    return image_out_front, image_out_back


def demo_openlrm(infer_impl):

    def core_fn(image_front: str, image_back: str, source_cam_dist: float, working_dir):
        dump_video_path = os.path.join(working_dir.name, "output.mp4")
        dump_mesh_path = os.path.join(working_dir.name, "output.ply")
        infer_impl(
            image_path=image_front,
            source_cam_dist=source_cam_dist,
            export_video=True,
            export_mesh=False,
            dump_video_path=dump_video_path,
            dump_mesh_path=dump_mesh_path,
            image_path_back=image_back,
        )
        return dump_video_path

    def example_fn(input_front_image: np.ndarray, input_back_image: np.ndarray):
        from gradio.utils import get_cache_folder
        working_dir = get_cache_folder()
        processed_front_image, processed_back_image = preprocess_fn(
            image_in_front=input_front_image,
            image_in_back=input_back_image,
            remove_bg=True,
            recenter=True,
            working_dir=working_dir,
        )
        video = core_fn(
            image_front=processed_front_image,
            image_back=processed_back_image,
            source_cam_dist=2.0,
            working_dir=working_dir,
        )
        return processed_front_image, processed_back_image, video

    _TITLE = '''🔥 🔥 🔥 Tailor3D: Customized 3D Assets Editing and Generation with Dual-Side Images'''

    _DESCRIPTION = '''
        <div>
            <a style="display:inline-block" href='https://github.com/Qi-Zhangyang/Tailor3D'><img src='https://img.shields.io/github/stars/Qi-Zhangyang/Tailor3D?style=social'/></a>
            <a style="display:inline-block; margin-left: .5em" href="https://huggingface.co/alexzyqi"><img src='https://img.shields.io/badge/Model-Weights-blue'/></a>
        </div>
        We propose Tailor3D, a novel pipeline creating customized 3D assets from editable dual-side images and feed-forward reconstruction methods.

        Here we show the final step of Tailor3D. That is given the edited front and beck view of the object. We can produce the 3D object with several seconds.

        <strong>Disclaimer:</strong> This demo uses `Tailor3D-base-1.1` model with 288x288 rendering resolution here for a quick demonstration.
    '''

    with gr.Blocks(analytics_enabled=False) as demo:

        # HEADERS
        with gr.Row():
            with gr.Column(scale=1):
                gr.Markdown('# ' + _TITLE)
        with gr.Row():
            gr.Markdown(_DESCRIPTION)

        # DISPLAY
        with gr.Row():
            gr.Markdown(
                """
                ## 🖼️ Input: This is the input front and back images.
                """
            )
        with gr.Row():
            with gr.Column(variant='panel', scale=0.2):
                with gr.Tabs(elem_id="tailor3d_input_front_image"):
                    with gr.TabItem('Input Front-view Image'):
                        with gr.Row():
                            input_front_image = gr.Image(label="Input Front Image", image_mode="RGBA", width="auto", sources="upload", type="numpy", elem_id="content_image")

            with gr.Column(variant='panel', scale=0.2):
                with gr.Tabs(elem_id="tailor3d_input_back_image"):
                    with gr.TabItem('Input Back-view Image'):
                        with gr.Row():
                            input_back_image = gr.Image(label="Input Back Image", image_mode="RGBA", width="auto", sources="upload", type="numpy", elem_id="content_image")
        with gr.Row():
            gr.Markdown(
                """
                ## 🛠️ Preprocess: Remove the background and center the object.
                """
            )
        with gr.Row():
            with gr.Column(variant='panel', scale=0.2):
                with gr.Tabs(elem_id="tailor3d_processed_image"):
                    with gr.TabItem('Processed Front-view Image'):
                        with gr.Row():
                            processed_front_image = gr.Image(label="Processed Image", image_mode="RGBA", type="filepath", elem_id="processed_image", width="auto", interactive=False)
            with gr.Column(variant='panel', scale=0.2):
                with gr.Tabs(elem_id="tailor3d_processed_image"):
                    with gr.TabItem('Processed Back-view Image'):
                        with gr.Row():
                            processed_back_image = gr.Image(label="Processed Image", image_mode="RGBA", type="filepath", elem_id="processed_image", width="auto", interactive=False)
        with gr.Row():
            gr.Markdown(
                """
                ## 🚀 Output: The rendering video of the 3D object.
                Note that the output is the 3D mesh, for convience, we showcase it through a video that circles around.
                """
            )
        with gr.Row():
            with gr.Column(variant='panel', scale=0.2):
                with gr.Tabs(elem_id="tailor3d_render_video"):
                    with gr.TabItem('Rendered Video'):
                        with gr.Row():
                            output_video = gr.Video(label="Rendered Video", format="mp4", width="auto", autoplay=True)

        # SETTING
        with gr.Row():
            with gr.Column(variant='panel', scale=1):
                with gr.Tabs(elem_id="openlrm_attrs"):
                    with gr.TabItem('Settings'):
                        with gr.Column(variant='panel'):
                            gr.Markdown(
                                """
                                <strong>Best Practice</strong>:
                                    Centered objects in reasonable sizes. Try adjusting source camera distances.
                                """
                            )
                            checkbox_rembg = gr.Checkbox(True, label='Remove background')
                            checkbox_recenter = gr.Checkbox(True, label='Recenter the object')
                            slider_cam_dist = gr.Slider(1.0, 3.5, value=2.0, step=0.1, label="Source Camera Distance")
                            submit = gr.Button('Generate', elem_id="openlrm_generate", variant='primary')

        # EXAMPLES
        with gr.Row():
            gr.Markdown(
                """
                ## Example in the paper.
                ### A. 3D Style Transfer
                Here we keep the object ID and just transfer the style. <br>

                **Line 1: A pop-mart boy with astronaut, blue, traditional Chinese and grey style.**
                """
            )
        with gr.Row():
            examples = [
                ['assets/sample_input/demo/front/boy_astronaut.png', 'assets/sample_input/demo/back/boy_astronaut.png'],
                ['assets/sample_input/demo/front/boy_blue.png', 'assets/sample_input/demo/back/boy_blue.png'],
                ['assets/sample_input/demo/front/boy_chinese_style.png', 'assets/sample_input/demo/back/boy_chinese_style.png'],
                ['assets/sample_input/demo/front/boy_grey_clothes.png', 'assets/sample_input/demo/back/boy_grey_clothes.png'],
            ]

            for example in examples:
                with gr.Column(scale=1):
                    gr.Examples(
                        examples=[example],
                        inputs=[input_front_image, input_back_image], 
                        outputs=[processed_front_image, processed_back_image, output_video],
                        fn=example_fn,
                        cache_examples=bool(os.getenv('SPACE_ID')),
                        examples_per_page=3,
                    )

        # # EXAMPLES
        # with gr.Row():
        #     gr.Markdown(
        #         """
        #         **Line 2: A LEGO model featuring an astronaut, green and red elements, and a wizard theme.**
        #         """
        #     )
        # with gr.Row():
        #     examples = [
        #         ['assets/sample_input/demo/front/lego_astronaut.png', 'assets/sample_input/demo/back/lego_astronaut.png'],
        #         ['assets/sample_input/demo/front/lego_green.png', 'assets/sample_input/demo/back/lego_green.png'],
        #         ['assets/sample_input/demo/front/lego_red.png', 'assets/sample_input/demo/front/lego_red.png'],
        #         ['assets/sample_input/demo/front/lego_wizard.png', 'assets/sample_input/demo/back/lego_wizard.png'],
        #     ]

        #     for example in examples:
        #         with gr.Column(scale=0.3):
        #             gr.Examples(
        #                 examples=[example],
        #                 inputs=[input_front_image, input_back_image], 
        #                 outputs=None, # [processed_image, output_video],
        #                 fn=None, # example_fn,
        #                 cache_examples=bool(os.getenv('SPACE_ID')),
        #                 examples_per_page=3,
        #             )
        # with gr.Row():
        #     gr.Markdown(
        #         """
        #         **Line 3: A marvel boy featuring an Captain America, Ironman and Spiderman, and a Superman theme.**
        #         """
        #     )
        # with gr.Row():
        #     examples = [
        #         ['assets/sample_input/demo/front/marvel_captain.png', 'assets/sample_input/demo/back/marvel_captain.png'],
        #         ['assets/sample_input/demo/front/marvel_ironman.png', 'assets/sample_input/demo/front/marvel_ironman.png'],
        #         ['assets/sample_input/demo/front/marvel_spiderman.png', 'assets/sample_input/demo/back/marvel_spiderman.png'],
        #         ['assets/sample_input/demo/front/marvel_superman.png', 'assets/sample_input/demo/back/marvel_superman.png'],
        #     ]

        #     for example in examples:
        #         with gr.Column(scale=0.3):
        #             gr.Examples(
        #                 examples=[example],
        #                 inputs=[input_front_image, input_back_image], 
        #                 outputs=None, # [processed_image, output_video],
        #                 fn=None, # example_fn,
        #                 cache_examples=bool(os.getenv('SPACE_ID')),
        #                 examples_per_page=3,
        #             )
        # # EXAMPLES
        # with gr.Row():
        #     gr.Markdown(
        #         """
        #         ### B. 3D Generative Geometry or Pattern Fill
                
        #         Here, we start with a simple object and gradually add various accessories, costumes, or patterns step by step. We only showcase the final effect after multiple rounds of decoration. <br>

        #         **Line 4: Initial object: sofa, dog, penguin, house.**
        #         """
        #     )
        # with gr.Row():
        #     examples = [
        #         ['assets/sample_input/demo/front/sofa.png', 'assets/sample_input/demo/back/sofa.png'],
        #         ['assets/sample_input/demo/front/penguin.png', 'assets/sample_input/demo/back/penguin.png'],
        #         ['assets/sample_input/demo/front/house.png', 'assets/sample_input/demo/back/house.png'],
        #     ]

        #     for example in examples:
        #         with gr.Column(scale=0.3):
        #             gr.Examples(
        #                 examples=[example],
        #                 inputs=[input_front_image, input_back_image], 
        #                 outputs=None, # [processed_image, output_video],
        #                 fn=None, # example_fn,
        #                 cache_examples=bool(os.getenv('SPACE_ID')),
        #                 examples_per_page=3,
        #             )

        # with gr.Row():
        #     gr.Markdown(
        #         """
        #         ### C. 3D Style Fusion
                
        #         We will maintain a consistent front style of the object while continuously changing the back style, blending the two different styles into one object.<br>

        #         **Line 5: A bird with different back styles.**
        #         """
        #     )
        # with gr.Row():
        #     examples = [
        #         ['assets/sample_input/demo/front/bird.png', 'assets/sample_input/demo/back/bird.png'],
        #         ['assets/sample_input/demo/front/bird_brownblue.png', 'assets/sample_input/demo/back/bird_brownblue.png'],
        #         ['assets/sample_input/demo/front/bird_rainbow.png', 'assets/sample_input/demo/back/bird_rainbow.png'],
        #         ['assets/sample_input/demo/front/bird_whitered.png', 'assets/sample_input/demo/back/bird_whitered.png'],
        #     ]

        #     for example in examples:
        #         with gr.Column(scale=0.3):
        #             gr.Examples(
        #                 examples=[example],
        #                 inputs=[input_front_image, input_back_image], 
        #                 outputs=None, # [processed_image, output_video],
        #                 fn=None, # example_fn,
        #                 cache_examples=bool(os.getenv('SPACE_ID')),
        #                 examples_per_page=3,
        #             )

        # with gr.Row():
        #     gr.Markdown(
        #         """
        #         ### Others
        #         I vote for kunkun forever, I am really an I-kUN and have heard many of his songs.
        #         """
        #     )
        # with gr.Row():
        #     examples = [
        #         ['assets/sample_input/demo/front/loopy.png', 'assets/sample_input/demo/back/loopy.png'],
        #         ['assets/sample_input/demo/front/mario.png', 'assets/sample_input/demo/back/mario.png'],
        #         ['assets/sample_input/demo/front/armor.png', 'assets/sample_input/demo/back/armor.png'],
        #         ['assets/sample_input/demo/front/kunkun_law.png', 'assets/sample_input/demo/back/kunkun_law.png'],
        #     ]

        #     for example in examples:
        #         with gr.Column(scale=0.3):
        #             gr.Examples(
        #                 examples=[example],
        #                 inputs=[input_front_image, input_back_image], 
        #                 outputs=None, # [processed_image, output_video],
        #                 fn=None, # example_fn,
        #                 cache_examples=bool(os.getenv('SPACE_ID')),
        #                 examples_per_page=3,
        #             )

        working_dir = gr.State()
        submit.click(
            fn=assert_input_image,
            inputs=[input_front_image, input_back_image],
            queue=False,
        ).success(
            fn=prepare_working_dir,
            outputs=[working_dir],
            queue=False,
        ).success(
            fn=preprocess_fn,
            inputs=[input_front_image, input_back_image, checkbox_rembg, checkbox_recenter, working_dir],
            outputs=[processed_front_image, processed_back_image],
        ).success(
            fn=core_fn,
            inputs=[processed_front_image, processed_back_image, slider_cam_dist, working_dir],
            outputs=[output_video],
        )

        demo.queue()
        demo.launch()


def launch_gradio_app():

    os.environ.update({
        "APP_ENABLED": "1",
        "APP_MODEL_NAME": "alexzyqi/Tailor3D-Base-1.0",
        "APP_PRETRAIN_MODEL_NAME": "zxhezexin/openlrm-mix-base-1.1",
        "APP_INFER": "./configs/infer-gradio-base.yaml",
        "APP_TYPE": "infer.lrm",
        "NUMBA_THREADING_LAYER": 'omp',
    })

    from openlrm.runners import REGISTRY_RUNNERS
    from openlrm.runners.infer.base_inferrer import Inferrer
    InferrerClass : Inferrer = REGISTRY_RUNNERS[os.getenv("APP_TYPE")]
    with InferrerClass() as inferrer:
        init_preprocessor()
        if not bool(os.getenv('SPACE_ID')):
            from openlrm.utils.proxy import no_proxy
            demo = no_proxy(demo_openlrm)
        else:
            demo = demo_openlrm
        demo(infer_impl=inferrer.infer_single)


if __name__ == '__main__':

    launch_gradio_app()