i

.gitattributes

@@ -38,3 +38,4 @@ apps/third_party/Wonder3D/assets/fig_teaser.png filter=lfs diff=lfs merge=lfs -t
 asset/video_cover.png filter=lfs diff=lfs merge=lfs -text
 apps/InstantMeshes filter=lfs diff=lfs merge=lfs -text
 apps/third_party/InstantMeshes filter=lfs diff=lfs merge=lfs -text
+apps/third_party/quadriflow filter=lfs diff=lfs merge=lfs -text

apps/gradio_app.py

@@ -20,6 +20,8 @@ from utils import *
 proj_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 sys.path.append(os.path.join(proj_dir))
 
+import tempfile
+
 import craftsman
 from craftsman.systems.base import BaseSystem
 from craftsman.utils.config import ExperimentConfig, load_config
@@ -104,16 +106,21 @@ def image2mesh(view_front: np.ndarray,
     )
     assert len(mesh_outputs) == 1, "Only support single mesh output for gradio demo"
     mesh = trimesh.Trimesh(mesh_outputs[0][0], mesh_outputs[0][1])
-    filepath = f"{cached_dir}/{time.time()}.obj"
+    # filepath = f"{cached_dir}/{time.time()}.obj"
+    filepath = tempfile.NamedTemporaryFile(suffix=f".obj", delete=False).name
     mesh.export(filepath, include_normals=True)
 
     if 'Remesh' in more:
+        remeshed_filepath = tempfile.NamedTemporaryFile(suffix=f"_remeshed.obj", delete=False).name
         print("Remeshing with Instant Meshes...")
         target_face_count = int(len(mesh.faces)/10)
         # command = f"{proj_dir}/apps/third_party/InstantMeshes {filepath} -f {target_face_count} -d -S 0 -r 6 -p 6 -o {filepath.replace('.obj', '_remeshed.obj')}"
-        command = f"{proj_dir}/apps/third_party/InstantMeshes {filepath} -f {target_face_count} -d -S 0 -r 4 -p 4 -o {filepath.replace('.obj', '_remeshed.obj')}"
+        # command = f"{proj_dir}/apps/third_party/InstantMeshes {filepath} -f {target_face_count} -d -S 0 -r 4 -p 4 -o {filepath.replace('.obj', '_remeshed.obj')}"
+        # command = f"{proj_dir}/apps/third_party/InstantMeshes {filepath} -f {target_face_count} -o {filepath.replace('.obj', '_remeshed.obj')}"
+        command = f"{proj_dir}/apps/third_party/quadriflow -i {filepath} -f {target_face_count} -o {remeshed_filepath}"
         os.system(command)
-        filepath = filepath.replace('.obj', '_remeshed.obj')
+        filepath = remeshed_filepath
+        # filepath = filepath.replace('.obj', '_remeshed.obj')
     
     return filepath
     

apps/mv_models.py

@@ -19,6 +19,37 @@ from huggingface_hub import hf_hub_download
 
 parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
+    
+@dataclass
+class TestConfig:
+    pretrained_model_name_or_path: str
+    pretrained_unet_path: str
+    revision: Optional[str]
+    validation_dataset: Dict
+    save_dir: str
+    seed: Optional[int]
+    validation_batch_size: int
+    dataloader_num_workers: int
+
+    local_rank: int
+
+    pipe_kwargs: Dict
+    pipe_validation_kwargs: Dict
+    unet_from_pretrained_kwargs: Dict
+    validation_guidance_scales: List[float]
+    validation_grid_nrow: int
+    camera_embedding_lr_mult: float
+
+    num_views: int
+    camera_embedding_type: str
+
+    pred_type: str  # joint, or ablation
+
+    enable_xformers_memory_efficient_attention: bool
+
+    cond_on_normals: bool
+    cond_on_colors: bool
+
 class GenMVImage(object):
     def __init__(self, device):
         self.seed = 1024
@@ -96,19 +127,19 @@ class GenMVImage(object):
         return mv_imgs[1], mv_imgs[2], mv_imgs[3], mv_imgs[0]
     
     def gen_image_from_wonder3d(self, image, crop_size):
-        sys.path.append(f"{parent_dir}/apps/third_party/Wonder3D")
-        from .third_party.Wonder3D.mvdiffusion.pipelines.pipeline_mvdiffusion_image import MVDiffusionImagePipeline
+        from diffusers import DiffusionPipeline  # only tested on diffusers[torch]==0.19.3, may have conflicts with newer versions of diffusers
+
         weight_dtype = torch.float16
         batch = prepare_data(image, crop_size)
 
         if "wonder3d" in self.pipelines.keys():
             pipeline = self.pipelines['wonder3d']
         else:
-            self.pipelines['wonder3d'] = MVDiffusionImagePipeline.from_pretrained(
-                        'flamehaze1115/wonder3d-v1.0',
-                        custom_pipeline=f'{parent_dir}/apps/third_party/Wonder3D/mvdiffusion/pipelines/pipeline_mvdiffusion_image.py',
-                        torch_dtype=weight_dtype
-                    )
+            self.pipelines['wonder3d'] =  DiffusionPipeline.from_pretrained(
+            'flamehaze1115/wonder3d-v1.0', # or use local checkpoint './ckpts'
+            custom_pipeline='flamehaze1115/wonder3d-pipeline',
+            torch_dtype=torch.float16
+            )
             self.pipelines['wonder3d'].unet.enable_xformers_memory_efficient_attention()
             self.pipelines['wonder3d'].to(self.device)
             self.pipelines['wonder3d'].set_progress_bar_config(disable=True)

apps/third_party/CRM/imagedream/ldm/util.py

@@ -95,9 +95,9 @@ def get_obj_from_str(string, reload=False):
         importlib.reload(module_imp)
         
     if 'imagedream' in module:
-        module = 'third_party.CRM.'+module
+        module = 'apps.third_party.CRM.'+module
     if 'lib' in module:
-        module = 'third_party.CRM.'+module
+        module = 'apps.third_party.CRM.'+module
     return getattr(importlib.import_module(module, package=None), cls)
 
 

apps/third_party/CRM/libs/sample.py

@@ -6,7 +6,7 @@ from imagedream.ldm.util import set_seed, add_random_background
 # import sys
 # proj_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 # sys.path.append(proj_dir)
-from third_party.CRM.libs.base_utils import do_resize_content
+from apps.third_party.CRM.libs.base_utils import do_resize_content
 from imagedream.ldm.models.diffusion.ddim import DDIMSampler
 from torchvision import transforms as T
 

apps/third_party/Wonder3D/mvdiffusion/data/single_image_dataset.py

@@ -107,7 +107,7 @@ class SingleImageDataset(Dataset):
         elif self.num_views == 6:
             self.view_types  = ['front', 'front_right', 'right', 'back', 'left', 'front_left']
         
-        self.fix_cam_pose_dir = "./third_party/Wonder3D/mvdiffusion/data/fixed_poses/nine_views"
+        self.fix_cam_pose_dir = "apps/third_party/Wonder3D/mvdiffusion/data/fixed_poses/nine_views"
         
         self.fix_cam_poses = self.load_fixed_poses()  # world2cam matrix
 

apps/utils.py

@@ -17,6 +17,7 @@ rembg_session = rembg.new_session()
 from segment_anything import sam_model_registry, SamPredictor
 
 import craftsman
+from craftsman.systems.base import BaseSystem
 from craftsman.utils.config import ExperimentConfig, load_config
 
 parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
@@ -47,7 +48,7 @@ def load_model(
     )
     
     print(f"Restoring states from the checkpoint path at {ckpt_path} with config {cfg}")
-    system.load_state_dict(torch.load(ckpt_path)['state_dict'])
+    system.load_state_dict(torch.load(ckpt_path, map_location=torch.device('cpu'))['state_dict'])
     system = system.to(device).eval()
 
     return system
@@ -135,7 +136,7 @@ def save_image(tensor):
     return ndarr
 
 def prepare_data(single_image, crop_size):
-    from third_party.Wonder3D.mvdiffusion.data.single_image_dataset import SingleImageDataset
+    from apps.third_party.Wonder3D.mvdiffusion.data.single_image_dataset import SingleImageDataset
     dataset = SingleImageDataset(root_dir='', num_views=6, img_wh=[256, 256], bg_color='white', crop_size=crop_size, single_image=single_image)
     return dataset[0]
 

gradio_app.py

@@ -19,6 +19,7 @@ import gradio as gr
 proj_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 sys.path.append(os.path.join(proj_dir))
 
+import tempfile
 import craftsman
 from craftsman.systems.base import BaseSystem
 from craftsman.utils.config import ExperimentConfig, load_config
@@ -104,18 +105,22 @@ def image2mesh(view_front: np.ndarray,
     )
     assert len(mesh_outputs) == 1, "Only support single mesh output for gradio demo"
     mesh = trimesh.Trimesh(mesh_outputs[0][0], mesh_outputs[0][1])
-    filepath = f"{cached_dir}/{time.time()}.obj"
+    # filepath = f"{cached_dir}/{time.time()}.obj"
+    filepath = tempfile.NamedTemporaryFile(suffix=f".obj", delete=False).name
     mesh.export(filepath, include_normals=True)
 
     if 'Remesh' in more:
+        remeshed_filepath = tempfile.NamedTemporaryFile(suffix=f"_remeshed.obj", delete=False).name
         print("Remeshing with Instant Meshes...")
-        target_face_count = int(len(mesh.faces)/10)
-        command = f"{proj_dir}/apps/third_party/InstantMeshes {filepath} -f {target_face_count} -d -S 0 -r 6 -p 6 -o {filepath.replace('.obj', '_remeshed.obj')}"
+        # target_face_count = int(len(mesh.faces)/10)
+        target_face_count = 1000
+        command = f"{proj_dir}/apps/third_party/InstantMeshes {filepath} -f {target_face_count} -o {remeshed_filepath}"
         os.system(command)
-        filepath = filepath.replace('.obj', '_remeshed.obj')
+        filepath = remeshed_filepath
+        # filepath = filepath.replace('.obj', '_remeshed.obj')
     
     return filepath
-    
+
 if __name__=="__main__":
     parser = argparse.ArgumentParser()
     # parser.add_argument("--model_path", type=str, required=True, help="Path to the object file",)
@@ -201,10 +206,10 @@ if __name__=="__main__":
                     view_back = gr.Image(label="Back", interactive=True, show_label=True)
                     view_left = gr.Image(label="Left", interactive=True, show_label=True)
                     
-                with gr.Accordion('Advanced options', open=False):
-                    with gr.Row(equal_height=True):
-                        run_mv_btn = gr.Button('Only Generate 2D', interactive=True)
-                        run_3d_btn = gr.Button('Only Generate 3D', interactive=True)
+                # with gr.Accordion('Advanced options', open=False):
+                with gr.Row(equal_height=True):
+                    run_mv_btn = gr.Button('Only Generate 2D', interactive=True)
+                    run_3d_btn = gr.Button('Only Generate 3D', interactive=True)
 
                 with gr.Accordion('Advanced options (2D)', open=False):
                     with gr.Row():

requirements.txt

@@ -1,5 +1,5 @@
 datasets==2.19.0
-diffusers==0.27.2
+diffusers==0.19.3
 einops==0.8.0
 huggingface-hub==0.22.2
 imageio==2.34.1