Upload folder using huggingface_hub

.gitattributes

@@ -133,3 +133,4 @@ gradio_cached_examples/23/3D[[:space:]]Gaussians[[:space:]]ply[[:space:]]format/
 gradio_cached_examples/23/3D[[:space:]]Gaussians[[:space:]]ply[[:space:]]format/e611dc2e9fbf1acd1cfc/_a[[:space:]]toy[[:space:]]robot..._013020.ply filter=lfs diff=lfs merge=lfs -text
 tmp/input_image.png filter=lfs diff=lfs merge=lfs -text
 wheel/diff_gaussian_rasterization-0.0.0-cp310-cp310-linux_x86_64.whl filter=lfs diff=lfs merge=lfs -text
+gradio_cached_examples/23/3D[[:space:]]Gaussians[[:space:]]ply[[:space:]]format/a6d4bba77a6f190743e6/_a[[:space:]]toy[[:space:]]robot..._013020.ply filter=lfs diff=lfs merge=lfs -text

app.py

@@ -13,63 +13,70 @@ current_path = os.path.dirname(os.path.abspath(__file__))
 MAX_SEED = np.iinfo(np.int32).max
 TMP_DIR = os.path.join(current_path, 'out')
 os.makedirs(TMP_DIR, exist_ok=True)
-TAG = "gsdiff_gobj83k_sd15__render"
-# os.system("cd extensions/RaDe-GS/submodules && pip3 install diff-gaussian-rasterization")
-# os.system("cd extensions/RaDe-GS/submodules/diff-gaussian-rasterization && python3 setup.py bdist_wheel ")
 
+TAG = {
+    "SD15": ["gsdiff_gobj83k_sd15__render", "gsdiff_gobj83k_sd15_image__render"], # Best efficiency
+    "PixArt-Sigma": ["gsdiff_gobj83k_pas_fp16__render","gsdiff_gobj83k_pas_fp16_image__render"], 
+    "SD3": ["gsdiff_gobj83k_sd35m__render", "gsdiff_gobj83k_sd35m_image__render"] # Best performance
+}
+MODEL_TYPE = "PixArt-Sigma"
+
+# for PixArt-Sigma
+subprocess.run(shlex.split("python3 download_ckpt.py --model_type pas")) # for txt condition
+subprocess.run(shlex.split("python3 download_ckpt.py --model_type pas --image_cond")) # for img condition
+img_commands = "PYTHONPATH=./ bash scripts/infer.sh src/infer_gsdiff_pas.py configs/gsdiff_pas.yaml {} \
+--rembg_and_center --triangle_cfg_scaling --save_ply --output_video_type mp4 --guidance_scale {} \
+--image_path {} --elevation {} --prompt {}  --seed {}"
 
-# download checkpoints
-# subprocess.run(shlex.split("python3 download_ckpt.py --model_type pas")) # for txt condition
-# subprocess.run(shlex.split("python3 download_ckpt.py --model_type pas --image_cond")) # for img condition
+txt_commands = "PYTHONPATH=./ bash scripts/infer.sh src/infer_gsdiff_pas.py  configs/gsdiff_pas.yaml {} \
+ --save_ply --output_video_type mp4 \
+ --prompt {} --seed {}"
 
-# img_commands = "PYTHONPATH=./ bash scripts/infer.sh src/infer_gsdiff_pas.py configs/gsdiff_pas.yaml {} \
-# --rembg_and_center --triangle_cfg_scaling --save_ply --output_video_type mp4 --guidance_scale {} \
-# --image_path {} --elevation {} --prompt {} --seed {}"
 
-# txt_commands = "PYTHONPATH=./ bash scripts/infer.sh src/infer_gsdiff_pas.py configs/gsdiff_pas.yaml \
+# for SD1.5
+# subprocess.run(shlex.split("python3 download_ckpt.py --model_type sd15")) # for txt condition
+# subprocess.run(shlex.split("python3 download_ckpt.py --model_type sd15 --image_cond")) # for img condition
+
+# img_commands = "PYTHONPATH=./ bash scripts/infer.sh src/infer_gsdiff_sd.py configs/gsdiff_sd15.yaml {} \
 # --rembg_and_center --triangle_cfg_scaling --save_ply --output_video_type mp4 --guidance_scale {} \
 # --image_path {} --elevation {} --prompt {} --seed {}"
 
-# SD1.5
-subprocess.run(shlex.split("python3 download_ckpt.py --model_type sd15")) # for txt condition
-# subprocess.run(shlex.split("python3 download_ckpt.py --model_type sd15 --image_cond")) # for img condition
-img_commands = "PYTHONPATH=./ bash scripts/infer.sh src/infer_gsdiff_sd.py configs/gsdiff_sd15.yaml {} \
---rembg_and_center --triangle_cfg_scaling --save_ply --output_video_type mp4 --guidance_scale {} \
---image_path {} --elevation {} --prompt {} --seed {}"
-
-txt_commands = "PYTHONPATH=./ bash scripts/infer.sh src/infer_gsdiff_sd.py  configs/gsdiff_sd15.yaml {} \
---rembg_and_center --save_ply --output_video_type mp4 --guidance_scale {} \
- --elevation {} --prompt {} --seed {}"
+# txt_commands = "PYTHONPATH=./ bash scripts/infer.sh src/infer_gsdiff_sd.py  configs/gsdiff_sd15.yaml {} \
+#  --save_ply --output_video_type mp4 --guidance_scale {} \
+#  --elevation {} --prompt {} --seed {}"
 
 
 
 # process function
 @spaces.GPU
-def process(input_image, prompt='', prompt_neg='ugly, blurry, pixelated obscure, unnatural colors, poor lighting, dull, unclear, cropped, lowres, low quality, artifacts, duplicate', input_elevation=20, guidance_scale=2., input_seed=0):
-    subprocess.run("cd extensions/RaDe-GS/submodules && pip3 install diff-gaussian-rasterization", shell=True)
-    subprocess.run("cd extensions/RaDe-GS/submodules/diff-gaussian-rasterization && python3 setup.py bdist_wheel ", shell=True)
-    
+def process(input_image, prompt='a_high_quality_3D_asset', prompt_neg='poor_quality', input_elevation=20, guidance_scale=2., input_seed=0):
+    # subprocess.run("cd extensions/RaDe-GS/submodules && pip3 install diff-gaussian-rasterization", shell=True)
+    # subprocess.run("cd extensions/RaDe-GS/submodules/diff-gaussian-rasterization && python3 setup.py bdist_wheel ", shell=True)
+
     if input_image is not None:
         image_path = os.path.join(TMP_DIR, "input_image.png")
         image_name = image_path.split('/')[-1].split('.')[0] + "_rgba"
         input_image.save(image_path)
-        full_command = img_commands.format(TAG, guidance_scale, image_path, input_elevation, prompt, input_seed)
+        TAG_DEST = TAG[MODEL_TYPE][1]
+        full_command = img_commands.format(TAG_DEST, guidance_scale, image_path, input_elevation, prompt, input_seed)
     else:
-        full_command = txt_commands.format(TAG, guidance_scale, input_elevation, prompt, input_seed)
+        TAG_DEST = TAG[MODEL_TYPE][0]
+        # without guidance_scale and input_elevation
+        full_command = txt_commands.format(TAG_DEST, prompt, input_seed)
         image_name = ""
     
     os.system(full_command)
     
     # save video and ply files
-    ckpt_dir = os.path.join(TMP_DIR, TAG, "checkpoints")
+    ckpt_dir = os.path.join(TMP_DIR, TAG_DEST, "checkpoints")
     infer_from_iter = int(sorted(os.listdir(ckpt_dir))[-1])
     MAX_NAME_LEN = 20  # TODO: make `20` configurable
     prompt = prompt.replace("_", " ")
     prompt_name = prompt[:MAX_NAME_LEN] + "..." if prompt[:MAX_NAME_LEN] != "" else prompt
     name = f"[{image_name}]_[{prompt_name}]_{infer_from_iter:06d}"
-    output_video_path = os.path.join(TMP_DIR, TAG, "inference",  name + ".mp4")
-    output_ply_path = os.path.join(TMP_DIR, TAG, "inference",  name + ".ply")
-    output_img_path = os.path.join(TMP_DIR, TAG, "inference",  name + "_gs.png")
+    output_video_path = os.path.join(TMP_DIR, TAG_DEST, "inference",  name + ".mp4")
+    output_ply_path = os.path.join(TMP_DIR, TAG_DEST, "inference",  name + ".ply")
+    output_img_path = os.path.join(TMP_DIR, TAG_DEST, "inference",  name + "_gs.png")
 
     logger.info(full_command, output_video_path, output_ply_path)
 
@@ -116,7 +123,7 @@ with block:
             guidance_scale = gr.Slider(label="guidance scale", minimum=1., maximum=7.5, step=0.5, value=2.0)
             
             # elevation
-            input_elevation = gr.Slider(label="elevation", minimum=-90, maximum=90, step=1, value=20)
+            input_elevation = gr.Slider(label="elevation", minimum=-90, maximum=90, step=1, value=10)
             # # inference steps
             # input_num_steps = gr.Slider(label="inference steps", minimum=1, maximum=100, step=1, value=30)
             # random seed
@@ -125,7 +132,7 @@ with block:
             button_gen = gr.Button("Generate")
 
 
-        with gr.Column(scale=1):
+        with gr.Column(scale=0.8):
             with gr.Tab("Video"):
                 # final video results
                 output_video = gr.Video(label="video")
@@ -153,8 +160,8 @@ with block:
     gr.Examples(
         examples=[
             "a_toy_robot",
-            # "a_cute_panda",
-            # "a_book"
+            "a_cute_panda",
+            "an_ancient_leather-bound_book"
         ],
         inputs=[input_text],
         outputs=[output_image, output_video, output_file],

gradio_cached_examples/23/3D Gaussians ply format/a6d4bba77a6f190743e6/_a toy robot..._013020.ply

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3eb386a0bd35349d3271f7ac5431b26af1f8549b7a8a16c10af894877bb22c09
+size 1476719

gradio_cached_examples/23/log.csv

@@ -1,4 +1,2 @@
 component 0,video,3D Gaussians (ply format),flag,username,timestamp
-"{""path"": ""gradio_cached_examples/23/component 0/60e43731ac393e3fd280/image.webp"", ""url"": ""/file=/tmp/gradio/dbf713c255683bb9a0817141f85a0a122fcfda997c9ee184c11cadc4ec699dbf/image.webp"", ""size"": null, ""orig_name"": ""image.webp"", ""mime_type"": null, ""is_stream"": false, ""meta"": {""_type"": ""gradio.FileData""}}","{""video"": {""path"": ""gradio_cached_examples/23/video/a573fa23b5f2b0944aef/_a toy robot..._013020.mp4"", ""url"": ""/file=/tmp/gradio/93e4187e3359a1457f501416c2ee86ed9e0ab485007ff0784cd226ac2f6e8e0b/_a toy robot..._013020.mp4"", ""size"": null, ""orig_name"": ""[]_[a toy robot...]_013020.mp4"", ""mime_type"": null, ""is_stream"": false, ""meta"": {""_type"": ""gradio.FileData""}}, ""subtitles"": null}","{""path"": ""gradio_cached_examples/23/3D Gaussians ply format/e611dc2e9fbf1acd1cfc/_a toy robot..._013020.ply"", ""url"": ""/file=/tmp/gradio/3eb386a0bd35349d3271f7ac5431b26af1f8549b7a8a16c10af894877bb22c09/_a toy robot..._013020.ply"", ""size"": 1476719, ""orig_name"": ""[]_[a toy robot...]_013020.ply"", ""mime_type"": null, ""is_stream"": false, ""meta"": {""_type"": ""gradio.FileData""}}",,,2025-02-05 17:53:34.441679
-"{""path"": ""gradio_cached_examples/23/component 0/c4591e8b87a757447a47/image.webp"", ""url"": ""/file=/tmp/gradio/7879af8e96c9236c48faefd790d084d712f2f5f45fbfe2dc54798540be16479f/image.webp"", ""size"": null, ""orig_name"": ""image.webp"", ""mime_type"": null, ""is_stream"": false, ""meta"": {""_type"": ""gradio.FileData""}}","{""video"": {""path"": ""gradio_cached_examples/23/video/8e15e0239b0523adb9ea/_a cute panda..._013020.mp4"", ""url"": ""/file=/tmp/gradio/f9588f3f7218e15a80c4c8393b1335b3d70cbedc1f9ff91347daae9945325bd7/_a cute panda..._013020.mp4"", ""size"": null, ""orig_name"": ""[]_[a cute panda...]_013020.mp4"", ""mime_type"": null, ""is_stream"": false, ""meta"": {""_type"": ""gradio.FileData""}}, ""subtitles"": null}","{""path"": ""gradio_cached_examples/23/3D Gaussians ply format/743ad8490995e372c5f3/_a cute panda..._013020.ply"", ""url"": ""/file=/tmp/gradio/2ad3fb46683cdbfdb83050203762230a7a3d4889bb133f4376b8da0e3119fd1d/_a cute panda..._013020.ply"", ""size"": 2347854, ""orig_name"": ""[]_[a cute panda...]_013020.ply"", ""mime_type"": null, ""is_stream"": false, ""meta"": {""_type"": ""gradio.FileData""}}",,,2025-02-05 17:54:24.100453
-"{""path"": ""gradio_cached_examples/23/component 0/31e3ecdad1b51eb1d11e/image.webp"", ""url"": ""/file=/tmp/gradio/0381c7b9d1a19b575d2568376ee9bfa18292716cc1e45ae95bb4528ca92557d6/image.webp"", ""size"": null, ""orig_name"": ""image.webp"", ""mime_type"": null, ""is_stream"": false, ""meta"": {""_type"": ""gradio.FileData""}}","{""video"": {""path"": ""gradio_cached_examples/23/video/09491404fda7bfb0eab7/_a book..._013020.mp4"", ""url"": ""/file=/tmp/gradio/3d8e3fa54c906aff498f443284df7d5b2e9e9e8018558ea4f3199074aa342813/_a book..._013020.mp4"", ""size"": null, ""orig_name"": ""[]_[a book...]_013020.mp4"", ""mime_type"": null, ""is_stream"": false, ""meta"": {""_type"": ""gradio.FileData""}}, ""subtitles"": null}","{""path"": ""gradio_cached_examples/23/3D Gaussians ply format/1174ce83af0bcabb215a/_a book..._013020.ply"", ""url"": ""/file=/tmp/gradio/191f72e3e00089869450ba9fe70cab622e94089489c0cf5104a724f2fa98d5ee/_a book..._013020.ply"", ""size"": 2224072, ""orig_name"": ""[]_[a book...]_013020.ply"", ""mime_type"": null, ""is_stream"": false, ""meta"": {""_type"": ""gradio.FileData""}}",,,2025-02-05 17:55:14.451860
+"{""path"": ""gradio_cached_examples/23/component 0/e4f43cec7e020a66d596/image.webp"", ""url"": ""/file=/tmp/gradio/dbf713c255683bb9a0817141f85a0a122fcfda997c9ee184c11cadc4ec699dbf/image.webp"", ""size"": null, ""orig_name"": ""image.webp"", ""mime_type"": null, ""is_stream"": false, ""meta"": {""_type"": ""gradio.FileData""}}","{""video"": {""path"": ""gradio_cached_examples/23/video/3f82eb6a8d5e1cc88e28/_a toy robot..._013020.mp4"", ""url"": ""/file=/tmp/gradio/93e4187e3359a1457f501416c2ee86ed9e0ab485007ff0784cd226ac2f6e8e0b/_a toy robot..._013020.mp4"", ""size"": null, ""orig_name"": ""[]_[a toy robot...]_013020.mp4"", ""mime_type"": null, ""is_stream"": false, ""meta"": {""_type"": ""gradio.FileData""}}, ""subtitles"": null}","{""path"": ""gradio_cached_examples/23/3D Gaussians ply format/a6d4bba77a6f190743e6/_a toy robot..._013020.ply"", ""url"": ""/file=/tmp/gradio/3eb386a0bd35349d3271f7ac5431b26af1f8549b7a8a16c10af894877bb22c09/_a toy robot..._013020.ply"", ""size"": 1476719, ""orig_name"": ""[]_[a toy robot...]_013020.ply"", ""mime_type"": null, ""is_stream"": false, ""meta"": {""_type"": ""gradio.FileData""}}",,,2025-02-06 12:39:16.678627

requirements.txt

@@ -29,4 +29,6 @@ spaces
 triton
 wandb
 opencv-python==4.11.0.86
+https://huggingface.co/spaces/JeffreyXiang/TRELLIS/resolve/main/wheels/diff_gaussian_rasterization-0.0.0-cp310-cp310-linux_x86_64.whl?download=true
+
 

src/infer_gsdiff_sd.py

@@ -347,13 +347,6 @@ def main():
         level=logging.INFO
     )
     logger = logging.getLogger(__name__)
-    # file_handler = logging.FileHandler(os.path.join(args.output_dir, args.tag, "log_infer.txt"))  # output to file
-    # file_handler.setFormatter(logging.Formatter(
-    #     fmt="%(asctime)s - %(message)s",
-    #     datefmt="%Y/%m/%d %H:%M:%S"
-    # ))
-    # logger.addHandler(file_handler)
-    # logger.propagate = True  # propagate to the root logger (console)
 
     # Set the random seed
     if args.seed >= 0:
@@ -368,7 +361,7 @@ def main():
     if (args.image_path is not None or args.image_dir is not None) and args.load_pretrained_controlnet is None:
         opt.prediction_type = "v_prediction"
         opt.view_concat_condition = True
-        opt.input_concat_binary_mask = False
+        opt.input_concat_binary_mask = True
         if args.guidance_scale > 3.:
             logger.info(
                 f"WARNING: guidance scale ({args.guidance_scale}) is too large for image-conditioned models. " +

src/models/gs_render/gs_renderer.py

@@ -104,9 +104,9 @@ class GaussianRenderer:
                             self.opt.vis_coords or self.opt.vis_normals,  # whether render depth & normal
                     )
                     images[i, j] = render_results["image"]
-                    alphas[i, j] = render_results["alpha"]
-                    depths[i, j] = render_results["depth"]
-                    normals[i, j] = render_results["normal"]
+                    # alphas[i, j] = render_results["alpha"]
+                    # depths[i, j] = render_results["depth"]
+                    # normals[i, j] = render_results["normal"]
 
         if not isinstance(bg_color, Tensor):
             bg_color = torch.tensor(list(bg_color), dtype=torch.float32, device=device)

src/models/gs_render/gs_util.py

@@ -179,12 +179,13 @@ def render(
         bg_color = bg_color.to(C2W.device, dtype=torch.float32)
 
     pc = pc.to(dtype=torch.float32)
-
+    subpixel_offset = torch.zeros((int(viewpoint_camera.h), int(viewpoint_camera.w), 2), dtype=torch.float32, device="cuda")
     raster_settings = GaussianRasterizationSettings(
         image_height=int(viewpoint_camera.h),
         image_width=int(viewpoint_camera.w),
         tanfovx=viewpoint_camera.tanfovX,
         tanfovy=viewpoint_camera.tanfovY,
+        subpixel_offset=subpixel_offset,
         kernel_size=0.,  # cf. Mip-Splatting; not used
         bg=bg_color,
         scale_modifier=scaling_modifier,
@@ -194,13 +195,13 @@ def render(
         campos=viewpoint_camera.camera_center,
         prefiltered=False,
         debug=False,
-        # cf. RaDe-GS
-        require_depth=render_dn,
-        require_coord=False,
+
     )
+    alpha = normal = depth = None
+    
     rasterizer = GaussianRasterizer(raster_settings=raster_settings)
-
-    image, _, _, _, depth, _, alpha, normal = rasterizer(  # not used: radii, coord, mcoord, mdepth
+    # Rasterize visible Gaussians to image, obtain their radii (on screen).
+    image, radii = rasterizer(  # not used: radii, coord, mcoord, mdepth
         means3D=pc.xyz,
         means2D=torch.zeros_like(pc.xyz, dtype=torch.float32, device=pc.xyz.device),
         shs=None,