Upload folder using huggingface_hub

ckpts/slat_dec_gs_swin8_B_64l8gs32_fp16.json

@@ -0,0 +1,31 @@
+{
+    "name": "SLatGaussianDecoder",
+    "args": {
+        "resolution": 64,
+        "model_channels": 768,
+        "latent_channels": 8,
+        "num_blocks": 12,
+        "num_heads": 12,
+        "mlp_ratio": 4,
+        "attn_mode": "swin",
+        "window_size": 8,
+        "use_fp16": true,
+        "representation_config": {
+            "lr": {
+                "_xyz": 1.0,
+                "_features_dc": 1.0,
+                "_opacity": 1.0,
+                "_scaling": 1.0,
+                "_rotation": 0.1
+            },
+            "perturb_offset": true,
+            "voxel_size": 1.5,
+            "num_gaussians": 32,
+            "2d_filter_kernel_size": 0.1,
+            "3d_filter_kernel_size": 9e-4,
+            "scaling_bias": 4e-3,
+            "opacity_bias": 0.1,
+            "scaling_activation": "softplus"
+        }
+    }
+}

ckpts/slat_dec_gs_swin8_B_64l8gs32_fp16.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:38c84bcef5ce0af1f48b1b5558dabc7575a13346043c41a7e0610f1fa619a161
+size 171450952

ckpts/slat_dec_mesh_swin8_B_64l8m256c_fp16.json

@@ -0,0 +1,17 @@
+{
+    "name": "SLatMeshDecoder",
+    "args": {
+        "resolution": 64,
+        "model_channels": 768,
+        "latent_channels": 8,
+        "num_blocks": 12,
+        "num_heads": 12,
+        "mlp_ratio": 4,
+        "attn_mode": "swin",
+        "window_size": 8,
+        "use_fp16": true,
+        "representation_config": {
+            "use_color": true
+        }
+    }
+}

ckpts/slat_dec_mesh_swin8_B_64l8m256c_fp16.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e87aba94b5786407eb06d0502c1ed0885a0027a3f2b8537bfe15b0a92c01859
+size 181903412

ckpts/slat_dec_rf_swin8_B_64l8r16_fp16.json

@@ -0,0 +1,18 @@
+{
+    "name": "SLatRadianceFieldDecoder",
+    "args": {
+        "resolution": 64,
+        "model_channels": 768,
+        "latent_channels": 8,
+        "num_blocks": 12,
+        "num_heads": 12,
+        "mlp_ratio": 4,
+        "attn_mode": "swin",
+        "window_size": 8,
+        "use_fp16": true,
+        "representation_config": {
+            "rank": 16,
+            "dim": 8
+        }
+    }
+}

ckpts/slat_dec_rf_swin8_B_64l8r16_fp16.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:737da6578d01948016b7c39786113af0d64a46f7922f6b8b5e698b84643be514
+size 171450488

ckpts/ss_dec_conv3d_16l8_fp16.json

@@ -0,0 +1,12 @@
+
+{
+    "name": "SparseStructureDecoder",
+    "args": {
+        "out_channels": 1,
+        "latent_channels": 8,
+        "num_res_blocks": 2,
+        "num_res_blocks_middle": 2,
+        "channels": [512, 128, 32],
+        "use_fp16": true
+    }
+}

ckpts/ss_dec_conv3d_16l8_fp16.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c76d4a40519aa2d711cc263a8404105231ac26db31d946bed48b84fee79009a
+size 147591972

pipeline.json

@@ -0,0 +1,60 @@
+{
+    "name": "RecGenImageTo3DPipeline",
+    "args": {
+        "models": {
+            "sparse_structure_decoder": "ckpts/ss_dec_conv3d_16l8_fp16",
+            "sparse_structure_flow_model": "ckpts/ss_flow_img_dit_L_16l8_fp16",
+            "slat_decoder_gs": "ckpts/slat_dec_gs_swin8_B_64l8gs32_fp16",
+            "slat_decoder_rf": "ckpts/slat_dec_rf_swin8_B_64l8r16_fp16",
+            "slat_decoder_mesh": "ckpts/slat_dec_mesh_swin8_B_64l8m256c_fp16",
+            "slat_flow_model": "ckpts/slat_flow_img_dit_L_64l8p2_fp16"
+        },
+        "sparse_structure_sampler": {
+            "name": "FlowEulerGuidanceIntervalSampler",
+            "args": {
+                "sigma_min": 1e-5
+            },
+            "params": {
+                "steps": 25,
+                "cfg_strength": 5.0,
+                "cfg_interval": [0.5, 1.0],
+                "rescale_t": 3.0
+            }
+        },
+        "slat_sampler": {
+            "name": "FlowEulerGuidanceIntervalSampler",
+            "args": {
+                "sigma_min": 1e-5
+            },
+            "params": {
+                "steps": 25,
+                "cfg_strength": 5.0,
+                "cfg_interval": [0.5, 1.0],
+                "rescale_t": 3.0
+            }
+        },
+        "slat_normalization": {
+            "mean": [
+                -2.1687545776367188,
+                -0.004347046371549368,
+                -0.13352349400520325,
+                -0.08418072760105133,
+                -0.5271206498146057,
+                0.7238689064979553,
+                -1.1414450407028198,
+                1.2039363384246826
+            ],
+            "std": [
+                2.377650737762451,
+                2.386378288269043,
+                2.124418020248413,
+                2.1748552322387695,
+                2.663944721221924,
+                2.371192216873169,
+                2.6217446327209473,
+                2.684523105621338
+            ]
+        },
+        "image_cond_model": "dinov2_vitl14_reg"
+    }
+}

slat_config.json

@@ -0,0 +1,142 @@
+{
+    "models": {
+        "denoiser": {
+            "name": "ElasticSLatCondFlowModel",
+            "args": {
+                "resolution": 64,
+                "in_channels": 8,
+                "out_channels": 8,
+                "model_channels": 1024,
+                "cond_channels": 1024,
+                "num_blocks": 24,
+                "num_heads": 16,
+                "mlp_ratio": 4,
+                "patch_size": 2,
+                "num_io_res_blocks": 2,
+                "io_block_channels": [
+                    128
+                ],
+                "pe_mode": "ape",
+                "qk_rms_norm": true,
+                "use_fp16": true,
+                "use_point_embedder": true,
+                "point_embedder_out_channels": 1024,
+                "use_mask_embedder": true,
+                "mask_embedder_out_channels": 1024,
+                "use_pose_embedder": true,
+                "pose_embedder_out_channels": 1024,
+                "pose_representation": "9d_translation_scale"
+            }
+        }
+    },
+    "dataset": {
+        "name": "WebDatasetMultiViewStructuredLatent",
+        "s3_data_root": "s3://data_root",
+        "args": {
+            "shards": "ABO_wds/shard-{000000..000398},HSSD_wds/shard-{000000..000559},Objaverse_recgen_new_wds/shard-{000000..008374},Objaverse_recgen_new_wds/shard-{010000..018208},PartNeXt_wds_full/shard-{000000..001981}}.tar",
+            "num_views": 2,
+            "image_size": 518,
+            "min_aesthetic_score": 4.5,
+            "shuffle_buffer": 200,
+            "length": 189119,
+            "require_pose_data": true,
+            "require_pointmap": true,
+            "use_pose_normalization": true,
+            "pose_variant": "median_quantile_5per",
+            "mix_stereo_depth": true,
+            "normalization": {
+                "mean": [
+                    -2.1687545776367188,
+                    -0.004347046371549368,
+                    -0.13352349400520325,
+                    -0.08418072760105133,
+                    -0.5271206498146057,
+                    0.7238689064979553,
+                    -1.1414450407028198,
+                    1.2039363384246826
+                ],
+                "std": [
+                    2.377650737762451,
+                    2.386378288269043,
+                    2.124418020248413,
+                    2.1748552322387695,
+                    2.663944721221924,
+                    2.371192216873169,
+                    2.6217446327209473,
+                    2.684523105621338
+                ]
+            },
+            "metadata_root": "/tmp/wds_metadata"
+        }
+    },
+    "trainer": {
+        "name": "MultiImageConditionedSparseFlowMatchingCFGTrainer",
+        "args": {
+            "max_steps": 100000,
+            "batch_size_per_gpu": 8,
+            "batch_split": 2,
+            "finetune_ckpt": {
+                "denoiser": "checkpoints/TRELLIS-image-large/ckpts/slat_flow_img_dit_L_64l8p2_fp16.pt"
+            },
+            "optimizer": {
+                "name": "AdamW",
+                "args": {
+                    "lr": 0.0001,
+                    "weight_decay": 0.0
+                }
+            },
+            "ema_rate": [
+                0.9999
+            ],
+            "fp16_mode": "inflat_all",
+            "fp16_scale_growth": 0.001,
+            "elastic": {
+                "name": "LinearMemoryController",
+                "args": {
+                    "target_ratio": 0.85,
+                    "max_mem_ratio_start": 0.6
+                }
+            },
+            "grad_clip": {
+                "name": "AdaptiveGradClipper",
+                "args": {
+                    "max_norm": 1.0,
+                    "clip_percentile": 95
+                }
+            },
+            "i_print": 200,
+            "i_log": 200,
+            "i_sample": 1000,
+            "i_save": 5000,
+            "i_ddpcheck": 5000,
+            "p_uncond": 0.1,
+            "p_single_view": 0.33,
+            "single_view_drop_mode": "last",
+            "t_schedule": {
+                "name": "logitNormal",
+                "args": {
+                    "mean": 1.0,
+                    "std": 1.0
+                }
+            },
+            "sigma_min": 1e-05,
+            "image_cond_model": "dinov2_vitl14_reg",
+            "use_pose_conditioning": true,
+            "pose_representation": "9d_translation_scale",
+            "enable_snapshot": true
+        }
+    },
+    "use_wandb": true,
+    "wandb_name": "slat-multiview-2pose-s3",
+    "wandb_config": {
+        "experiment_type": "slat_multiview_2pose",
+        "datasets": [
+            "ABO_wds",
+            "HSSD_wds",
+            "Objaverse_recgen_new_wds",
+            "PartNeXt_wds"
+        ],
+        "num_views": 2,
+        "notes": "SLat multi-view training with per-view additive pose conditioning (9D rotation). Stereo depth mixing, flexible view dropout (33% single), shared CFG dropout (10% uncond). Robust pose normalization (median_quantile_5per). S3 streaming."
+    }
+}

slat_denoiser_ema0.9999_step0075000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:30e1a77ff0112bc44b4776d9581104a6540863e2cf8a3b6281e73c11c8a67e9d
+size 2405099466

slat_pose_stats.json

@@ -0,0 +1,92 @@
+{
+  "version": "1.0",
+  "total_samples": 3139624,
+  "total_filtered": 325187,
+  "total_skipped": 4,
+  "global_statistics": {
+    "6d_rotation": {
+      "mean": [
+        0.00672668259405818,
+        -0.00028748245331301585,
+        -0.00042603845627211357,
+        0.0003177030444939418,
+        -0.002702449791756494,
+        0.006512702087776858
+      ],
+      "std": [
+        0.6698880635345893,
+        0.5178697258359981,
+        0.531991934265617,
+        0.6696462546694643,
+        0.5176897297354682,
+        0.532467293998094
+      ],
+      "count": 3139624
+    },
+    "quaternion": {
+      "mean": [
+        0.40202742695880705,
+        0.3709366059007151,
+        -0.14091058618033667,
+        0.17420504011845106
+      ],
+      "std": [
+        0.48802215631403023,
+        0.5074279705934912,
+        0.28583868614616614,
+        0.27059983034656887
+      ],
+      "count": 3139624
+    },
+    "9d_rotation": {
+      "mean": [
+        0.00672668259405818,
+        0.0003177030444939418,
+        0.00010665358305283915,
+        -0.00028748245331301585,
+        -0.002702449791756494,
+        -0.6407421183758072,
+        -0.00042603845627211357,
+        0.006512702087776858,
+        -0.5897375744882928
+      ],
+      "std": [
+        0.6698880635345893,
+        0.6696462546694643,
+        0.32059089002866686,
+        0.5178697258359981,
+        0.5176897297354682,
+        0.23076057571940659,
+        0.531991934265617,
+        0.532467293998094,
+        0.2926263062024619
+      ],
+      "count": 3139624
+    },
+    "translation": {
+      "mean": [
+        0.49996764833757257,
+        0.49676753816020613,
+        0.6597641581284234
+      ],
+      "std": [
+        0.08953293305302792,
+        0.10581431792818304,
+        0.16211090593499025
+      ],
+      "count": 3139624
+    },
+    "scale": {
+      "mean": 1.2128176122797614,
+      "std": 0.29710298456089623,
+      "count": 3139624
+    }
+  },
+  "aggregated_from": [
+    "ABO_wds",
+    "HSSD_wds",
+    "Objaverse_recgen_new_wds",
+    "PartNeXt_wds_full"
+  ],
+  "pose_variant": "median_quantile_5per"
+}

stereo_config.json

@@ -0,0 +1,116 @@
+{
+    "models": {
+        "denoiser": {
+            "name": "SparseStructurePoseFlowModel",
+            "args": {
+                "resolution": 16,
+                "in_channels": 8,
+                "out_channels": 8,
+                "model_channels": 1024,
+                "cond_channels": 1024,
+                "num_blocks": 24,
+                "num_heads": 16,
+                "mlp_ratio": 4,
+                "patch_size": 1,
+                "pe_mode": "ape",
+                "qk_rms_norm": true,
+                "use_fp16": true,
+                "use_point_embedder": true,
+                "point_embedder_out_channels": 1024,
+                "use_mask_embedder": true,
+                "mask_embedder_out_channels": 1024,
+                "use_frame_token_embedder": true,
+                "pose_representation": "6d_translation_scale",
+                "num_pose_tokens": 2
+            }
+        }
+    },
+    "dataset": {
+        "name": "WebDatasetMultiViewSparseStructureLatent",
+        "s3_data_root": "s3://data_root",
+        "args": {
+            "shards": "{ABO_wds/shard-{000000..000346},HSSD_wds/shard-{000000..000519},Objaverse_recgen_new_wds/shard-{000000..015735},PhysX3DParts_wds/shard-{000000..000933},PartNeXt_wds/shard-{000000..001528},PartNetMobility_wds/shard-{000000..000212}}.tar",
+            "num_views": 2,
+            "min_aesthetic_score": 0.0,
+            "image_size": 518,
+            "shuffle_buffer": 200,
+            "length": 198455,
+            "require_pose_data": true,
+            "require_pointmap": true,
+            "use_pose_normalization": true,
+            "pose_variant": "median_quantile_5per",
+            "mix_stereo_depth": true,
+            "blacklist_file": "blacklist.txt",
+            "metadata_root": ""
+        }
+    },
+    "trainer": {
+        "name": "MultiImageConditionedFlowMatchingCFGTrainer",
+        "args": {
+            "max_steps": 100000,
+            "batch_size_per_gpu": 8,
+            "batch_split": 2,
+            "finetune_ckpt": {
+                "denoiser": "checkpoints/TRELLIS-image-large/ckpts/ss_flow_img_dit_L_16l8_fp16.pt"
+            },
+            "optimizer": {
+                "name": "AdamW",
+                "args": {
+                    "lr": 0.0001,
+                    "weight_decay": 0.0
+                }
+            },
+            "ema_rate": [
+                0.9999
+            ],
+            "fp16_mode": "inflat_all",
+            "fp16_scale_growth": 0.001,
+            "grad_clip": {
+                "name": "AdaptiveGradClipper",
+                "args": {
+                    "max_norm": 1.0,
+                    "clip_percentile": 95
+                }
+            },
+            "i_print": 200,
+            "i_log": 200,
+            "i_sample": 1000,
+            "i_save": 5000,
+            "i_ddpcheck": 5000,
+            "generate_alignment_overlays": false,
+            "num_alignment_samples": 4,
+            "p_uncond": 0.1,
+            "p_single_view": 0.33,
+            "single_view_drop_mode": "last",
+            "t_schedule": {
+                "name": "logitNormal",
+                "args": {
+                    "mean": 1.0,
+                    "std": 1.0
+                }
+            },
+            "sigma_min": 1e-05,
+            "pose_alpha": 0.01,
+            "pose_representation": "6d_translation_scale",
+            "use_pose_normalization": true,
+            "image_cond_model": "dinov2_vitl14_reg"
+        }
+    },
+    "use_wandb": true,
+    "wandb_name": "flexible-dropout-stereo-depth-frame-token-2pose-s3",
+    "wandb_config": {
+        "experiment_type": "multiview_flexible_dropout_stereo_depth_frame_token_2pose",
+        "datasets": [
+            "ABO_wds",
+            "HSSD_wds",
+            "Objaverse_recgen_new_wds",
+            "PhysX3DParts_wds",
+            "PartNeXt_wds",
+            "PartNetMobility_wds"
+        ],
+        "num_views": 2,
+        "num_pose_tokens": 2,
+        "view_dropout": "66% both / 33% single (independent) + 10% uncond (CFG)",
+        "notes": "Multi-view training with 2 pose tokens (one per view). Learned per-frame token-type embeddings. Stereo depth mixing (50/50), flexible view dropout (33% single, pose loss masked for dropped view), shared CFG dropout (10% uncond). Robust pose normalization (median_quantile_5per). S3 streaming."
+    }
+}

stereo_denoiser_ema0.9999_step0055000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a1ab25e2567a815120a3d99f66be981ab6335ce937ebb660919c8db0ef8466c9
+size 2242359709

stereo_pose_stats.json

@@ -0,0 +1,94 @@
+{
+  "version": "1.0",
+  "total_samples": 3399392,
+  "total_filtered": 333348,
+  "total_skipped": 0,
+  "global_statistics": {
+    "6d_rotation": {
+      "mean": [
+        0.009779387796884436,
+        -0.00044972407308648,
+        0.0006475642313313102,
+        -0.0007121744486558434,
+        -0.00437051658402508,
+        0.009057175830170358
+      ],
+      "std": [
+        0.6690212855917725,
+        0.5105222770906866,
+        0.5400752420705095,
+        0.6679740185732931,
+        0.5105687598057009,
+        0.5413211728859146
+      ],
+      "count": 3399392
+    },
+    "quaternion": {
+      "mean": [
+        0.4012234764027302,
+        0.36329670211172765,
+        -0.14045364267325602,
+        0.17961031083561407
+      ],
+      "std": [
+        0.4853347332025693,
+        0.5074156435655324,
+        0.29313469814290605,
+        0.27586182932082876
+      ],
+      "count": 3399392
+    },
+    "9d_rotation": {
+      "mean": [
+        0.009779387796884436,
+        -0.0007121744486558434,
+        9.94776112746347e-05,
+        -0.00044972407308648,
+        -0.00437051658402508,
+        -0.65288854961659,
+        0.0006475642313313102,
+        0.009057175830170358,
+        -0.5719704192662759
+      ],
+      "std": [
+        0.6690212855917725,
+        0.6679740185732931,
+        0.32576844500383073,
+        0.5105222770906866,
+        0.5105687598057009,
+        0.22891872969699903,
+        0.5400752420705095,
+        0.5413211728859146,
+        0.29674484742279506
+      ],
+      "count": 3399392
+    },
+    "translation": {
+      "mean": [
+        0.4999169444618038,
+        0.4941151961813019,
+        0.660972462961323
+      ],
+      "std": [
+        0.08863329549734027,
+        0.1052032812508121,
+        0.1608558079515348
+      ],
+      "count": 3399392
+    },
+    "scale": {
+      "mean": 1.20487141783028,
+      "std": 0.2952203182742648,
+      "count": 3399392
+    }
+  },
+  "aggregated_from": [
+    "ABO_wds",
+    "HSSD_wds",
+    "Objaverse_recgen_new_wds",
+    "PhysX3DParts_wds",
+    "PartNeXt_wds",
+    "PartNetMobility_wds"
+  ],
+  "pose_variant": "median_quantile_5per"
+}