Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+asset/teaser.jpg filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,23 @@
+---
+license: other
+library_name: DiffusionRenderer
+arxiv: 2501.18590
+tags:
+- nvidia
+- rendering
+- diffusion
+---
+
+# DiffusionRenderer: Neural Inverse and Forward Rendering with Video Diffusion Models
+
+![img](asset/teaser.jpg)
+
+[Ruofan Liang](https://www.cs.toronto.edu/~ruofan/)\*, [Zan Gojcic](https://zgojcic.github.io/), [Huan Ling](https://www.cs.toronto.edu/~linghuan/),  [Jacob Munkberg](https://research.nvidia.com/person/jacob-munkberg), [Jon Hasselgren](https://research.nvidia.com/person/jon-hasselgren), [Zhi-Hao Lin](https://chih-hao-lin.github.io/), [Jun Gao](https://www.cs.toronto.edu/~jungao/), [Alexander Keller](https://research.nvidia.com/person/alex-keller), [Nandita Vijaykumar](https://www.cs.toronto.edu/~nandita/), [Sanja Fidler](https://www.cs.toronto.edu/~fidler/),   [Zian Wang](https://www.cs.toronto.edu/~zianwang/)\*
+
+\* indicates equal contribution
+
+**[Paper](https://arxiv.org/abs/2501.18590) | [Project Page](https://research.nvidia.com/labs/toronto-ai/DiffusionRenderer/)**
+
+**Overview.**
+DiffusionRenderer is a general-purpose framework that achieves high-quality geometry and material estimation from real-world videos (inverse rendering), and photorealistic image/video synthesis from G-buffers and lighting (forward rendering). Both the inverse and forward renderers are video diffusion models trained on a combination of curated synthetic datasets and auto-labeled real-world videos.
+Unlike classic PBR, which relies on precise geometry and path tracing, DiffusionRenderer provides a data-driven approximation of light transport through video GenAI models. It synthesizes realistic lighting effects without explicit simulation, complementing PBR for real-world applications such as relighting and material editing, especially when explicit geometry is inaccessible or inaccurate.

env_decoder/config.json

@@ -0,0 +1,23 @@
+{
+  "_class_name": "EnvDecoder",
+  "_diffusers_version": "0.28.0",
+  "_name_or_path": "outputs/env_encoding_latent/checkpoint-48000",
+  "act_fn": "none",
+  "channels": [
+    1280,
+    640,
+    320,
+    320
+  ],
+  "feat_input": false,
+  "latent_encoder": true,
+  "num_attention_heads": [
+    20,
+    20,
+    10,
+    5
+  ],
+  "num_res_blocks": 2,
+  "only_cross_attention": true,
+  "out_channels": 8
+}

env_decoder/diffusion_pytorch_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:66bbda72d25a3d6bf534241528053955a6d0da843b5a75ad879e9062bad58ad5
+size 639050104

env_encoder/config.json

@@ -0,0 +1,14 @@
+{
+  "_class_name": "EnvEncoder",
+  "_diffusers_version": "0.28.0",
+  "_name_or_path": "outputs/env_encoding_latent/checkpoint-48000",
+  "channels": [
+    320,
+    640,
+    1280,
+    1280
+  ],
+  "in_channels": 12,
+  "latent_encoder": true,
+  "num_res_blocks": 2
+}

env_encoder/diffusion_pytorch_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8b482aedf651bd9f0c70523168d828b0dba1fecfcfd027dc552c3283916a762c
+size 645859016

lora/pytorch_lora_weights.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aca396e76261c37c5a42b9d8f4c5fea2d4970847bf45b4998a61ebad49f83786
+size 13632920

model_index.json

@@ -0,0 +1,28 @@
+{
+  "_class_name": "RGBXVideoDiffusionPipeline",
+  "_diffusers_version": "0.28.0",
+  "env_encoder": [
+    "src.models.env_encoder",
+    "EnvEncoder"
+  ],
+  "feature_extractor": [
+    null,
+    null
+  ],
+  "image_encoder": [
+    null,
+    null
+  ],
+  "scheduler": [
+    "diffusers",
+    "EulerDiscreteScheduler"
+  ],
+  "unet": [
+    "src.models.custom_unet_st",
+    "UNetCustomSpatioTemporalConditionModel"
+  ],
+  "vae": [
+    "diffusers",
+    "AutoencoderKLTemporalDecoder"
+  ]
+}

nrm_encoder/config.json

@@ -0,0 +1,15 @@
+{
+  "_class_name": "EnvNormalEncoder",
+  "_diffusers_version": "0.28.0",
+  "_name_or_path": "outputs/env_encoding_latent/checkpoint-48000",
+  "channels": [
+    320,
+    640,
+    1280,
+    1280
+  ],
+  "in_channels": 4,
+  "latent_encoder": true,
+  "light_block": false,
+  "num_res_blocks": 2
+}

nrm_encoder/diffusion_pytorch_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e5cae950adce2f7a719f3223efb247896748d743f875240a90944d902ce26789
+size 645766856

scheduler/scheduler_config.json

@@ -0,0 +1,22 @@
+{
+  "_class_name": "EulerDiscreteScheduler",
+  "_diffusers_version": "0.28.0",
+  "beta_end": 0.012,
+  "beta_schedule": "scaled_linear",
+  "beta_start": 0.00085,
+  "clip_sample": false,
+  "final_sigmas_type": "zero",
+  "interpolation_type": "linear",
+  "num_train_timesteps": 1000,
+  "prediction_type": "v_prediction",
+  "rescale_betas_zero_snr": false,
+  "set_alpha_to_one": false,
+  "sigma_max": 700.0,
+  "sigma_min": 0.002,
+  "skip_prk_steps": true,
+  "steps_offset": 1,
+  "timestep_spacing": "leading",
+  "timestep_type": "continuous",
+  "trained_betas": null,
+  "use_karras_sigmas": true
+}

unet/config.json

@@ -0,0 +1,48 @@
+{
+  "_class_name": "UNetCustomSpatioTemporalConditionModel",
+  "_diffusers_version": "0.28.0",
+  "_name_or_path": "outputs/svd_render_env_enc_img_obj_X_dl3dvLoRA_ema_r16_vid_fs24/unet",
+  "addition_time_embed_dim": 256,
+  "block_out_channels": [
+    320,
+    640,
+    1280,
+    1280
+  ],
+  "context_embedding_type": null,
+  "context_vocab_size": 16,
+  "cross_attention_dim": [
+    320,
+    640,
+    1280,
+    1280
+  ],
+  "down_block_types": [
+    "CrossAttnDownBlockSpatioTemporal",
+    "CrossAttnDownBlockSpatioTemporal",
+    "CrossAttnDownBlockSpatioTemporal",
+    "DownBlockSpatioTemporal"
+  ],
+  "in_channels": 24,
+  "layers_per_block": 2,
+  "multi_res_encoder_hidden_states": true,
+  "num_attention_heads": [
+    5,
+    10,
+    20,
+    20
+  ],
+  "num_frames": 14,
+  "out_channels": 4,
+  "projection_class_embeddings_input_dim": 768,
+  "sample_size": 96,
+  "temporal_cross_attention_dim": null,
+  "transformer_layers_per_block": 1,
+  "up_block_types": [
+    "UpBlockSpatioTemporal",
+    "CrossAttnUpBlockSpatioTemporal",
+    "CrossAttnUpBlockSpatioTemporal",
+    "CrossAttnUpBlockSpatioTemporal"
+  ],
+  "use_skip_1_attention": true
+}

unet/diffusion_pytorch_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a4a08509483087c40e091294ec944150c2a0183c664f8f1dffcbde5a1d7389f2
+size 5894228272

vae/config.json

@@ -0,0 +1,24 @@
+{
+  "_class_name": "AutoencoderKLTemporalDecoder",
+  "_diffusers_version": "0.28.0",
+  "_name_or_path": "stabilityai/stable-video-diffusion-img2vid",
+  "block_out_channels": [
+    128,
+    256,
+    512,
+    512
+  ],
+  "down_block_types": [
+    "DownEncoderBlock2D",
+    "DownEncoderBlock2D",
+    "DownEncoderBlock2D",
+    "DownEncoderBlock2D"
+  ],
+  "force_upcast": true,
+  "in_channels": 3,
+  "latent_channels": 4,
+  "layers_per_block": 2,
+  "out_channels": 3,
+  "sample_size": 768,
+  "scaling_factor": 0.18215
+}

vae/diffusion_pytorch_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af602cd0eb4ad6086ec94fbf1438dfb1be5ec9ac03fd0215640854e90d6463a3
+size 195531910