Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,13 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+samples/sample-1/A[[:space:]]360[[:space:]]degree[[:space:]]view[[:space:]]video[[:space:]]of[[:space:]]a[[:space:]]street[[:space:]]with[[:space:]]Skies,[[:space:]]Walls,[[:space:]]Roads,[[:space:]]Poles,[[:space:]]Buildings,[[:space:]]Riders,[[:space:]]Sidewalks,[[:space:]]People,[[:space:]]Traffic[[:space:]]signs,[[:space:]]Bicycles[[:space:]]and[[:space:]]Cars.gif filter=lfs diff=lfs merge=lfs -text
+samples/sample-1.gif filter=lfs diff=lfs merge=lfs -text
+samples/sample-2/A[[:space:]]360[[:space:]]degree[[:space:]]view[[:space:]]video[[:space:]]of[[:space:]]a[[:space:]]street[[:space:]]with[[:space:]]Skies,[[:space:]]Walls,[[:space:]]Roads,[[:space:]]Poles,[[:space:]]Buildings,[[:space:]]Riders,[[:space:]]Sidewalks,[[:space:]]People,[[:space:]]Traffic[[:space:]]signs,[[:space:]]Bicycles[[:space:]]and[[:space:]]Cars.gif filter=lfs diff=lfs merge=lfs -text
+samples/sample-2.gif filter=lfs diff=lfs merge=lfs -text
+samples/sample-3/A[[:space:]]360[[:space:]]degree[[:space:]]view[[:space:]]video[[:space:]]of[[:space:]]a[[:space:]]street[[:space:]]with[[:space:]]Skies,[[:space:]]Walls,[[:space:]]Roads,[[:space:]]Poles,[[:space:]]Buildings,[[:space:]]Riders,[[:space:]]Sidewalks,[[:space:]]People,[[:space:]]Traffic[[:space:]]signs,[[:space:]]Bicycles[[:space:]]and[[:space:]]Cars.gif filter=lfs diff=lfs merge=lfs -text
+samples/sample-3.gif filter=lfs diff=lfs merge=lfs -text
+samples/sample-4/A[[:space:]]360[[:space:]]degree[[:space:]]view[[:space:]]video[[:space:]]of[[:space:]]a[[:space:]]street[[:space:]]with[[:space:]]Skies,[[:space:]]Walls,[[:space:]]Roads,[[:space:]]Poles,[[:space:]]Buildings,[[:space:]]Riders,[[:space:]]Sidewalks,[[:space:]]People,[[:space:]]Traffic[[:space:]]signs,[[:space:]]Bicycles[[:space:]]and[[:space:]]Cars.gif filter=lfs diff=lfs merge=lfs -text
+samples/sample-4.gif filter=lfs diff=lfs merge=lfs -text
+samples/sample-5/A[[:space:]]360[[:space:]]degree[[:space:]]view[[:space:]]video[[:space:]]of[[:space:]]a[[:space:]]street[[:space:]]with[[:space:]]Skies,[[:space:]]Walls,[[:space:]]Roads,[[:space:]]Poles,[[:space:]]Buildings,[[:space:]]Riders,[[:space:]]Sidewalks,[[:space:]]People,[[:space:]]Traffic[[:space:]]signs,[[:space:]]Bicycles[[:space:]]and[[:space:]]Cars.gif filter=lfs diff=lfs merge=lfs -text
+samples/sample-5.gif filter=lfs diff=lfs merge=lfs -text

checkpoint-5/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:16f2a720b0c3ec30a70984dbb101aba172344c3461d7fe4be72d33902d09bece
+size 3636579464

checkpoint-5/optimizer.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c0f7f30c4656b5a1c9c749b0cb43d1f70de6e9472f57240560afd1b094367391
+size 388426170

checkpoint-5/random_states_0.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f4b2d54673c974d525cc73197489f8e33c2e021242b8e0dd57cf501821f772e
+size 14408

checkpoint-5/scaler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb5a5ad08868fe42e2f75cacb3b64bd2ffd86dde2774d59af9d8eee9099fbd29
+size 988

checkpoint-5/scheduler.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:37a83e77b836a961106201f255a7debe337b3c696ff438fe2cbbd7d10f551a95
+size 1000

config.yaml

@@ -0,0 +1,46 @@
+pretrained_model_path: CompVis/stable-diffusion-v1-4
+output_dir: /content/Tune-A-Video/outputs/city
+train_data:
+  video_path: /content/Tune-A-Video/data/city.mp4
+  prompt: A 360 degree view video of a street with Skies, Walls, Roads, Poles, Buildings,
+    Riders, Sidewalks, People, Traffic signs, Bicycles and Cars
+  n_sample_frames: 24
+  width: 512
+  height: 512
+  sample_start_idx: 0
+  sample_frame_rate: 1
+validation_data:
+  prompts:
+  - A 360 degree view video of a street with Skies, Walls, Roads, Poles, Buildings,
+    Riders, Sidewalks, People, Traffic signs, Bicycles and Cars
+  video_length: 24
+  width: 512
+  height: 512
+  num_inference_steps: 50
+  guidance_scale: 12.5
+  use_inv_latent: true
+  num_inv_steps: 50
+validation_steps: 1
+trainable_modules:
+- attn1.to_q
+- attn2.to_q
+- attn_temp
+train_batch_size: 1
+max_train_steps: 5
+learning_rate: 3.0e-05
+scale_lr: false
+lr_scheduler: constant
+lr_warmup_steps: 0
+adam_beta1: 0.9
+adam_beta2: 0.999
+adam_weight_decay: 0.01
+adam_epsilon: 1.0e-08
+max_grad_norm: 1.0
+gradient_accumulation_steps: 1
+gradient_checkpointing: true
+checkpointing_steps: 5
+resume_from_checkpoint: null
+mixed_precision: fp16
+use_8bit_adam: false
+enable_xformers_memory_efficient_attention: true
+seed: 33

inv_latents/ddim_latent-1.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:24a0e4576ec092efebdc2226ad30afb7beb91dfde8603adb13222e50b5b5907e
+size 787642

inv_latents/ddim_latent-2.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9fc84357c2b51ca59dd86265f18b5420b53bfeb8b993bcdcafc418d35263faa8
+size 787642

inv_latents/ddim_latent-3.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7642b26649ec8f9ac600f26e9587b187378596060bb8a8dcd20453517af8ef23
+size 787642

inv_latents/ddim_latent-4.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:096583eebd41b60e52305a483298bcbdb76721d6221f321f7a07336a083de446
+size 787642

inv_latents/ddim_latent-5.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4512db318fa921f68a53acdd2a159ff779fbf021ec56e15f6375facf9c851631
+size 787642

model_index.json

@@ -0,0 +1,24 @@
+{
+  "_class_name": "TuneAVideoPipeline",
+  "_diffusers_version": "0.11.1",
+  "scheduler": [
+    "diffusers",
+    "PNDMScheduler"
+  ],
+  "text_encoder": [
+    "transformers",
+    "CLIPTextModel"
+  ],
+  "tokenizer": [
+    "transformers",
+    "CLIPTokenizer"
+  ],
+  "unet": [
+    "models",
+    "UNet3DConditionModel"
+  ],
+  "vae": [
+    "diffusers",
+    "AutoencoderKL"
+  ]
+}

scheduler/scheduler_config.json

@@ -0,0 +1,14 @@
+{
+  "_class_name": "PNDMScheduler",
+  "_diffusers_version": "0.11.1",
+  "beta_end": 0.012,
+  "beta_schedule": "scaled_linear",
+  "beta_start": 0.00085,
+  "clip_sample": false,
+  "num_train_timesteps": 1000,
+  "prediction_type": "epsilon",
+  "set_alpha_to_one": false,
+  "skip_prk_steps": true,
+  "steps_offset": 1,
+  "trained_betas": null
+}

text_encoder/config.json

@@ -0,0 +1,25 @@
+{
+  "_name_or_path": "CompVis/stable-diffusion-v1-4",
+  "architectures": [
+    "CLIPTextModel"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 0,
+  "dropout": 0.0,
+  "eos_token_id": 2,
+  "hidden_act": "quick_gelu",
+  "hidden_size": 768,
+  "initializer_factor": 1.0,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 77,
+  "model_type": "clip_text_model",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 1,
+  "projection_dim": 512,
+  "torch_dtype": "float16",
+  "transformers_version": "4.26.0",
+  "vocab_size": 49408
+}

text_encoder/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:87c1b379e54b3c87654e7661109bd1315019c3b357cf6237544002abf76d1258
+size 246189278

tokenizer/special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<|startoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<|endoftext|>",
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer/tokenizer_config.json

@@ -0,0 +1,34 @@
+{
+  "add_prefix_space": false,
+  "bos_token": {
+    "__type": "AddedToken",
+    "content": "<|startoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "do_lower_case": true,
+  "eos_token": {
+    "__type": "AddedToken",
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "errors": "replace",
+  "model_max_length": 77,
+  "name_or_path": "/root/.cache/huggingface/diffusers/models--CompVis--stable-diffusion-v1-4/snapshots/133a221b8aa7292a167afc5127cb63fb5005638b/tokenizer",
+  "pad_token": "<|endoftext|>",
+  "special_tokens_map_file": "./special_tokens_map.json",
+  "tokenizer_class": "CLIPTokenizer",
+  "unk_token": {
+    "__type": "AddedToken",
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

unet/config.json

@@ -0,0 +1,44 @@
+{
+  "_class_name": "UNet3DConditionModel",
+  "_diffusers_version": "0.11.1",
+  "act_fn": "silu",
+  "attention_head_dim": 8,
+  "block_out_channels": [
+    320,
+    640,
+    1280,
+    1280
+  ],
+  "center_input_sample": false,
+  "class_embed_type": null,
+  "cross_attention_dim": 768,
+  "down_block_types": [
+    "CrossAttnDownBlock3D",
+    "CrossAttnDownBlock3D",
+    "CrossAttnDownBlock3D",
+    "DownBlock3D"
+  ],
+  "downsample_padding": 1,
+  "dual_cross_attention": false,
+  "flip_sin_to_cos": true,
+  "freq_shift": 0,
+  "in_channels": 4,
+  "layers_per_block": 2,
+  "mid_block_scale_factor": 1,
+  "mid_block_type": "UNetMidBlock3DCrossAttn",
+  "norm_eps": 1e-05,
+  "norm_num_groups": 32,
+  "num_class_embeds": null,
+  "only_cross_attention": false,
+  "out_channels": 4,
+  "resnet_time_scale_shift": "default",
+  "sample_size": 64,
+  "up_block_types": [
+    "UpBlock3D",
+    "CrossAttnUpBlock3D",
+    "CrossAttnUpBlock3D",
+    "CrossAttnUpBlock3D"
+  ],
+  "upcast_attention": false,
+  "use_linear_projection": false
+}

unet/diffusion_pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:54c60dba721c0df484165e85c161fb4d660a9430185824982cc89fe347d9e717
+size 3636825494

vae/config.json

@@ -0,0 +1,31 @@
+{
+  "_class_name": "AutoencoderKL",
+  "_diffusers_version": "0.11.1",
+  "_name_or_path": "CompVis/stable-diffusion-v1-4",
+  "act_fn": "silu",
+  "block_out_channels": [
+    128,
+    256,
+    512,
+    512
+  ],
+  "down_block_types": [
+    "DownEncoderBlock2D",
+    "DownEncoderBlock2D",
+    "DownEncoderBlock2D",
+    "DownEncoderBlock2D"
+  ],
+  "in_channels": 3,
+  "latent_channels": 4,
+  "layers_per_block": 2,
+  "norm_num_groups": 32,
+  "out_channels": 3,
+  "sample_size": 512,
+  "scaling_factor": 0.18215,
+  "up_block_types": [
+    "UpDecoderBlock2D",
+    "UpDecoderBlock2D",
+    "UpDecoderBlock2D",
+    "UpDecoderBlock2D"
+  ]
+}

vae/diffusion_pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:95bf483e6eadb4a0d0b0968b36156445ce4dfcf5961910b0a9a5a2e10c8a715b
+size 167408066