hysts HF staff commited on
Commit
396e795
1 Parent(s): 4396a04

Upload . with huggingface_hub

Browse files
.gitattributes CHANGED
@@ -32,3 +32,7 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
32
  *.zip filter=lfs diff=lfs merge=lfs -text
33
  *.zst filter=lfs diff=lfs merge=lfs -text
34
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
32
  *.zip filter=lfs diff=lfs merge=lfs -text
33
  *.zst filter=lfs diff=lfs merge=lfs -text
34
  *tfevents* filter=lfs diff=lfs merge=lfs -text
35
+ samples/sample-100.gif filter=lfs diff=lfs merge=lfs -text
36
+ samples/sample-200.gif filter=lfs diff=lfs merge=lfs -text
37
+ samples/sample-100/A[[:space:]]panda[[:space:]]is[[:space:]]surfing.gif filter=lfs diff=lfs merge=lfs -text
38
+ samples/sample-200/A[[:space:]]panda[[:space:]]is[[:space:]]surfing.gif filter=lfs diff=lfs merge=lfs -text
README.md ADDED
@@ -0,0 +1,23 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ license: creativeml-openrail-m
3
+ base_model: CompVis/stable-diffusion-v1-4
4
+ training_prompt: A man is surfing
5
+ tags:
6
+ - stable-diffusion
7
+ - stable-diffusion-diffusers
8
+ - text-to-image
9
+ - diffusers
10
+ - text-to-video
11
+ - tune-a-video
12
+ inference: false
13
+ ---
14
+
15
+ # Tune-A-Video - a-man-is-surfing
16
+
17
+ Base model: [CompVis/stable-diffusion-v1-4](https://huggingface.co/CompVis/stable-diffusion-v1-4).
18
+
19
+ Training prompt: A man is surfing
20
+
21
+ Test prompt: A panda is surfing
22
+ ![sample-300](samples/sample-300.gif)
23
+
config.yaml ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ pretrained_model_path: checkpoints/CompVis/stable-diffusion-v1-4
2
+ output_dir: /home/user/app/experiments/a-man-is-surfing
3
+ train_data:
4
+ video_path: /tmp/man-surfing6ja7t25j.mp4
5
+ prompt: A man is surfing
6
+ n_sample_frames: 8
7
+ width: 512
8
+ height: 512
9
+ sample_start_idx: 0
10
+ sample_frame_rate: 1
11
+ validation_data:
12
+ prompts:
13
+ - A panda is surfing
14
+ video_length: 8
15
+ width: 512
16
+ height: 512
17
+ num_inference_steps: 50
18
+ guidance_scale: 7.5
19
+ validation_steps: 100
20
+ trainable_modules:
21
+ - attn1.to_q
22
+ - attn2.to_q
23
+ - attn_temp
24
+ train_batch_size: 1
25
+ max_train_steps: 300
26
+ learning_rate: 3.5e-05
27
+ scale_lr: false
28
+ lr_scheduler: constant
29
+ lr_warmup_steps: 0
30
+ adam_beta1: 0.9
31
+ adam_beta2: 0.999
32
+ adam_weight_decay: 0.01
33
+ adam_epsilon: 1.0e-08
34
+ max_grad_norm: 1.0
35
+ gradient_accumulation_steps: 1
36
+ gradient_checkpointing: true
37
+ checkpointing_steps: 1000
38
+ resume_from_checkpoint: null
39
+ mixed_precision: fp16
40
+ use_8bit_adam: false
41
+ enable_xformers_memory_efficient_attention: true
42
+ seed: 33
model_index.json ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_class_name": "TuneAVideoPipeline",
3
+ "_diffusers_version": "0.11.1",
4
+ "scheduler": [
5
+ "diffusers",
6
+ "PNDMScheduler"
7
+ ],
8
+ "text_encoder": [
9
+ "transformers",
10
+ "CLIPTextModel"
11
+ ],
12
+ "tokenizer": [
13
+ "transformers",
14
+ "CLIPTokenizer"
15
+ ],
16
+ "unet": [
17
+ "models",
18
+ "UNet3DConditionModel"
19
+ ],
20
+ "vae": [
21
+ "diffusers",
22
+ "AutoencoderKL"
23
+ ]
24
+ }
samples/sample-100.gif ADDED

Git LFS Details

  • SHA256: 21bd1d82f123fd9fdd52fb0194cfe761557923052c0709dc3ec5c0e48467b5fb
  • Pointer size: 132 Bytes
  • Size of remote file: 1.02 MB
samples/sample-100/A panda is surfing.gif ADDED

Git LFS Details

  • SHA256: 21bd1d82f123fd9fdd52fb0194cfe761557923052c0709dc3ec5c0e48467b5fb
  • Pointer size: 132 Bytes
  • Size of remote file: 1.02 MB
samples/sample-200.gif ADDED

Git LFS Details

  • SHA256: 9d649b9b1174b6afae4e3ad28b86b6870d74af18d01ae16e4ef7724f9c1d7294
  • Pointer size: 132 Bytes
  • Size of remote file: 1.04 MB
samples/sample-200/A panda is surfing.gif ADDED

Git LFS Details

  • SHA256: 9d649b9b1174b6afae4e3ad28b86b6870d74af18d01ae16e4ef7724f9c1d7294
  • Pointer size: 132 Bytes
  • Size of remote file: 1.04 MB
samples/sample-300.gif ADDED
samples/sample-300/A panda is surfing.gif ADDED
scheduler/scheduler_config.json ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_class_name": "PNDMScheduler",
3
+ "_diffusers_version": "0.11.1",
4
+ "beta_end": 0.012,
5
+ "beta_schedule": "scaled_linear",
6
+ "beta_start": 0.00085,
7
+ "clip_sample": false,
8
+ "num_train_timesteps": 1000,
9
+ "prediction_type": "epsilon",
10
+ "set_alpha_to_one": false,
11
+ "skip_prk_steps": true,
12
+ "steps_offset": 1,
13
+ "trained_betas": null
14
+ }
text_encoder/config.json ADDED
@@ -0,0 +1,25 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "checkpoints/CompVis/stable-diffusion-v1-4",
3
+ "architectures": [
4
+ "CLIPTextModel"
5
+ ],
6
+ "attention_dropout": 0.0,
7
+ "bos_token_id": 0,
8
+ "dropout": 0.0,
9
+ "eos_token_id": 2,
10
+ "hidden_act": "quick_gelu",
11
+ "hidden_size": 768,
12
+ "initializer_factor": 1.0,
13
+ "initializer_range": 0.02,
14
+ "intermediate_size": 3072,
15
+ "layer_norm_eps": 1e-05,
16
+ "max_position_embeddings": 77,
17
+ "model_type": "clip_text_model",
18
+ "num_attention_heads": 12,
19
+ "num_hidden_layers": 12,
20
+ "pad_token_id": 1,
21
+ "projection_dim": 512,
22
+ "torch_dtype": "float16",
23
+ "transformers_version": "4.26.0",
24
+ "vocab_size": 49408
25
+ }
text_encoder/pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6a34f30098988d85dc0fb0fc272a842ebcf552e2ebc6ce4adbcf3695d08e8a90
3
+ size 246188833
tokenizer/merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer/special_tokens_map.json ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": {
3
+ "content": "<|startoftext|>",
4
+ "lstrip": false,
5
+ "normalized": true,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "eos_token": {
10
+ "content": "<|endoftext|>",
11
+ "lstrip": false,
12
+ "normalized": true,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "pad_token": "<|endoftext|>",
17
+ "unk_token": {
18
+ "content": "<|endoftext|>",
19
+ "lstrip": false,
20
+ "normalized": true,
21
+ "rstrip": false,
22
+ "single_word": false
23
+ }
24
+ }
tokenizer/tokenizer_config.json ADDED
@@ -0,0 +1,34 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "bos_token": {
4
+ "__type": "AddedToken",
5
+ "content": "<|startoftext|>",
6
+ "lstrip": false,
7
+ "normalized": true,
8
+ "rstrip": false,
9
+ "single_word": false
10
+ },
11
+ "do_lower_case": true,
12
+ "eos_token": {
13
+ "__type": "AddedToken",
14
+ "content": "<|endoftext|>",
15
+ "lstrip": false,
16
+ "normalized": true,
17
+ "rstrip": false,
18
+ "single_word": false
19
+ },
20
+ "errors": "replace",
21
+ "model_max_length": 77,
22
+ "name_or_path": "checkpoints/CompVis/stable-diffusion-v1-4/tokenizer",
23
+ "pad_token": "<|endoftext|>",
24
+ "special_tokens_map_file": "./special_tokens_map.json",
25
+ "tokenizer_class": "CLIPTokenizer",
26
+ "unk_token": {
27
+ "__type": "AddedToken",
28
+ "content": "<|endoftext|>",
29
+ "lstrip": false,
30
+ "normalized": true,
31
+ "rstrip": false,
32
+ "single_word": false
33
+ }
34
+ }
tokenizer/vocab.json ADDED
The diff for this file is too large to render. See raw diff
 
unet/config.json ADDED
@@ -0,0 +1,44 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_class_name": "UNet3DConditionModel",
3
+ "_diffusers_version": "0.11.1",
4
+ "act_fn": "silu",
5
+ "attention_head_dim": 8,
6
+ "block_out_channels": [
7
+ 320,
8
+ 640,
9
+ 1280,
10
+ 1280
11
+ ],
12
+ "center_input_sample": false,
13
+ "class_embed_type": null,
14
+ "cross_attention_dim": 768,
15
+ "down_block_types": [
16
+ "CrossAttnDownBlock3D",
17
+ "CrossAttnDownBlock3D",
18
+ "CrossAttnDownBlock3D",
19
+ "DownBlock3D"
20
+ ],
21
+ "downsample_padding": 1,
22
+ "dual_cross_attention": false,
23
+ "flip_sin_to_cos": true,
24
+ "freq_shift": 0,
25
+ "in_channels": 4,
26
+ "layers_per_block": 2,
27
+ "mid_block_scale_factor": 1,
28
+ "mid_block_type": "UNetMidBlock3DCrossAttn",
29
+ "norm_eps": 1e-05,
30
+ "norm_num_groups": 32,
31
+ "num_class_embeds": null,
32
+ "only_cross_attention": false,
33
+ "out_channels": 4,
34
+ "resnet_time_scale_shift": "default",
35
+ "sample_size": 64,
36
+ "up_block_types": [
37
+ "UpBlock3D",
38
+ "CrossAttnUpBlock3D",
39
+ "CrossAttnUpBlock3D",
40
+ "CrossAttnUpBlock3D"
41
+ ],
42
+ "upcast_attention": false,
43
+ "use_linear_projection": false
44
+ }
unet/diffusion_pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0e12eaefdc69c0d94e9f196822bf2a2d712cc479754cadb4114c8ae09cacacab
3
+ size 3636825029
vae/config.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_class_name": "AutoencoderKL",
3
+ "_diffusers_version": "0.11.1",
4
+ "_name_or_path": "checkpoints/CompVis/stable-diffusion-v1-4",
5
+ "act_fn": "silu",
6
+ "block_out_channels": [
7
+ 128,
8
+ 256,
9
+ 512,
10
+ 512
11
+ ],
12
+ "down_block_types": [
13
+ "DownEncoderBlock2D",
14
+ "DownEncoderBlock2D",
15
+ "DownEncoderBlock2D",
16
+ "DownEncoderBlock2D"
17
+ ],
18
+ "in_channels": 3,
19
+ "latent_channels": 4,
20
+ "layers_per_block": 2,
21
+ "norm_num_groups": 32,
22
+ "out_channels": 3,
23
+ "sample_size": 512,
24
+ "up_block_types": [
25
+ "UpDecoderBlock2D",
26
+ "UpDecoderBlock2D",
27
+ "UpDecoderBlock2D",
28
+ "UpDecoderBlock2D"
29
+ ]
30
+ }
vae/diffusion_pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3524731ccb7c457ec5481e252c774463677e51ef4df7fccb6f88534b632acf92
3
+ size 167407601