Hugging Face's logo

WestAI-SC
/
high_fidelity_video_background_music_generation_with_transformers

Safetensors
English
Model card Files Community
3
Niklas Schulte commited on
Commit
0e8f8bc
•
1 Parent(s): 1313507

add configurations

Browse files
Files changed (13) hide show
  1. models_frozen_decoder/{symmv_small/dataset_split.json → dataset_split.json} +0 -0
  2. models_frozen_decoder/nature_large/configuration.yml +26 -0
  3. models_frozen_decoder/nature_medium/configuration.yml +26 -0
  4. models_frozen_decoder/nature_small/dataset_split.json +0 -0
  5. models_frozen_decoder/symmv_large/configuration.yml +26 -0
  6. models_frozen_decoder/symmv_medium/configuration.yml +26 -0
  7. models_peft/dataset_split.json +0 -0
  8. models_peft/nature_large/configuration.yml +26 -0
  9. models_peft/nature_medium/configuration.yml +26 -0
  10. models_peft/nature_small/configuration.yml +26 -0
  11. models_peft/symmv_large/configuration.yml +26 -0
  12. models_peft/symmv_medium/configuration.yml +26 -0
  13. models_peft/symmv_small/configuration.yml +26 -0
models_frozen_decoder/{symmv_small/dataset_split.json → dataset_split.json} RENAMED
File without changes
models_frozen_decoder/nature_large/configuration.yml ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ log_level: INFO
2
+ num_epochs: 20
3
+ batch_size: 2
4
+ save_step: 1800
5
+ device: cuda
6
+ use_wandb: true
7
+ wandb_project_name: thesis_audiocraft
8
+ gradient_clipping: 1.0
9
+ beta1: 0.9
10
+ beta2: 0.95
11
+ weight_decay: 0.1
12
+ learning_rate: 1.0e-05
13
+ warmup_steps: 16
14
+ video_extraction_framerate: 1
15
+ encoder_input_dimension: 2048
16
+ encoder_dim_feedforward: 32768
17
+ encoder_heads: 64
18
+ encoder_layers: 24
19
+ musicgen_model_id: facebook/musicgen-stereo-large
20
+ use_peft: false
21
+ output_dir_name: audiocraft_nature_large
22
+ dataset_video_folder: ../../datasets/nature2/videos_30sec
23
+ dataset_audio_folder: ../../datasets/nature2/audio_30sec
24
+ use_demucs_folder_structure: false
25
+ dataset_validation_split: 0.1
26
+ dataset_shuffling_seed: 42
models_frozen_decoder/nature_medium/configuration.yml ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ log_level: INFO
2
+ num_epochs: 20
3
+ batch_size: 8
4
+ save_step: 350
5
+ device: cuda
6
+ use_wandb: true
7
+ wandb_project_name: thesis_audiocraft
8
+ gradient_clipping: 1.0
9
+ beta1: 0.9
10
+ beta2: 0.95
11
+ weight_decay: 0.1
12
+ learning_rate: 1.0e-05
13
+ warmup_steps: 16
14
+ video_extraction_framerate: 1
15
+ encoder_input_dimension: 2048
16
+ encoder_dim_feedforward: 16384
17
+ encoder_heads: 32
18
+ encoder_layers: 12
19
+ musicgen_model_id: facebook/musicgen-stereo-medium
20
+ use_peft: false
21
+ output_dir_name: audiocraft_nature_medium
22
+ dataset_video_folder: ../../datasets/nature2/videos_30sec
23
+ dataset_audio_folder: ../../datasets/nature2/audio_30sec
24
+ use_demucs_folder_structure: false
25
+ dataset_validation_split: 0.1
26
+ dataset_shuffling_seed: 42
models_frozen_decoder/nature_small/dataset_split.json DELETED
The diff for this file is too large to render. See raw diff
 
models_frozen_decoder/symmv_large/configuration.yml ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ log_level: INFO
2
+ num_epochs: 20
3
+ batch_size: 2
4
+ save_step: 100
5
+ device: cuda
6
+ use_wandb: true
7
+ wandb_project_name: thesis_audiocraft
8
+ gradient_clipping: 1.0
9
+ beta1: 0.9
10
+ beta2: 0.95
11
+ weight_decay: 0.1
12
+ learning_rate: 1.0e-05
13
+ warmup_steps: 16
14
+ video_extraction_framerate: 1
15
+ encoder_input_dimension: 2048
16
+ encoder_dim_feedforward: 32768
17
+ encoder_heads: 64
18
+ encoder_layers: 24
19
+ musicgen_use_cfg: false
20
+ musicgen_model_id: facebook/musicgen-stereo-large
21
+ use_peft: false
22
+ dataset_video_folder: ../../datasets/symmv/videos_30sec
23
+ dataset_audio_folder: ../../datasets/symmv/audio_30sec_separated
24
+ use_demucs_folder_structure: true
25
+ dataset_validation_split: 0.1
26
+ dataset_shuffling_seed: 42
models_frozen_decoder/symmv_medium/configuration.yml ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ log_level: INFO
2
+ num_epochs: 20
3
+ batch_size: 8
4
+ save_step: 100
5
+ device: cuda
6
+ use_wandb: true
7
+ wandb_project_name: thesis_audiocraft
8
+ gradient_clipping: 1.0
9
+ beta1: 0.9
10
+ beta2: 0.95
11
+ weight_decay: 0.1
12
+ learning_rate: 1.0e-05
13
+ warmup_steps: 16
14
+ video_extraction_framerate: 1
15
+ encoder_input_dimension: 2048
16
+ encoder_dim_feedforward: 16384
17
+ encoder_heads: 32
18
+ encoder_layers: 12
19
+ musicgen_use_cfg: false
20
+ musicgen_model_id: facebook/musicgen-stereo-medium
21
+ use_peft: false
22
+ dataset_video_folder: ../../datasets/symmv/videos_30sec
23
+ dataset_audio_folder: ../../datasets/symmv/audio_30sec_separated
24
+ use_demucs_folder_structure: true
25
+ dataset_validation_split: 0.1
26
+ dataset_shuffling_seed: 42
models_peft/dataset_split.json ADDED
The diff for this file is too large to render. See raw diff
 
models_peft/nature_large/configuration.yml ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ log_level: INFO
2
+ num_epochs: 20
3
+ batch_size: 2
4
+ save_step: 1720
5
+ device: cuda
6
+ use_wandb: true
7
+ wandb_project_name: thesis_peft
8
+ gradient_clipping: 1.0
9
+ beta1: 0.9
10
+ beta2: 0.95
11
+ weight_decay: 0.1
12
+ learning_rate: 1.0e-05
13
+ warmup_steps: 16
14
+ video_extraction_framerate: 1
15
+ encoder_input_dimension: 2048
16
+ encoder_dim_feedforward: 32768
17
+ encoder_heads: 64
18
+ encoder_layers: 24
19
+ musicgen_model_id: facebook/musicgen-stereo-large
20
+ use_peft: true
21
+ output_dir_name: nature_large
22
+ dataset_video_folder: ../../datasets/nature2/videos_30sec
23
+ dataset_audio_folder: ../../datasets/nature2/audio_30sec
24
+ use_demucs_folder_structure: false
25
+ dataset_validation_split: 0.1
26
+ dataset_shuffling_seed: 42
models_peft/nature_medium/configuration.yml ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ log_level: INFO
2
+ num_epochs: 20
3
+ batch_size: 8
4
+ save_step: 450
5
+ device: cuda
6
+ use_wandb: true
7
+ wandb_project_name: thesis_peft
8
+ gradient_clipping: 1.0
9
+ beta1: 0.9
10
+ beta2: 0.95
11
+ weight_decay: 0.1
12
+ learning_rate: 1.0e-05
13
+ warmup_steps: 16
14
+ video_extraction_framerate: 1
15
+ encoder_input_dimension: 2048
16
+ encoder_dim_feedforward: 16384
17
+ encoder_heads: 32
18
+ encoder_layers: 12
19
+ musicgen_model_id: facebook/musicgen-stereo-medium
20
+ use_peft: true
21
+ output_dir_name: nature_medium
22
+ dataset_video_folder: ../../datasets/nature2/videos_30sec
23
+ dataset_audio_folder: ../../datasets/nature2/audio_30sec
24
+ use_demucs_folder_structure: false
25
+ dataset_validation_split: 0.1
26
+ dataset_shuffling_seed: 42
models_peft/nature_small/configuration.yml ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ log_level: INFO
2
+ num_epochs: 20
3
+ batch_size: 16
4
+ save_step: 200
5
+ device: cuda
6
+ use_wandb: true
7
+ wandb_project_name: thesis_peft
8
+ gradient_clipping: 1.0
9
+ beta1: 0.9
10
+ beta2: 0.95
11
+ weight_decay: 0.1
12
+ learning_rate: 1.0e-05
13
+ warmup_steps: 16
14
+ video_extraction_framerate: 1
15
+ encoder_input_dimension: 2048
16
+ encoder_dim_feedforward: 8192
17
+ encoder_heads: 16
18
+ encoder_layers: 6
19
+ musicgen_model_id: facebook/musicgen-stereo-small
20
+ use_peft: true
21
+ output_dir_name: nature_small
22
+ dataset_video_folder: ../../datasets/nature2/videos_30sec
23
+ dataset_audio_folder: ../../datasets/nature2/audio_30sec
24
+ use_demucs_folder_structure: false
25
+ dataset_validation_split: 0.1
26
+ dataset_shuffling_seed: 42
models_peft/symmv_large/configuration.yml ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ log_level: INFO
2
+ num_epochs: 20
3
+ batch_size: 2
4
+ save_step: 1720
5
+ device: cuda
6
+ use_wandb: true
7
+ wandb_project_name: thesis_peft
8
+ gradient_clipping: 1.0
9
+ beta1: 0.9
10
+ beta2: 0.95
11
+ weight_decay: 0.1
12
+ learning_rate: 1.0e-05
13
+ warmup_steps: 16
14
+ video_extraction_framerate: 1
15
+ encoder_input_dimension: 2048
16
+ encoder_dim_feedforward: 32768
17
+ encoder_heads: 64
18
+ encoder_layers: 24
19
+ musicgen_model_id: facebook/musicgen-stereo-large
20
+ use_peft: true
21
+ output_dir_name: symmv_large
22
+ dataset_video_folder: ../../datasets/symmv/videos_30sec
23
+ dataset_audio_folder: ../../datasets/symmv/audio_30sec_separated
24
+ use_demucs_folder_structure: true
25
+ dataset_validation_split: 0.1
26
+ dataset_shuffling_seed: 42
models_peft/symmv_medium/configuration.yml ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ log_level: INFO
2
+ num_epochs: 20
3
+ batch_size: 8
4
+ save_step: 400
5
+ device: cuda
6
+ use_wandb: true
7
+ wandb_project_name: thesis_peft
8
+ gradient_clipping: 1.0
9
+ beta1: 0.9
10
+ beta2: 0.95
11
+ weight_decay: 0.1
12
+ learning_rate: 1.0e-05
13
+ warmup_steps: 16
14
+ video_extraction_framerate: 1
15
+ encoder_input_dimension: 2048
16
+ encoder_dim_feedforward: 16384
17
+ encoder_heads: 32
18
+ encoder_layers: 12
19
+ musicgen_model_id: facebook/musicgen-stereo-medium
20
+ use_peft: true
21
+ output_dir_name: symmv_medium
22
+ dataset_video_folder: ../../datasets/symmv/videos_30sec
23
+ dataset_audio_folder: ../../datasets/symmv/audio_30sec_separated
24
+ use_demucs_folder_structure: true
25
+ dataset_validation_split: 0.1
26
+ dataset_shuffling_seed: 42
models_peft/symmv_small/configuration.yml ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ log_level: INFO
2
+ num_epochs: 20
3
+ batch_size: 16
4
+ save_step: 200
5
+ device: cuda
6
+ use_wandb: true
7
+ wandb_project_name: thesis_peft
8
+ gradient_clipping: 1.0
9
+ beta1: 0.9
10
+ beta2: 0.95
11
+ weight_decay: 0.1
12
+ learning_rate: 1.0e-05
13
+ warmup_steps: 16
14
+ video_extraction_framerate: 1
15
+ encoder_input_dimension: 2048
16
+ encoder_dim_feedforward: 8192
17
+ encoder_heads: 16
18
+ encoder_layers: 6
19
+ musicgen_model_id: facebook/musicgen-stereo-small
20
+ use_peft: true
21
+ output_dir_name: symmv_small
22
+ dataset_video_folder: ../../datasets/symmv/videos_30sec
23
+ dataset_audio_folder: ../../datasets/symmv/audio_30sec_separated
24
+ use_demucs_folder_structure: true
25
+ dataset_validation_split: 0.1
26
+ dataset_shuffling_seed: 42