Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+contentstyle_modeling/posttrained/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+contentstyle_modeling/pretrained/tokenizer.json filter=lfs diff=lfs merge=lfs -text

acoustic_modeling/fm_emilia101k_singnet7k_repa/config.json

@@ -0,0 +1,62 @@
+{
+    "model_type": "FlowMatchingTransformer",
+    "preprocess": {
+        "hop_size": 480,
+        "sample_rate": 24000,
+        "n_fft": 1920,
+        "num_mels": 128,
+        "win_size": 1920,
+        "fmin": 0,
+        "fmax": 12000,
+        "mel_var": 8.14,
+        "mel_mean": -4.92,
+        "f0_fmin": 50.0,
+        "f0_fmax": 1100.0,
+        "load_phone": false,
+        "load_chromagram": true,
+        "load_semantic_features": true,
+    },
+    "model": {
+        "flow_matching_transformer": {
+            "mel_dim": 128,
+            "hidden_size": 1024,
+            "num_layers": 16,
+            "num_heads": 16,
+            "cfg_scale": 0.2,
+            "use_cond_code": true, // false means Hidden features
+            // "cond_dim": 1024, // HuBERT features dimension
+            "cond_codebook_size": 16384, // VQ Codebook Size
+            "cond_scale_factor": 4, // 1 means not use ReTrans. 4 means 12.5Hz * 4 = 50Hz. This should be aligned with the frame rate with Mels
+            "sigma": 1e-5,
+            "time_scheduler": "cos",
+            "whisper_perturb": false,
+            "repa": {
+                "layer_index": 5, // Use the Wav2Vec2Bert features to align. 5 means the 6th layer.
+                "output_dim": 1024, // The dimension of the Wav2Vec2Bert features.
+                "loss_type": "cos", // "cos" or "l1". By default is "l1"
+            },
+        },
+        "cond_sample_rate": 16000, // whisper: 16000
+        "coco": {
+            "coco_type": "content_style", // content, style, or content_style
+            "downsample_rate": 4, // The original frame rate is 50 Hz, downsample to 12.5 Hz
+            "codebook_size": 16384,
+            "hidden_size": 1024, // Representations Dim
+            "codebook_dim": 8,
+            "encoder": {
+                "vocos_dim": 384,
+                "vocos_intermediate_dim": 2048,
+                "vocos_num_layers": 12,
+            },
+            "decoder": {
+                "vocos_dim": 384,
+                "vocos_intermediate_dim": 2048,
+                "vocos_num_layers": 12,
+            },
+            "use_normed_whisper": true,
+            "whisper_stats_path": "models/svc/vevosing/config/whisper_stats.pt",
+            "whisper_dim": 1024,
+            "chromagram_dim": 24,
+        },
+    },
+}

acoustic_modeling/fm_emilia101k_singnet7k_repa/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef3733b3f92cf8f38e32f6d161f58751247b199cffc15cc1562e76c1289a7186
+size 1451496488

acoustic_modeling/fm_emilia101k_singnet7k_repa/whisper_stats.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6117052b3e23e9075a79cc208ecc24328ae2f71a7dd4c9793db7c90a88b4a519
+size 9215

acoustic_modeling/fm_emilia101k_singnet7k_repa_text/config.json

@@ -0,0 +1,63 @@
+{
+    "model_type": "FlowMatchingTransformer",
+    "preprocess": {
+        "hop_size": 480,
+        "sample_rate": 24000,
+        "n_fft": 1920,
+        "num_mels": 128,
+        "win_size": 1920,
+        "fmin": 0,
+        "fmax": 12000,
+        "mel_var": 8.14,
+        "mel_mean": -4.92,
+        "f0_fmin": 50.0,
+        "f0_fmax": 1100.0,
+        "load_phone": false,
+        "load_singnet_phone": false,
+        "load_chromagram": true,
+        "load_semantic_features": true,
+        "load_text": true,
+        "pad_token_id": 32000,
+        "tokenizer_path": "microsoft/Phi-3.5-mini-instruct"
+    },
+    "model": {
+        "flow_matching_transformer": {
+            "mel_dim": 128,
+            "hidden_size": 1024,
+            "num_layers": 16,
+            "num_heads": 16,
+            "cfg_scale": 0.2,
+            "use_cond_code": true, // false means Hidden features
+            // "cond_dim": 1024, // HuBERT features dimension
+            "cond_codebook_size": 16384, // VQ Codebook Size
+            "cond_scale_factor": 4, // 1 means not use ReTrans. 4 means 12.5Hz * 4 = 50Hz. This should be aligned with the frame rate with Mels
+            "sigma": 1e-5,
+            "time_scheduler": "cos",
+            "whisper_perturb": false,
+            "repa": {
+                "layer_index": 5, // Use the Wav2Vec2Bert features to align. 5 means the 6th layer.
+                "output_dim": 1024, // The dimension of the Wav2Vec2Bert features.
+                "loss_type": "cos", // "cos" or "l1". By default is "l1"
+            },
+            "use_text_as_condition": true,
+            "text_cond_codebook_size": 32100
+        },
+        "cond_sample_rate": 16000, // whisper: 16000
+        "coco": {
+            "coco_type": "content_style", // content, style, or content_style
+            "downsample_rate": 4, // The original frame rate is 50 Hz, downsample to 12.5 Hz
+            "codebook_size": 16384,
+            "hidden_size": 1024, // Representations Dim
+            "codebook_dim": 8,
+            "encoder": {
+                "vocos_dim": 384,
+                "vocos_intermediate_dim": 2048,
+                "vocos_num_layers": 12,
+            },
+            "use_normed_whisper": true,
+            "whisper_stats_path": "whisper_stats.pt",
+            "whisper_dim": 1024,
+            "chromagram_dim": 24,
+        },
+    },
+}

acoustic_modeling/fm_emilia101k_singnet7k_repa_text/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c972f24183e1347d6b767c9d2804959de0506bfcb4836992aba9cd33ce4e9ed1
+size 1616553544

acoustic_modeling/fm_emilia101k_singnet7k_repa_text/whisper_stats.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6117052b3e23e9075a79cc208ecc24328ae2f71a7dd4c9793db7c90a88b4a519
+size 9215

contentstyle_modeling/posttrained/amphion_config.json

@@ -0,0 +1,73 @@
+{
+    "preprocess": {
+        "hop_size": 480,
+        "sample_rate": 24000,
+        "n_fft": 1920,
+        "num_mels": 128,
+        "win_size": 1920,
+        "fmin": 0,
+        "fmax": 12000,
+        "mel_var": 8.14,
+        "mel_mean": -4.92,
+        "f0_fmin": 50.0,
+        "f0_fmax": 1100.0,
+        "wav_code_frame_rate": 18.75, // Vevo2: 12.5 (Content-Style Code) + 6.25 (Prosody Code) = 18.75
+        "min_dur": 1,
+        "max_dur": 30,
+        "drop_prosody_id_prob": -1, // Dropping prosody ids means the Text-to-CS, while not dropping means the Text+Note-to-CS,
+        "pad_token_id": 151643, // <|endoftext|> for Qwen2.5-0.5B-Instruct,
+        "eos_token": "<|im_end|>",
+        "eos_token_id": 151645, // <|im_end|> for Qwen2.5-0.5B-Instruct,
+        // "tokenizer_path": "/mnt/data4/zhangxueyao/SpeechGenerationYC_ckpts/ckpts/vevo2/pretrained/Qwen2.5-0.5B-Instruct-add_prosody_contentstyle"
+    },
+    "model": {
+        // "pretrained_model_path": "/mnt/data4/zhangxueyao/SpeechGenerationYC_ckpts/ckpts/vevo2/pretrained/Qwen2.5-0.5B-Instruct-add_prosody_contentstyle", // Qwen2.5 Model
+        // "rl_init_model_path": "/mnt/data4/zhangxueyao/SpeechGenerationYC_ckpts/ckpts/vevo2/llm_dpo/dpo_qwen0.5B_intp2_highsim_3e-5/checkpoint_backup/epoch-0023_step-0027000_loss-0.000961", // DPO Model
+        "use_intelligibility_reward": true,
+        "use_chromagram_reward": true,
+        "use_target_length_reward": true,
+        "reward_combination_strategy": "advantage_first", // "reward_first" or "advantage_first"
+        "coco_style": {
+            "coco_type": "style", // content, style, or content_style
+            "downsample_rate": 8, // The original frame rate is 50 Hz, downsample to 6.25 Hz
+            "codebook_size": 512,
+            "hidden_size": 1024, // Representations Dim
+            "codebook_dim": 8,
+            "encoder": {
+                "vocos_dim": 384,
+                "vocos_intermediate_dim": 2048,
+                "vocos_num_layers": 12,
+            },
+            "decoder": {
+                "vocos_dim": 384,
+                "vocos_intermediate_dim": 2048,
+                "vocos_num_layers": 12,
+            },
+            "use_normed_whisper": true,
+            "whisper_stats_path": "models/svc/vevosing/config/whisper_stats.pt",
+            "whisper_dim": 1024,
+            "chromagram_dim": 24,
+        },
+        "coco_content_style": {
+            "coco_type": "content_style", // content, style, or content_style
+            "downsample_rate": 4, // The original frame rate is 50 Hz, downsample to 12.5 Hz
+            "codebook_size": 16384,
+            "hidden_size": 1024, // Representations Dim
+            "codebook_dim": 8,
+            "encoder": {
+                "vocos_dim": 384,
+                "vocos_intermediate_dim": 2048,
+                "vocos_num_layers": 12,
+            },
+            "decoder": {
+                "vocos_dim": 384,
+                "vocos_intermediate_dim": 2048,
+                "vocos_num_layers": 12,
+            },
+            "use_normed_whisper": true,
+            "whisper_stats_path": "models/svc/vevosing/config/whisper_stats.pt",
+            "whisper_dim": 1024,
+            "chromagram_dim": 24,
+        },
+    },
+}

contentstyle_modeling/posttrained/config.json

@@ -0,0 +1,29 @@
+{
+  "_name_or_path": "epoch-0023_step-0027000_loss-0.000961",
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151645,
+  "hidden_act": "silu",
+  "hidden_size": 896,
+  "initializer_range": 0.02,
+  "intermediate_size": 4864,
+  "max_position_embeddings": 32768,
+  "max_window_layers": 21,
+  "model_type": "qwen2",
+  "num_attention_heads": 14,
+  "num_hidden_layers": 24,
+  "num_key_value_heads": 2,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": true,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.47.1",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 168565
+}

contentstyle_modeling/posttrained/generation_config.json

@@ -0,0 +1,14 @@
+{
+  "bos_token_id": 151643,
+  "do_sample": true,
+  "eos_token_id": [
+    151645,
+    151643
+  ],
+  "pad_token_id": 151643,
+  "repetition_penalty": 1.1,
+  "temperature": 0.7,
+  "top_k": 20,
+  "top_p": 0.8,
+  "transformers_version": "4.47.1"
+}

contentstyle_modeling/posttrained/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b056f8b4f0616dd5eb8598a0cc3fe5a4bd9dcdffc2dbe344156a16691496cf16
+size 1017897016

contentstyle_modeling/posttrained/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:00d6a8b09b3c9b33a89928078d4419801da83503ee168ad218b54bcb4b2de80c
+size 2035976634

contentstyle_modeling/posttrained/rng_state_0.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a0d21fa6169554d53edc945a579970dc3448968f10ab1416295d1014dbbb9f49
+size 15728

contentstyle_modeling/posttrained/rng_state_1.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:50cace7ce599554e1b01251eb0e29892e1a9894d33e4aa3ac9b2cc0a33b31d45
+size 15792

contentstyle_modeling/posttrained/rng_state_2.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f7dbd212f619e6e1aa021322d3a3eb2379e92d7530cc581bf9d5da6d99dac64
+size 15728

contentstyle_modeling/posttrained/rng_state_3.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0d595703e986b376bcbbbe983db3f7f9d54467320ea6892831e36d4ddcb28c7d
+size 15792

contentstyle_modeling/posttrained/rng_state_4.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:54858b6df6db301adc27c00cf899bf8783437a01f590241a20b7b3b0c576436f
+size 15728

contentstyle_modeling/posttrained/rng_state_5.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8209d49950616ee1e3115d8ba85e44f92746f2cf274d7e65daa66a5ddbffb516
+size 15792

contentstyle_modeling/posttrained/rng_state_6.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4c9c457ef48482488944e26e066bff5b43a88ecd73457a8cf068e2568980401d
+size 15664

contentstyle_modeling/posttrained/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:42f755b43c342264783f3a2fa30b921b7bf7353b3e3649a4cf4c8ac31ede21ad
+size 1064

contentstyle_modeling/posttrained/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b4afa3f042a3dd5ddf573c035c68f88810d01d714fe3341054e9688c0eebef7
+size 13891825

contentstyle_modeling/posttrained/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0d8b222db8e20e9927c6193a02f9e01d452e03a5f706ae69e0bc6b500a0711f7
+size 6264

contentstyle_modeling/pretrained/config.json

@@ -0,0 +1,28 @@
+{
+  "_name_or_path": "/storage/zhangxueyao/workspace/SpeechGenerationYC/pretrained/Qwen2.5-0.5B-Instruct",
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151645,
+  "hidden_act": "silu",
+  "hidden_size": 896,
+  "initializer_range": 0.02,
+  "intermediate_size": 4864,
+  "max_position_embeddings": 32768,
+  "max_window_layers": 21,
+  "model_type": "qwen2",
+  "num_attention_heads": 14,
+  "num_hidden_layers": 24,
+  "num_key_value_heads": 2,
+  "rms_norm_eps": 1e-06,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": true,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.44.1",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 168565
+}

contentstyle_modeling/pretrained/generation_config.json

@@ -0,0 +1,14 @@
+{
+  "bos_token_id": 151643,
+  "do_sample": true,
+  "eos_token_id": [
+    151645,
+    151643
+  ],
+  "pad_token_id": 151643,
+  "repetition_penalty": 1.1,
+  "temperature": 0.7,
+  "top_k": 20,
+  "top_p": 0.8,
+  "transformers_version": "4.44.1"
+}

contentstyle_modeling/pretrained/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:531be07b9eb7337988cda77842ced780f0cea3d6d30395ca19224f338698551f
+size 1017897016

contentstyle_modeling/pretrained/optimizer.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5773426880bb14310f7f2091ecdb25c40ca5fb0ff13a1673462d8f9c4d4bfd77
+size 2035976442

contentstyle_modeling/pretrained/random_states_0.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd94f420a504ac49f408ed27b8ed54e94ccb1843fdae48a98404069db675e292
+size 16036

contentstyle_modeling/pretrained/scheduler.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e945d4b96759596852af2d7406b1573fb5c0ef023de89ac9a7a1a7ab37cdbdf
+size 1000

contentstyle_modeling/pretrained/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fc6a67d19953836a0b0f3ab9385f2a29194fe98feadf07ef8628a9eb6d58509e
+size 12544343

tokenizer/contentstyle_fvq16384_12.5hz/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bebdcea39e2d0134dbcce193aed0e7b0d393a866346317cd196389e40e9151b0
+size 244781696

tokenizer/prosody_fvq512_6.25hz/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:754ba5928dfe4dd3c4b6e91ce396e2a50cff52937053c0e3e4ac3a854a4b8ae4
+size 273642504

vocoder/config.json

@@ -0,0 +1,84 @@
+{
+    "model_type": "Vocoder",
+    "preprocess": {
+        "hop_size": 480,
+        "sample_rate": 24000,
+        "max_length": 36000,
+        "n_fft": 1920,
+        "num_mels": 128,
+        "win_size": 1920,
+        "fmin": 0,
+        "fmax": 12000,
+        "mel_var": 8.14,
+        "mel_mean": -4.92,
+        "load_phone": false,
+        "load_chromagram": false
+    },
+    "model": {
+        "vocos": {
+            "input_channels": 128,
+            "dim": 1024,
+            "intermediate_dim": 4096,
+            "num_layers": 30,
+            "n_fft": 1920,
+            "hop_size": 480,
+            "padding": "same"
+        },
+        "period_gan": {
+            "max_downsample_channels": 1024,
+            "channels": 64,
+            "channel_increasing_factor": 2
+        },
+        "spec_gan": {
+            "stft_params": {
+                "fft_sizes": [
+                    128,
+                    256,
+                    512,
+                    1024,
+                    2048
+                ],
+                "hop_sizes": [
+                    32,
+                    64,
+                    128,
+                    256,
+                    512
+                ],
+                "win_lengths": [
+                    128,
+                    256,
+                    512,
+                    1024,
+                    2048
+                ],
+                "window": "hann_window"
+            },
+            "in_channels": 1,
+            "out_channels": 1,
+            "channels": 64,
+            "kernel_sizes": [
+                5,
+                3
+            ],
+            "max_downsample_channels": 1024,
+            "down_scales": [
+                2,
+                2,
+                2
+            ],
+            "use_weight_norm": true,
+            "use_complex": false
+        }
+    },
+    "loss": {
+        "mel_loss": {
+            "sample_rate": 24000
+        },
+        "disc_loss_weight": 1.0,
+        "mel_loss_weight": 10.0,
+        "adv_loss_weight": 2.0,
+        "fm_loss_weight": 2.0,
+        "spec_fm_loss_weight": 1.0
+    },
+}

vocoder/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5b5d1a46b19351c9a71bd8a5a59dd16be0be2ddefe70d3a0b4915d9a425e56d3
+size 1020206416

vocoder/model_1.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:850799d78699134b969056183fc9d490c51f8d8154d0ed00fae3e738b6b30af6
+size 69768280

vocoder/model_2.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:56130fd13d5fbe828e56d61edb0049d35700db0472a866b8167d1d217d2687f8
+size 180693296