fix ds ckpt

checkpoints/ds_exp/config.yaml

@@ -0,0 +1,393 @@
+K_step: 71
+accumulate_grad_batches: 1
+amp: false
+audio_num_mel_bins: 80
+audio_sample_rate: 22050
+base_config:
+- egs/egs_bases/tts/ds.yaml
+- ./fs2_orig.yaml
+binarization_args:
+  min_sil_duration: 0.1
+  shuffle: false
+  test_range:
+  - 0
+  - 523
+  train_range:
+  - 871
+  - -1
+  trim_eos_bos: false
+  valid_range:
+  - 523
+  - 871
+  with_align: true
+  with_f0: true
+  with_f0cwt: true
+  with_linear: false
+  with_spk_embed: false
+  with_wav: false
+binarizer_cls: data_gen.tts.base_binarizer.BaseBinarizer
+binary_data_dir: data/binary/ljspeech_cwt
+check_val_every_n_epoch: 10
+clip_grad_norm: 1
+clip_grad_value: 0
+conv_use_pos: false
+cwt_std_scale: 0.8
+debug: false
+dec_dilations:
+- 1
+- 1
+- 1
+- 1
+dec_ffn_kernel_size: 9
+dec_inp_add_noise: false
+dec_kernel_size: 5
+dec_layers: 4
+dec_post_net_kernel: 3
+decay_steps: 50000
+decoder_rnn_dim: 0
+decoder_type: fft
+diff_decoder_type: wavenet
+diff_loss_type: l1
+dilation_cycle_length: 1
+dropout: 0.1
+ds_workers: 2
+dur_predictor_kernel: 3
+dur_predictor_layers: 2
+enc_dec_norm: ln
+enc_dilations:
+- 1
+- 1
+- 1
+- 1
+enc_ffn_kernel_size: 9
+enc_kernel_size: 5
+enc_layers: 4
+enc_post_net_kernel: 3
+enc_pre_ln: true
+enc_prenet: true
+encoder_K: 8
+encoder_type: fft
+endless_ds: true
+eval_max_batches: -1
+f0_max: 600
+f0_min: 80
+ffn_act: gelu
+ffn_hidden_size: 1024
+fft_size: 1024
+fmax: 7600
+fmin: 80
+frames_multiple: 1
+fs2_ckpt: checkpoints/fs2orig_exp/model_ckpt_steps_100000.ckpt
+gen_dir_name: ''
+griffin_lim_iters: 30
+hidden_size: 256
+hop_size: 256
+infer: false
+keep_bins: 80
+lambda_commit: 0.25
+lambda_energy: 0.1
+lambda_f0: 1.0
+lambda_ph_dur: 0.1
+lambda_sent_dur: 1.0
+lambda_uv: 1.0
+lambda_word_dur: 1.0
+layers_in_block: 2
+load_ckpt: ''
+loud_norm: false
+lr: 0.001
+max_beta: 0.06
+max_epochs: 1000
+max_frames: 1548
+max_input_tokens: 1550
+max_sentences: 128
+max_tokens: 30000
+max_updates: 160000
+max_valid_sentences: 1
+max_valid_tokens: 60000
+mel_losses: l1
+mel_vmax: 1.5
+mel_vmin: -6
+min_frames: 0
+num_ckpt_keep: 3
+num_heads: 2
+num_sanity_val_steps: 5
+num_spk: 1
+num_valid_plots: 10
+optimizer_adam_beta1: 0.9
+optimizer_adam_beta2: 0.98
+out_wav_norm: false
+pitch_extractor: parselmouth
+pitch_key: pitch
+pitch_type: cwt
+predictor_dropout: 0.5
+predictor_grad: 0.1
+predictor_hidden: -1
+predictor_kernel: 5
+predictor_layers: 5
+preprocess_args:
+  add_eos_bos: true
+  mfa_group_shuffle: false
+  mfa_offset: 0.02
+  nsample_per_mfa_group: 1000
+  reset_phone_dict: true
+  reset_word_dict: true
+  save_sil_mask: true
+  txt_processor: en
+  use_mfa: true
+  vad_max_silence_length: 12
+  wav_processors: []
+  with_phsep: true
+preprocess_cls: egs.datasets.audio.lj.preprocess.LJPreprocess
+print_nan_grads: false
+processed_data_dir: data/processed/ljspeech
+profile_infer: false
+raw_data_dir: data/raw/LJSpeech-1.1
+ref_norm_layer: bn
+rename_tmux: true
+residual_channels: 256
+residual_layers: 20
+resume_from_checkpoint: 0
+save_best: false
+save_codes:
+- tasks
+- modules
+- egs
+save_f0: false
+save_gt: true
+schedule_type: linear
+scheduler: warmup
+seed: 1234
+sort_by_len: true
+spec_max:
+- -0.5982
+- -0.0778
+- 0.1205
+- 0.2747
+- 0.4657
+- 0.5123
+- 0.583
+- 0.7093
+- 0.6461
+- 0.6101
+- 0.7316
+- 0.7715
+- 0.7681
+- 0.8349
+- 0.7815
+- 0.7591
+- 0.791
+- 0.7433
+- 0.7352
+- 0.6869
+- 0.6854
+- 0.6623
+- 0.5353
+- 0.6492
+- 0.6909
+- 0.6106
+- 0.5761
+- 0.5236
+- 0.5638
+- 0.4054
+- 0.4545
+- 0.3407
+- 0.3037
+- 0.338
+- 0.1599
+- 0.1603
+- 0.2741
+- 0.213
+- 0.1569
+- 0.1911
+- 0.2324
+- 0.1586
+- 0.1221
+- 0.0341
+- -0.0558
+- 0.0553
+- -0.1153
+- -0.0933
+- -0.1171
+- -0.005
+- -0.1519
+- -0.1629
+- -0.0522
+- -0.0739
+- -0.2069
+- -0.2405
+- -0.1244
+- -0.2582
+- -0.1361
+- -0.1575
+- -0.1442
+- 0.0513
+- -0.1567
+- -0.2
+- 0.0086
+- -0.0698
+- 0.1385
+- 0.0941
+- 0.1864
+- 0.1225
+- 0.1389
+- 0.1382
+- 0.167
+- 0.1007
+- 0.1444
+- 0.0888
+- 0.1998
+- 0.228
+- 0.2932
+- 0.3047
+spec_min:
+- -4.7574
+- -4.6783
+- -4.6431
+- -4.5832
+- -4.539
+- -4.6771
+- -4.8089
+- -4.7672
+- -4.5784
+- -4.7755
+- -4.715
+- -4.8919
+- -4.8271
+- -4.7389
+- -4.6047
+- -4.7759
+- -4.6799
+- -4.8201
+- -4.7823
+- -4.8262
+- -4.7857
+- -4.7545
+- -4.9358
+- -4.9733
+- -5.1134
+- -5.1395
+- -4.9016
+- -4.8434
+- -5.0189
+- -4.846
+- -5.0529
+- -4.951
+- -5.0217
+- -5.0049
+- -5.1831
+- -5.1445
+- -5.1015
+- -5.0281
+- -4.9887
+- -4.9916
+- -4.9785
+- -4.9071
+- -4.9488
+- -5.0342
+- -4.9332
+- -5.065
+- -4.8924
+- -5.0875
+- -5.0483
+- -5.0848
+- -5.0655
+- -5.0279
+- -5.0015
+- -5.0792
+- -5.0636
+- -5.2413
+- -5.1421
+- -5.171
+- -5.3256
+- -5.0511
+- -5.1186
+- -5.0057
+- -5.0446
+- -5.1173
+- -5.0325
+- -5.1085
+- -5.0053
+- -5.0755
+- -5.1176
+- -5.1004
+- -5.2153
+- -5.2757
+- -5.3025
+- -5.2867
+- -5.2918
+- -5.3328
+- -5.2731
+- -5.2985
+- -5.24
+- -5.2211
+task_cls: tasks.tts.diffspeech.DiffSpeechTask
+tb_log_interval: 100
+test_ids:
+- 0
+- 1
+- 2
+- 3
+- 4
+- 5
+- 6
+- 7
+- 8
+- 9
+- 10
+- 11
+- 12
+- 13
+- 14
+- 15
+- 16
+- 17
+- 18
+- 19
+- 68
+- 70
+- 74
+- 87
+- 110
+- 172
+- 190
+- 215
+- 231
+- 294
+- 316
+- 324
+- 402
+- 422
+- 485
+- 500
+- 505
+- 508
+- 509
+- 519
+test_input_yaml: ''
+test_num: 100
+test_set_name: test
+timesteps: 100
+train_set_name: train
+train_sets: ''
+use_energy_embed: false
+use_gt_dur: false
+use_gt_energy: false
+use_gt_f0: false
+use_pitch_embed: true
+use_pos_embed: true
+use_spk_embed: false
+use_spk_id: false
+use_uv: true
+use_word_input: false
+val_check_interval: 2000
+valid_infer_interval: 10000
+valid_monitor_key: val_loss
+valid_monitor_mode: min
+valid_set_name: valid
+vocoder: HifiGAN
+vocoder_ckpt: checkpoints/hifi_lj
+warmup_updates: 4000
+weight_decay: 0
+win_size: 1024
+word_dict_size: 10000
+work_dir: checkpoints/0214_ds

checkpoints/ds_exp/model_ckpt_steps_160000.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b41f14a72d74c706ffcda7ae319c3ad873cdc4313a8be47ed117b0c67e7c9aac
+size 161932032

egs/datasets/audio/lj/ds.yaml

@@ -2,7 +2,7 @@ base_config:
   - egs/egs_bases/tts/ds.yaml
   - ./fs2_orig.yaml
 
-fs2_ckpt: checkpoints/fs2_exp/model_ckpt_steps_160000.ckpt
+fs2_ckpt: checkpoints/aux_exp/model_ckpt_steps_100000.ckpt
 
 # spec_min and spec_max are calculated on the training set.
 spec_min: [ -4.7574, -4.6783, -4.6431, -4.5832, -4.5390, -4.6771, -4.8089, -4.7672,

egs/egs_bases/tts/ds.yaml

@@ -28,5 +28,6 @@ keep_bins: 80
 num_valid_plots: 10
 use_gt_dur: false
 use_gt_f0: false
+use_energy_embed: false
 #pitch_type: cwt
 max_updates: 160000

modules/tts/diffspeech/shallow_diffusion_tts.py

@@ -236,8 +236,6 @@ class GaussianDiffusion(nn.Module):
         b, *_, device = *txt_tokens.shape, txt_tokens.device
         ret = self.fs2(txt_tokens, mel2ph=mel2ph, spk_embed=spk_embed, spk_id=spk_id,
                                 f0=f0, uv=uv, energy=energy, infer=infer, skip_decoder=(not infer), **kwargs)
-            # (txt_tokens, mel2ph, spk_embed, ref_mels, f0, uv, energy,
-            #            skip_decoder=(not infer), infer=infer, **kwargs)
         cond = ret['decoder_inp'].transpose(1, 2)
 
         if not infer:

tasks/tts/diffspeech.py

@@ -26,12 +26,12 @@ class DiffSpeechTask(FastSpeech2OrigTask):
         self.model = GaussianDiffusion(dict_size, hparams)
         if hparams['fs2_ckpt'] != '':
             load_ckpt(self.model.fs2, hparams['fs2_ckpt'], 'model', strict=True)
-        for k, v in self.model.fs2.named_parameters():
-            if 'predictor' not in k:
-                v.requires_grad = False
-        # or
         # for k, v in self.model.fs2.named_parameters():
-        #     v.requires_grad = False
+        #     if 'predictor' not in k:
+        #         v.requires_grad = False
+        # or
+        for k, v in self.model.fs2.named_parameters():
+            v.requires_grad = False
 
     def build_optimizer(self, model):
         self.optimizer = optimizer = torch.optim.AdamW(