Louise-Belcher
/
Finetune Model for WAV 27db Sample Rate 22050
/finetune_archived_230424-020648
/train_louise_230417-140742.log
| 23-04-17 14:07:42.354 - INFO: name: louise | |
| model: extensibletrainer | |
| scale: 1 | |
| gpu_ids: [0] | |
| start_step: 0 | |
| checkpointing_enabled: True | |
| fp16: True | |
| bitsandbytes: True | |
| gpus: 1 | |
| datasets:[ | |
| train:[ | |
| name: training | |
| n_workers: 2 | |
| batch_size: 28 | |
| mode: paired_voice_audio | |
| path: ./training/louise/train.txt | |
| fetcher_mode: ['lj'] | |
| phase: train | |
| max_wav_length: 255995 | |
| max_text_length: 200 | |
| sample_rate: 22050 | |
| load_conditioning: True | |
| num_conditioning_candidates: 2 | |
| conditioning_length: 44000 | |
| use_bpe_tokenizer: True | |
| tokenizer_vocab: ./modules/tortoise-tts/tortoise/data/tokenizer.json | |
| load_aligned_codes: False | |
| data_type: img | |
| ] | |
| val:[ | |
| name: validation | |
| n_workers: 2 | |
| batch_size: 7 | |
| mode: paired_voice_audio | |
| path: ./training/louise/validation.txt | |
| fetcher_mode: ['lj'] | |
| phase: val | |
| max_wav_length: 255995 | |
| max_text_length: 200 | |
| sample_rate: 22050 | |
| load_conditioning: True | |
| num_conditioning_candidates: 2 | |
| conditioning_length: 44000 | |
| use_bpe_tokenizer: True | |
| tokenizer_vocab: ./modules/tortoise-tts/tortoise/data/tokenizer.json | |
| load_aligned_codes: False | |
| data_type: img | |
| ] | |
| ] | |
| steps:[ | |
| gpt_train:[ | |
| training: gpt | |
| loss_log_buffer: 500 | |
| optimizer: adamw | |
| optimizer_params:[ | |
| lr: 1e-05 | |
| weight_decay: 0.01 | |
| beta1: 0.9 | |
| beta2: 0.96 | |
| ] | |
| clip_grad_eps: 4 | |
| injectors:[ | |
| paired_to_mel:[ | |
| type: torch_mel_spectrogram | |
| mel_norm_file: ./modules/tortoise-tts/tortoise/data/mel_norms.pth | |
| in: wav | |
| out: paired_mel | |
| ] | |
| paired_cond_to_mel:[ | |
| type: for_each | |
| subtype: torch_mel_spectrogram | |
| mel_norm_file: ./modules/tortoise-tts/tortoise/data/mel_norms.pth | |
| in: conditioning | |
| out: paired_conditioning_mel | |
| ] | |
| to_codes:[ | |
| type: discrete_token | |
| in: paired_mel | |
| out: paired_mel_codes | |
| dvae_config: ./models/tortoise/train_diffusion_vocoder_22k_level.yml | |
| ] | |
| paired_fwd_text:[ | |
| type: generator | |
| generator: gpt | |
| in: ['paired_conditioning_mel', 'padded_text', 'text_lengths', 'paired_mel_codes', 'wav_lengths'] | |
| out: ['loss_text_ce', 'loss_mel_ce', 'logits'] | |
| ] | |
| ] | |
| losses:[ | |
| text_ce:[ | |
| type: direct | |
| weight: 0.01 | |
| key: loss_text_ce | |
| ] | |
| mel_ce:[ | |
| type: direct | |
| weight: 1 | |
| key: loss_mel_ce | |
| ] | |
| ] | |
| ] | |
| ] | |
| networks:[ | |
| gpt:[ | |
| type: generator | |
| which_model_G: unified_voice2 | |
| kwargs:[ | |
| layers: 30 | |
| model_dim: 1024 | |
| heads: 16 | |
| max_text_tokens: 402 | |
| max_mel_tokens: 604 | |
| max_conditioning_inputs: 2 | |
| mel_length_compression: 1024 | |
| number_text_tokens: 256 | |
| number_mel_codes: 8194 | |
| start_mel_token: 8192 | |
| stop_mel_token: 8193 | |
| start_text_token: 255 | |
| train_solo_embeddings: False | |
| use_mel_codes_as_input: True | |
| checkpointing: True | |
| tortoise_compat: True | |
| ] | |
| ] | |
| ] | |
| path:[ | |
| strict_load: True | |
| resume_state: ./training/louise/finetune/training_state//2550.state | |
| root: ./ | |
| experiments_root: ./training/louise/finetune | |
| models: ./training/louise/finetune/models | |
| training_state: ./training/louise/finetune/training_state | |
| log: ./training/louise/finetune | |
| val_images: ./training/louise/finetune/val_images | |
| ] | |
| train:[ | |
| niter: 2750 | |
| warmup_iter: -1 | |
| mega_batch_factor: 4 | |
| val_freq: 100 | |
| ema_enabled: False | |
| default_lr_scheme: MultiStepLR | |
| gen_lr_steps: [2, 4, 9, 18, 25, 33, 50, 59] | |
| lr_gamma: 0.5 | |
| ] | |
| eval:[ | |
| pure: False | |
| output_state: gen | |
| ] | |
| logger:[ | |
| save_checkpoint_freq: 100 | |
| visuals: ['gen', 'mel'] | |
| visual_debug_rate: 1100 | |
| is_mel_spectrogram: True | |
| ] | |
| is_train: True | |
| dist: False | |
| 23-04-17 14:07:42.354 - INFO: Set model [gpt] to ./training/louise/finetune/models/2550_gpt.pth | |
| 23-04-17 14:07:42.354 - INFO: Random seed: 3921 | |
| 23-04-17 14:07:43.055 - INFO: Number of training data elements: 293, iters: 11 | |
| 23-04-17 14:07:43.055 - INFO: Total epochs needed: 250 for iters 2,750 | |
| 23-04-17 14:07:50.468 - INFO: Loading model for [./training/louise/finetune/models/2550_gpt.pth] | |
| 23-04-17 14:07:53.367 - INFO: Resuming training from epoch: 250, iter: 2550. | |
| 23-04-17 14:07:53.426 - INFO: Start training from epoch: 250, iter: 2550 | |
| 23-04-17 14:08:00.904 - INFO: Training Metrics: {"loss_text_ce": 2.8150289058685303, "loss_mel_ce": 0.9089220762252808, "loss_gpt_total": 0.9370723962783813, "lr": 3.90625e-08, "it": 2551, "step": 1, "steps": 10, "epoch": 250, "iteration_rate": 7.1702799797058105} | |
| 23-04-17 14:08:06.010 - INFO: Training Metrics: {"loss_text_ce": 2.881300449371338, "loss_mel_ce": 0.9447442293167114, "loss_gpt_total": 0.9735573530197144, "lr": 3.90625e-08, "it": 2552, "step": 2, "steps": 10, "epoch": 250, "iteration_rate": 5.070466041564941} | |
| 23-04-17 14:08:11.526 - INFO: Training Metrics: {"loss_text_ce": 2.8664186000823975, "loss_mel_ce": 0.9824135303497314, "loss_gpt_total": 1.0110777616500854, "lr": 3.90625e-08, "it": 2553, "step": 3, "steps": 10, "epoch": 250, "iteration_rate": 5.483191728591919} | |
| 23-04-17 14:08:16.410 - INFO: Training Metrics: {"loss_text_ce": 2.8796322345733643, "loss_mel_ce": 0.9986575245857239, "loss_gpt_total": 1.027453899383545, "lr": 3.90625e-08, "it": 2554, "step": 4, "steps": 10, "epoch": 250, "iteration_rate": 4.852126359939575} | |
| 23-04-17 14:08:21.679 - INFO: Training Metrics: {"loss_text_ce": 2.8566620349884033, "loss_mel_ce": 1.006378173828125, "loss_gpt_total": 1.034944772720337, "lr": 3.90625e-08, "it": 2555, "step": 5, "steps": 10, "epoch": 250, "iteration_rate": 5.23550271987915} | |
| 23-04-17 14:08:27.067 - INFO: Training Metrics: {"loss_text_ce": 2.8422281742095947, "loss_mel_ce": 1.018097162246704, "loss_gpt_total": 1.0465195178985596, "lr": 3.90625e-08, "it": 2556, "step": 6, "steps": 10, "epoch": 250, "iteration_rate": 5.3555145263671875} | |
| 23-04-17 14:08:32.150 - INFO: Training Metrics: {"loss_text_ce": 2.835671901702881, "loss_mel_ce": 1.0255067348480225, "loss_gpt_total": 1.053863525390625, "lr": 3.90625e-08, "it": 2557, "step": 7, "steps": 10, "epoch": 250, "iteration_rate": 5.05054235458374} | |
| 23-04-17 14:08:37.199 - INFO: Training Metrics: {"loss_text_ce": 2.832628011703491, "loss_mel_ce": 1.0305334329605103, "loss_gpt_total": 1.0588598251342773, "lr": 3.90625e-08, "it": 2558, "step": 8, "steps": 10, "epoch": 250, "iteration_rate": 5.013982772827148} | |
| 23-04-17 14:08:42.598 - INFO: Training Metrics: {"loss_text_ce": 2.8266730308532715, "loss_mel_ce": 1.0352420806884766, "loss_gpt_total": 1.0635089874267578, "lr": 3.90625e-08, "it": 2559, "step": 9, "steps": 10, "epoch": 250, "iteration_rate": 5.366688013076782} | |
| 23-04-17 14:08:47.908 - INFO: Training Metrics: {"loss_text_ce": 2.81801700592041, "loss_mel_ce": 1.040480136871338, "loss_gpt_total": 1.0686603784561157, "lr": 3.90625e-08, "it": 2560, "step": 10, "steps": 10, "epoch": 250, "iteration_rate": 5.274477005004883} | |
| 23-04-17 14:08:53.134 - INFO: Saving models and training states. | |
| 23-04-17 14:08:53.187 - INFO: Finished training! | |