23-04-17 14:07:42.354 - INFO: name: louise model: extensibletrainer scale: 1 gpu_ids: [0] start_step: 0 checkpointing_enabled: True fp16: True bitsandbytes: True gpus: 1 datasets:[ train:[ name: training n_workers: 2 batch_size: 28 mode: paired_voice_audio path: ./training/louise/train.txt fetcher_mode: ['lj'] phase: train max_wav_length: 255995 max_text_length: 200 sample_rate: 22050 load_conditioning: True num_conditioning_candidates: 2 conditioning_length: 44000 use_bpe_tokenizer: True tokenizer_vocab: ./modules/tortoise-tts/tortoise/data/tokenizer.json load_aligned_codes: False data_type: img ] val:[ name: validation n_workers: 2 batch_size: 7 mode: paired_voice_audio path: ./training/louise/validation.txt fetcher_mode: ['lj'] phase: val max_wav_length: 255995 max_text_length: 200 sample_rate: 22050 load_conditioning: True num_conditioning_candidates: 2 conditioning_length: 44000 use_bpe_tokenizer: True tokenizer_vocab: ./modules/tortoise-tts/tortoise/data/tokenizer.json load_aligned_codes: False data_type: img ] ] steps:[ gpt_train:[ training: gpt loss_log_buffer: 500 optimizer: adamw optimizer_params:[ lr: 1e-05 weight_decay: 0.01 beta1: 0.9 beta2: 0.96 ] clip_grad_eps: 4 injectors:[ paired_to_mel:[ type: torch_mel_spectrogram mel_norm_file: ./modules/tortoise-tts/tortoise/data/mel_norms.pth in: wav out: paired_mel ] paired_cond_to_mel:[ type: for_each subtype: torch_mel_spectrogram mel_norm_file: ./modules/tortoise-tts/tortoise/data/mel_norms.pth in: conditioning out: paired_conditioning_mel ] to_codes:[ type: discrete_token in: paired_mel out: paired_mel_codes dvae_config: ./models/tortoise/train_diffusion_vocoder_22k_level.yml ] paired_fwd_text:[ type: generator generator: gpt in: ['paired_conditioning_mel', 'padded_text', 'text_lengths', 'paired_mel_codes', 'wav_lengths'] out: ['loss_text_ce', 'loss_mel_ce', 'logits'] ] ] losses:[ text_ce:[ type: direct weight: 0.01 key: loss_text_ce ] mel_ce:[ type: direct weight: 1 key: loss_mel_ce ] ] ] ] networks:[ gpt:[ type: generator which_model_G: unified_voice2 kwargs:[ layers: 30 model_dim: 1024 heads: 16 max_text_tokens: 402 max_mel_tokens: 604 max_conditioning_inputs: 2 mel_length_compression: 1024 number_text_tokens: 256 number_mel_codes: 8194 start_mel_token: 8192 stop_mel_token: 8193 start_text_token: 255 train_solo_embeddings: False use_mel_codes_as_input: True checkpointing: True tortoise_compat: True ] ] ] path:[ strict_load: True resume_state: ./training/louise/finetune/training_state//2550.state root: ./ experiments_root: ./training/louise/finetune models: ./training/louise/finetune/models training_state: ./training/louise/finetune/training_state log: ./training/louise/finetune val_images: ./training/louise/finetune/val_images ] train:[ niter: 2750 warmup_iter: -1 mega_batch_factor: 4 val_freq: 100 ema_enabled: False default_lr_scheme: MultiStepLR gen_lr_steps: [2, 4, 9, 18, 25, 33, 50, 59] lr_gamma: 0.5 ] eval:[ pure: False output_state: gen ] logger:[ save_checkpoint_freq: 100 visuals: ['gen', 'mel'] visual_debug_rate: 1100 is_mel_spectrogram: True ] is_train: True dist: False 23-04-17 14:07:42.354 - INFO: Set model [gpt] to ./training/louise/finetune/models/2550_gpt.pth 23-04-17 14:07:42.354 - INFO: Random seed: 3921 23-04-17 14:07:43.055 - INFO: Number of training data elements: 293, iters: 11 23-04-17 14:07:43.055 - INFO: Total epochs needed: 250 for iters 2,750 23-04-17 14:07:50.468 - INFO: Loading model for [./training/louise/finetune/models/2550_gpt.pth] 23-04-17 14:07:53.367 - INFO: Resuming training from epoch: 250, iter: 2550. 23-04-17 14:07:53.426 - INFO: Start training from epoch: 250, iter: 2550 23-04-17 14:08:00.904 - INFO: Training Metrics: {"loss_text_ce": 2.8150289058685303, "loss_mel_ce": 0.9089220762252808, "loss_gpt_total": 0.9370723962783813, "lr": 3.90625e-08, "it": 2551, "step": 1, "steps": 10, "epoch": 250, "iteration_rate": 7.1702799797058105} 23-04-17 14:08:06.010 - INFO: Training Metrics: {"loss_text_ce": 2.881300449371338, "loss_mel_ce": 0.9447442293167114, "loss_gpt_total": 0.9735573530197144, "lr": 3.90625e-08, "it": 2552, "step": 2, "steps": 10, "epoch": 250, "iteration_rate": 5.070466041564941} 23-04-17 14:08:11.526 - INFO: Training Metrics: {"loss_text_ce": 2.8664186000823975, "loss_mel_ce": 0.9824135303497314, "loss_gpt_total": 1.0110777616500854, "lr": 3.90625e-08, "it": 2553, "step": 3, "steps": 10, "epoch": 250, "iteration_rate": 5.483191728591919} 23-04-17 14:08:16.410 - INFO: Training Metrics: {"loss_text_ce": 2.8796322345733643, "loss_mel_ce": 0.9986575245857239, "loss_gpt_total": 1.027453899383545, "lr": 3.90625e-08, "it": 2554, "step": 4, "steps": 10, "epoch": 250, "iteration_rate": 4.852126359939575} 23-04-17 14:08:21.679 - INFO: Training Metrics: {"loss_text_ce": 2.8566620349884033, "loss_mel_ce": 1.006378173828125, "loss_gpt_total": 1.034944772720337, "lr": 3.90625e-08, "it": 2555, "step": 5, "steps": 10, "epoch": 250, "iteration_rate": 5.23550271987915} 23-04-17 14:08:27.067 - INFO: Training Metrics: {"loss_text_ce": 2.8422281742095947, "loss_mel_ce": 1.018097162246704, "loss_gpt_total": 1.0465195178985596, "lr": 3.90625e-08, "it": 2556, "step": 6, "steps": 10, "epoch": 250, "iteration_rate": 5.3555145263671875} 23-04-17 14:08:32.150 - INFO: Training Metrics: {"loss_text_ce": 2.835671901702881, "loss_mel_ce": 1.0255067348480225, "loss_gpt_total": 1.053863525390625, "lr": 3.90625e-08, "it": 2557, "step": 7, "steps": 10, "epoch": 250, "iteration_rate": 5.05054235458374} 23-04-17 14:08:37.199 - INFO: Training Metrics: {"loss_text_ce": 2.832628011703491, "loss_mel_ce": 1.0305334329605103, "loss_gpt_total": 1.0588598251342773, "lr": 3.90625e-08, "it": 2558, "step": 8, "steps": 10, "epoch": 250, "iteration_rate": 5.013982772827148} 23-04-17 14:08:42.598 - INFO: Training Metrics: {"loss_text_ce": 2.8266730308532715, "loss_mel_ce": 1.0352420806884766, "loss_gpt_total": 1.0635089874267578, "lr": 3.90625e-08, "it": 2559, "step": 9, "steps": 10, "epoch": 250, "iteration_rate": 5.366688013076782} 23-04-17 14:08:47.908 - INFO: Training Metrics: {"loss_text_ce": 2.81801700592041, "loss_mel_ce": 1.040480136871338, "loss_gpt_total": 1.0686603784561157, "lr": 3.90625e-08, "it": 2560, "step": 10, "steps": 10, "epoch": 250, "iteration_rate": 5.274477005004883} 23-04-17 14:08:53.134 - INFO: Saving models and training states. 23-04-17 14:08:53.187 - INFO: Finished training!