Upload 2 files

ckpt/ar+nar-retnet-8/fp32.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ecd8279a7e8658230b743ed66dac025059cfe292f869cd34257b3b568528767a
+size 441054655

config.ar_nar.yaml

@@ -1,10 +1,7 @@
 dataset:
-  training: [
-  ]
-  validation: [
-  ]
-  noise: [
-  ]
+  training: []
+  validation: []
+  noise: []
   
   speaker_name_getter: "lambda p: f'{p.parts[-3]}_{p.parts[-2]}'"
   
@@ -13,7 +10,7 @@ dataset:
   hdf5_flag: r
   validate: True
 
-  workers: 4
+  workers: 2
   cache: True
 
   phones_range: [4, 256]
@@ -21,41 +18,60 @@ dataset:
 
   random_utterance: 1.0
   max_prompts: 3
-  prompt_duration: 3.0
+  prompt_duration: 6.0
 
   sample_type: speaker
 
-  tasks_list: ["tts"] # , "ns", "sr", "tse", "cse", "nse", "tts"]
+  tasks_list: [ "tts" ] # , [ "tts", "tts-c", "ns", "sr", "tse", "cse", "nse", "tts"]
 
 models:
-  _prom_levels: 4
+  _prom_levels: 8
   _max_levels: 8
 
   _models:
   - name: "ar+nar"
     size: "full"
-    resp_levels: 4
-    prom_levels: 4
+    resp_levels: 8
+    prom_levels: 8
     tasks: 8
     arch_type: "retnet"
     training: True
+    version: 2
 
 hyperparameters:
   batch_size: 8
-  gradient_accumulation_steps: 1
+  gradient_accumulation_steps: 32
   gradient_clipping: 100
   
-  optimizer: AdamW
-  learning_rate: 1.0e-5
+  optimizer: Prodigy
+  torch_optimizer: True
+  learning_rate: 1.0
   
   scheduler_type: ""
+  #scheduler_type: OneCycle
+  #scheduler_params:
+  #  cycle_first_step_size: 10_000
+  #  cycle_first_stair_count: 10_000
+
+  #  cycle_second_step_size: 15_000
+  #  cycle_second_stair_count: 15_000
+
+  #  decay_step_size: 5_000
+
+  #  cycle_min_lr: 2.5e-4 # 1.0e-5
+  #  cycle_max_lr: 2.5e-4 # 1.0e-4
+  #  decay_lr_rate: 0.0
+
+  #  cycle_min_mom: 0.90
+  #  cycle_max_mom: 0.99
+  #  decay_mom_rate: 0.0
 
 evaluation:
   batch_size: 16
-  frequency: 500
+  frequency: 250
   size: 16
   
-  steps: 300
+  steps: 450
   ar_temperature: 0.95
   nar_temperature: 0.25
   load_disabled_engines: True
@@ -66,7 +82,7 @@ trainer:
   save_tag: step
   save_on_oom: True
   save_on_quit: True
-  save_frequency: 500
+  save_frequency: 100
   export_on_save: True
 
   keep_last_checkpoints: 4
@@ -74,32 +90,34 @@ trainer:
   aggressive_optimizations: False
   load_disabled_engines: False
 
-  load_state_dict: True
+  #load_state_dict: True
+  #strict_loading: False
+  #load_tag: "9500"
+  #load_states: False
+  #restart_step_count: True
   
   gc_mode: None # "global_step"
 
-  weight_dtype: float32
+  weight_dtype: bfloat16
   amp: False
 
-  backend: local
+  backend: deepspeed
   deepspeed:
     zero_optimization_level: 0
     use_compression_training: True
 
-inference:
-  weight_dtype: float32
-  amp: False
+  activation_checkpointing: True
 
+inference:
   use_vocos: True
   normalize: False
 
-  recurrent_chunk_size: 0
-  recurrent_forward: False
+  weight_dtype: bfloat16
+  amp: False
 
 bitsandbytes:
   enabled: False
   injects: True
   linear: True
   embedding: True
-
-device: cpu
+