ecker commited on
Commit
09f258a
1 Parent(s): 5fd193e

Update models/config.llama.yaml

Browse files
Files changed (1) hide show
  1. models/config.llama.yaml +116 -129
models/config.llama.yaml CHANGED
@@ -1,130 +1,117 @@
1
- sample_rate: 24_000
2
- audio_backend: "vocos"
3
-
4
- models:
5
- - name: "ar+nar-tts+stt"
6
- size: "full"
7
- resp_levels: 8
8
- prom_levels: 8
9
- tasks: 9
10
- langs: 4
11
- tones: 1
12
- arch_type: llama
13
- training: False
14
- version: 5
15
- attention: auto
16
- dropout: 0.1
17
- #loss_factors:
18
- # text: 0.01
19
- # prom: 0.5
20
- # resp: 1.0
21
- capabilities: ["ar", "nar"]
22
- experimental:
23
- p_rvq_levels: "auto"
24
- audio_embedding_sums: True
25
- unified_position_ids: False
26
- split_classifiers: True
27
- #
28
- causal_size: 1
29
- interleave: False
30
- rvq_level_range: []
31
- tie_classifier_to_embedding: False
32
-
33
- #loras:
34
- #- name : "lora"
35
- # rank: 128
36
- # alpha: 128
37
- # training: True
38
- # rvq_levels: []
39
-
40
- hyperparameters:
41
- batch_size: 32
42
- gradient_accumulation_steps: 8
43
- gradient_clipping: 1.0
44
- warmup_steps: 10
45
-
46
- optimizer: Prodigy
47
- learning_rate: 1.0
48
- torch_optimizer: True
49
-
50
- scheduler: "" # ScheduleFree
51
- torch_scheduler: True
52
-
53
- evaluation:
54
- batch_size: 4
55
- frequency: 250
56
- size: 4
57
-
58
- steps: 500
59
- ar_temperature: 1.0
60
- nar_temperature: 0.0
61
-
62
- trainer:
63
- iterations: 1_000_000
64
- save_frequency: 250
65
- keep_last_checkpoints: 4
66
-
67
- resize_modules: True
68
-
69
- check_for_oom: False
70
- gradient_checkpointing: True
71
-
72
- weight_dtype: bfloat16
73
- amp: True
74
-
75
- backend: deepspeed
76
- deepspeed:
77
- inferencing: False
78
- amp: False
79
-
80
- load_webui: False
81
-
82
- inference:
83
- backend: local
84
- normalize: False
85
-
86
- weight_dtype: bfloat16
87
- amp: True
88
-
89
- optimizations:
90
- injects: False
91
- replace: True
92
-
93
- linear: False
94
- embedding: False
95
- optimizers: True
96
-
97
- bitsandbytes: False
98
- dadaptation: False
99
- bitnet: False
100
- fp8: False
101
-
102
- dataset:
103
- use_hdf5: True
104
- hdf5_flag: r
105
-
106
- use_metadata: True
107
- validate: True
108
-
109
- workers: 1
110
- cache: True
111
-
112
- duration_range: [3.0, 12.0]
113
-
114
- random_utterance: 1.0
115
- max_prompts: 1
116
- prompt_duration_range: [3.0, 3.0]
117
-
118
- max_resps: 1
119
- p_resp_append: 0.25
120
-
121
- sample_type: path # path # speaker
122
- sample_order: duration
123
- sample_max_duration_batch: 300
124
- sample_shuffle: False
125
-
126
- tasks_list: [ "tts", "stt" ]
127
-
128
- training: []
129
- validation: []
130
  noise: []
 
1
+ sample_rate: 24_000
2
+ audio_backend: "vocos"
3
+
4
+ models:
5
+ - name: "ar+nar-tts+stt"
6
+ size: "full"
7
+ resp_levels: 8
8
+ prom_levels: 8
9
+ tasks: 9
10
+ langs: 4
11
+ tones: 1
12
+ arch_type: llama
13
+ training: False
14
+ version: 5
15
+ attention: auto
16
+ dropout: 0.1
17
+ #loss_factors:
18
+ # text: 0.01
19
+ # prom: 0.5
20
+ # resp: 1.0
21
+ capabilities: ["ar", "nar"]
22
+ experimental:
23
+ # modifies model arch
24
+ audio_embedding_sums: True
25
+ unified_position_ids: False
26
+ split_classifiers: True
27
+
28
+ #loras:
29
+ #- name : "lora"
30
+ # rank: 128
31
+ # alpha: 128
32
+ # training: True
33
+ # rvq_levels: []
34
+
35
+ hyperparameters:
36
+ batch_size: 32
37
+ gradient_accumulation_steps: 8
38
+ gradient_clipping: 1.0
39
+ warmup_steps: 10
40
+
41
+ optimizer: Prodigy
42
+ learning_rate: 1.0
43
+ torch_optimizer: True
44
+
45
+ scheduler: "" # ScheduleFree
46
+ torch_scheduler: True
47
+
48
+ evaluation:
49
+ batch_size: 4
50
+ frequency: 250
51
+ size: 4
52
+
53
+ steps: 500
54
+ ar_temperature: 1.0
55
+ nar_temperature: 0.0
56
+
57
+ trainer:
58
+ iterations: 1_000_000
59
+ save_frequency: 250
60
+ keep_last_checkpoints: 4
61
+
62
+ resize_modules: True
63
+ gradient_checkpointing: True
64
+
65
+ weight_dtype: bfloat16
66
+ amp: True
67
+
68
+ backend: deepspeed
69
+ deepspeed:
70
+ inferencing: False
71
+ amp: False
72
+
73
+ inference:
74
+ backend: local
75
+ weight_dtype: bfloat16
76
+ amp: True
77
+
78
+ optimizations:
79
+ injects: False
80
+ replace: True
81
+
82
+ linear: False
83
+ embedding: False
84
+ optimizers: True
85
+
86
+ bitsandbytes: False
87
+ dadaptation: False
88
+ bitnet: False
89
+ fp8: False
90
+
91
+ dataset:
92
+ use_hdf5: True
93
+ hdf5_flag: r
94
+
95
+ use_metadata: True
96
+ validate: True
97
+
98
+ workers: 1
99
+ cache: True
100
+
101
+ duration_range: [3.0, 12.0]
102
+
103
+ prompt_max_samples: 1
104
+ prompt_duration_range: [3.0, 3.0]
105
+
106
+ resps_max_samples: 1
107
+
108
+ sample_type: path # path # speaker
109
+ sample_order: duration
110
+ sample_max_duration_batch: 300
111
+ sample_shuffle: False
112
+
113
+ tasks_list: [ "tts", "stt" ]
114
+
115
+ training: []
116
+ validation: []
 
 
 
 
 
 
 
 
 
 
 
 
 
117
  noise: []