jcmc commited on
Commit
cee3305
1 Parent(s): e90ef2f

Upload lm-boosted decoder

Browse files
.gitattributes CHANGED
@@ -26,3 +26,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
26
  *.zstandard filter=lfs diff=lfs merge=lfs -text
27
  *tfevents* filter=lfs diff=lfs merge=lfs -text
28
  wandb/offline-run-20220203_154548-23cvd7o7/run-23cvd7o7.wandb filter=lfs diff=lfs merge=lfs -text
 
 
 
 
26
  *.zstandard filter=lfs diff=lfs merge=lfs -text
27
  *tfevents* filter=lfs diff=lfs merge=lfs -text
28
  wandb/offline-run-20220203_154548-23cvd7o7/run-23cvd7o7.wandb filter=lfs diff=lfs merge=lfs -text
29
+ 5gram.arpa filter=lfs diff=lfs merge=lfs -text
30
+ 5gram_correct.arpa filter=lfs diff=lfs merge=lfs -text
31
+ text.txt filter=lfs diff=lfs merge=lfs -text
.ipynb_checkpoints/n-gram-checkpoint.ipynb ADDED
@@ -0,0 +1,481 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "cells": [
3
+ {
4
+ "cell_type": "code",
5
+ "execution_count": 51,
6
+ "id": "831245a1",
7
+ "metadata": {},
8
+ "outputs": [],
9
+ "source": [
10
+ "import pandas as pd"
11
+ ]
12
+ },
13
+ {
14
+ "cell_type": "code",
15
+ "execution_count": 1,
16
+ "id": "2ac8a30f",
17
+ "metadata": {},
18
+ "outputs": [],
19
+ "source": [
20
+ "target_lang=\"ga-IE\" # change to your target lang"
21
+ ]
22
+ },
23
+ {
24
+ "cell_type": "code",
25
+ "execution_count": 101,
26
+ "id": "15710167",
27
+ "metadata": {},
28
+ "outputs": [
29
+ {
30
+ "name": "stderr",
31
+ "output_type": "stream",
32
+ "text": [
33
+ "Using custom data configuration ga-pl-lang1=ga,lang2=pl\n",
34
+ "Reusing dataset opus_dgt (/workspace/cache/hf/datasets/opus_dgt/ga-pl-lang1=ga,lang2=pl/0.0.0/a4db75cea3712eb5d4384f0539db82abf897c6b6da5e5e81693e8fd201efc346)\n"
35
+ ]
36
+ }
37
+ ],
38
+ "source": [
39
+ "from datasets import load_dataset\n",
40
+ "\n",
41
+ "# dataset = load_dataset(\"mozilla-foundation/common_voice_8_0\", \n",
42
+ "# \"ga-IE\", \n",
43
+ "# split=\"train\", \n",
44
+ "# use_auth_token = True)\n",
45
+ "\n",
46
+ "dataset = load_dataset(\"opus_dgt\", lang1=\"ga\", lang2=\"pl\", split = 'train')"
47
+ ]
48
+ },
49
+ {
50
+ "cell_type": "code",
51
+ "execution_count": 102,
52
+ "id": "fb20d4de",
53
+ "metadata": {},
54
+ "outputs": [],
55
+ "source": [
56
+ "# ga_txt = [i['ga'] for i in dataset['translation']]\n",
57
+ "# ga_txt = pd.Series(ga_txt)"
58
+ ]
59
+ },
60
+ {
61
+ "cell_type": "code",
62
+ "execution_count": 103,
63
+ "id": "eeca1851",
64
+ "metadata": {},
65
+ "outputs": [],
66
+ "source": [
67
+ "chars_to_ignore_regex = '[,?.!\\-\\;\\:\"“%‘”�—’…–]' # change to the ignored characters of your fine-tuned model"
68
+ ]
69
+ },
70
+ {
71
+ "cell_type": "code",
72
+ "execution_count": 107,
73
+ "id": "4df93c9c",
74
+ "metadata": {},
75
+ "outputs": [],
76
+ "source": [
77
+ "import re\n",
78
+ "\n",
79
+ "def extract_text(batch):\n",
80
+ " text = batch[\"translation\"]\n",
81
+ " ga_text = text['ga']\n",
82
+ " batch[\"text\"] = re.sub(chars_to_ignore_regex, \"\", ga_text.lower())\n",
83
+ " return batch"
84
+ ]
85
+ },
86
+ {
87
+ "cell_type": "code",
88
+ "execution_count": 108,
89
+ "id": "84bedd13",
90
+ "metadata": {},
91
+ "outputs": [
92
+ {
93
+ "data": {
94
+ "application/vnd.jupyter.widget-view+json": {
95
+ "model_id": "d9a11f167bb94faa8e9f6a511407acb4",
96
+ "version_major": 2,
97
+ "version_minor": 0
98
+ },
99
+ "text/plain": [
100
+ "0ex [00:00, ?ex/s]"
101
+ ]
102
+ },
103
+ "metadata": {},
104
+ "output_type": "display_data"
105
+ }
106
+ ],
107
+ "source": [
108
+ "dataset = dataset.map(extract_text, remove_columns=dataset.column_names)"
109
+ ]
110
+ },
111
+ {
112
+ "cell_type": "code",
113
+ "execution_count": 112,
114
+ "id": "31cb3c6b",
115
+ "metadata": {},
116
+ "outputs": [
117
+ {
118
+ "data": {
119
+ "application/vnd.jupyter.widget-view+json": {
120
+ "model_id": "342d92a5d9c44c59bcb5dca143ced3b6",
121
+ "version_major": 2,
122
+ "version_minor": 0
123
+ },
124
+ "text/plain": [
125
+ "Pushing dataset shards to the dataset hub: 0%| | 0/1 [00:00<?, ?it/s]"
126
+ ]
127
+ },
128
+ "metadata": {},
129
+ "output_type": "display_data"
130
+ }
131
+ ],
132
+ "source": [
133
+ "dataset.push_to_hub(f\"{target_lang}_opus_dgt_train\", split=\"train\")"
134
+ ]
135
+ },
136
+ {
137
+ "cell_type": "markdown",
138
+ "id": "70952673",
139
+ "metadata": {},
140
+ "source": [
141
+ "## N-gram KenLM"
142
+ ]
143
+ },
144
+ {
145
+ "cell_type": "code",
146
+ "execution_count": 116,
147
+ "id": "51756959",
148
+ "metadata": {},
149
+ "outputs": [
150
+ {
151
+ "data": {
152
+ "application/vnd.jupyter.widget-view+json": {
153
+ "model_id": "38d3c229117f4e60a7778f974ac609de",
154
+ "version_major": 2,
155
+ "version_minor": 0
156
+ },
157
+ "text/plain": [
158
+ "Downloading: 0%| | 0.00/1.60k [00:00<?, ?B/s]"
159
+ ]
160
+ },
161
+ "metadata": {},
162
+ "output_type": "display_data"
163
+ },
164
+ {
165
+ "name": "stderr",
166
+ "output_type": "stream",
167
+ "text": [
168
+ "Using custom data configuration jcmc--ga-IE_opus_dgt_train-aa318da91f5f84f6\n"
169
+ ]
170
+ },
171
+ {
172
+ "name": "stdout",
173
+ "output_type": "stream",
174
+ "text": [
175
+ "Downloading and preparing dataset opus_dgt/ga-pl (download: 12.11 MiB, generated: 28.99 MiB, post-processed: Unknown size, total: 41.11 MiB) to /workspace/cache/hf/datasets/parquet/jcmc--ga-IE_opus_dgt_train-aa318da91f5f84f6/0.0.0/1638526fd0e8d960534e2155dc54fdff8dce73851f21f031d2fb9c2cf757c121...\n"
176
+ ]
177
+ },
178
+ {
179
+ "data": {
180
+ "application/vnd.jupyter.widget-view+json": {
181
+ "model_id": "e5e07f18549b443ead74991a9b338593",
182
+ "version_major": 2,
183
+ "version_minor": 0
184
+ },
185
+ "text/plain": [
186
+ " 0%| | 0/1 [00:00<?, ?it/s]"
187
+ ]
188
+ },
189
+ "metadata": {},
190
+ "output_type": "display_data"
191
+ },
192
+ {
193
+ "data": {
194
+ "application/vnd.jupyter.widget-view+json": {
195
+ "model_id": "0e83c78fa1bc43f19a56b623c92a64a4",
196
+ "version_major": 2,
197
+ "version_minor": 0
198
+ },
199
+ "text/plain": [
200
+ "Downloading: 0%| | 0.00/12.7M [00:00<?, ?B/s]"
201
+ ]
202
+ },
203
+ "metadata": {},
204
+ "output_type": "display_data"
205
+ },
206
+ {
207
+ "data": {
208
+ "application/vnd.jupyter.widget-view+json": {
209
+ "model_id": "06649f5cd3324eb49a1bd09b68aa23b6",
210
+ "version_major": 2,
211
+ "version_minor": 0
212
+ },
213
+ "text/plain": [
214
+ " 0%| | 0/1 [00:00<?, ?it/s]"
215
+ ]
216
+ },
217
+ "metadata": {},
218
+ "output_type": "display_data"
219
+ },
220
+ {
221
+ "name": "stdout",
222
+ "output_type": "stream",
223
+ "text": [
224
+ "Dataset parquet downloaded and prepared to /workspace/cache/hf/datasets/parquet/jcmc--ga-IE_opus_dgt_train-aa318da91f5f84f6/0.0.0/1638526fd0e8d960534e2155dc54fdff8dce73851f21f031d2fb9c2cf757c121. Subsequent calls will reuse this data.\n"
225
+ ]
226
+ }
227
+ ],
228
+ "source": [
229
+ "from datasets import load_dataset\n",
230
+ "\n",
231
+ "dataset = load_dataset(\"jcmc/ga-IE_opus_dgt_train\", split=\"train\")\n",
232
+ "\n",
233
+ "with open(\"text.txt\", \"w\") as file:\n",
234
+ " file.write(\" \".join(dataset[\"text\"]))"
235
+ ]
236
+ },
237
+ {
238
+ "cell_type": "code",
239
+ "execution_count": 118,
240
+ "id": "77eb3a41",
241
+ "metadata": {},
242
+ "outputs": [
243
+ {
244
+ "name": "stdout",
245
+ "output_type": "stream",
246
+ "text": [
247
+ "=== 1/5 Counting and sorting n-grams ===\n",
248
+ "Reading /workspace/wav2vec-1b-cv8-ir/text.txt\n",
249
+ "----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
250
+ "****************************************************************************************************\n",
251
+ "Unigram tokens 4378228 types 70781\n",
252
+ "=== 2/5 Calculating and sorting adjusted counts ===\n",
253
+ "Chain sizes: 1:849372 2:14475680768 3:27141902336 4:43427041280 5:63331106816\n",
254
+ "Statistics:\n",
255
+ "1 70780 D1=0.684187 D2=1.0538 D3+=1.37643\n",
256
+ "2 652306 D1=0.766205 D2=1.12085 D3+=1.39031\n",
257
+ "3 1669326 D1=0.84217 D2=1.20654 D3+=1.39941\n",
258
+ "4 2514789 D1=0.896214 D2=1.29731 D3+=1.47431\n",
259
+ "5 3053088 D1=0.794858 D2=1.47897 D3+=1.5117\n",
260
+ "Memory estimate for binary LM:\n",
261
+ "type MB\n",
262
+ "probing 164 assuming -p 1.5\n",
263
+ "probing 192 assuming -r models -p 1.5\n",
264
+ "trie 77 without quantization\n",
265
+ "trie 42 assuming -q 8 -b 8 quantization \n",
266
+ "trie 69 assuming -a 22 array pointer compression\n",
267
+ "trie 34 assuming -a 22 -q 8 -b 8 array pointer compression and quantization\n",
268
+ "=== 3/5 Calculating and sorting initial probabilities ===\n",
269
+ "Chain sizes: 1:849360 2:10436896 3:33386520 4:60354936 5:85486464\n",
270
+ "----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
271
+ "####################################################################################################\n",
272
+ "=== 4/5 Calculating and writing order-interpolated probabilities ===\n",
273
+ "Chain sizes: 1:849360 2:10436896 3:33386520 4:60354936 5:85486464\n",
274
+ "----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
275
+ "####################################################################################################\n",
276
+ "=== 5/5 Writing ARPA model ===\n",
277
+ "----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
278
+ "****************************************************************************************************\n",
279
+ "Name:lmplz\tVmPeak:145097728 kB\tVmRSS:51788 kB\tRSSMax:25679020 kB\tuser:9.15304\tsys:14.1178\tCPU:23.2708\treal:20.9339\n"
280
+ ]
281
+ }
282
+ ],
283
+ "source": [
284
+ "!../kenlm/build/bin/lmplz -o 5 <\"text.txt\" > \"5gram.arpa\""
285
+ ]
286
+ },
287
+ {
288
+ "cell_type": "code",
289
+ "execution_count": 122,
290
+ "id": "0e043b87",
291
+ "metadata": {},
292
+ "outputs": [],
293
+ "source": [
294
+ "with open(\"5gram.arpa\", \"r\") as read_file, open(\"5gram_correct.arpa\", \"w\") as write_file:\n",
295
+ " has_added_eos = False\n",
296
+ " for line in read_file:\n",
297
+ " if not has_added_eos and \"ngram 1=\" in line:\n",
298
+ " count=line.strip().split(\"=\")[-1]\n",
299
+ " write_file.write(line.replace(f\"{count}\", f\"{int(count)+1}\"))\n",
300
+ " elif not has_added_eos and \"<s>\" in line:\n",
301
+ " write_file.write(line)\n",
302
+ " write_file.write(line.replace(\"<s>\", \"</s>\"))\n",
303
+ " has_added_eos = True\n",
304
+ " else:\n",
305
+ " write_file.write(line)"
306
+ ]
307
+ },
308
+ {
309
+ "cell_type": "code",
310
+ "execution_count": 123,
311
+ "id": "d106c7d1",
312
+ "metadata": {},
313
+ "outputs": [
314
+ {
315
+ "name": "stdout",
316
+ "output_type": "stream",
317
+ "text": [
318
+ "\\data\\\n",
319
+ "ngram 1=70781\n",
320
+ "ngram 2=652306\n",
321
+ "ngram 3=1669326\n",
322
+ "ngram 4=2514789\n",
323
+ "ngram 5=3053088\n",
324
+ "\n",
325
+ "\\1-grams:\n",
326
+ "-5.8501472\t<unk>\t0\n",
327
+ "0\t<s>\t-0.11565505\n",
328
+ "0\t</s>\t-0.11565505\n",
329
+ "-5.4088216\tmiontuairisc\t-0.20133564\n",
330
+ "-4.6517477\tcheartaitheach\t-0.24842946\n",
331
+ "-2.1893916\tmaidir\t-1.7147961\n",
332
+ "-2.1071756\tle\t-0.7007309\n",
333
+ "-4.156014\tcoinbhinsiún\t-0.31064242\n",
334
+ "-1.8876181\tar\t-0.9045828\n",
335
+ "-4.62287\tdhlínse\t-0.24268326\n",
336
+ "-1.6051095\tagus\t-0.8729715\n",
337
+ "-4.1465816\taithint\t-0.21693327\n"
338
+ ]
339
+ }
340
+ ],
341
+ "source": [
342
+ "!head -20 5gram_correct.arpa"
343
+ ]
344
+ },
345
+ {
346
+ "cell_type": "code",
347
+ "execution_count": 124,
348
+ "id": "85ef4c43",
349
+ "metadata": {},
350
+ "outputs": [],
351
+ "source": [
352
+ "from transformers import AutoProcessor\n",
353
+ "\n",
354
+ "processor = AutoProcessor.from_pretrained(\"./\")"
355
+ ]
356
+ },
357
+ {
358
+ "cell_type": "code",
359
+ "execution_count": 125,
360
+ "id": "cb2a2768",
361
+ "metadata": {},
362
+ "outputs": [],
363
+ "source": [
364
+ "vocab_dict = processor.tokenizer.get_vocab()\n",
365
+ "sorted_vocab_dict = {k.lower(): v for k, v in sorted(vocab_dict.items(), key=lambda item: item[1])}"
366
+ ]
367
+ },
368
+ {
369
+ "cell_type": "code",
370
+ "execution_count": 126,
371
+ "id": "d19eee6f",
372
+ "metadata": {},
373
+ "outputs": [
374
+ {
375
+ "name": "stderr",
376
+ "output_type": "stream",
377
+ "text": [
378
+ "Found entries of length > 1 in alphabet. This is unusual unless style is BPE, but the alphabet was not recognized as BPE type. Is this correct?\n",
379
+ "Unigrams and labels don't seem to agree.\n"
380
+ ]
381
+ }
382
+ ],
383
+ "source": [
384
+ "from pyctcdecode import build_ctcdecoder\n",
385
+ "\n",
386
+ "decoder = build_ctcdecoder(\n",
387
+ " labels=list(sorted_vocab_dict.keys()),\n",
388
+ " kenlm_model_path=\"5gram_correct.arpa\",\n",
389
+ ")"
390
+ ]
391
+ },
392
+ {
393
+ "cell_type": "code",
394
+ "execution_count": 127,
395
+ "id": "4e8031a9",
396
+ "metadata": {},
397
+ "outputs": [],
398
+ "source": [
399
+ "from transformers import Wav2Vec2ProcessorWithLM\n",
400
+ "\n",
401
+ "processor_with_lm = Wav2Vec2ProcessorWithLM(\n",
402
+ " feature_extractor=processor.feature_extractor,\n",
403
+ " tokenizer=processor.tokenizer,\n",
404
+ " decoder=decoder\n",
405
+ ")"
406
+ ]
407
+ },
408
+ {
409
+ "cell_type": "code",
410
+ "execution_count": 128,
411
+ "id": "6f32faf4",
412
+ "metadata": {},
413
+ "outputs": [
414
+ {
415
+ "name": "stderr",
416
+ "output_type": "stream",
417
+ "text": [
418
+ "/workspace/wav2vec-1b-cv8-ir/./ is already a clone of https://huggingface.co/jcmc/wav2vec-1b-cv8-ir. Make sure you pull the latest changes with `repo.git_pull()`.\n"
419
+ ]
420
+ }
421
+ ],
422
+ "source": [
423
+ "from huggingface_hub import Repository\n",
424
+ "\n",
425
+ "repo = Repository(local_dir=\"./\", clone_from=\"jcmc/wav2vec-1b-cv8-ir\")"
426
+ ]
427
+ },
428
+ {
429
+ "cell_type": "code",
430
+ "execution_count": 129,
431
+ "id": "a7e91068",
432
+ "metadata": {},
433
+ "outputs": [
434
+ {
435
+ "data": {
436
+ "text/plain": [
437
+ "'/workspace/wav2vec-1b-cv8-ir'"
438
+ ]
439
+ },
440
+ "execution_count": 129,
441
+ "metadata": {},
442
+ "output_type": "execute_result"
443
+ }
444
+ ],
445
+ "source": [
446
+ "pwd"
447
+ ]
448
+ },
449
+ {
450
+ "cell_type": "code",
451
+ "execution_count": null,
452
+ "id": "0a1de336",
453
+ "metadata": {},
454
+ "outputs": [],
455
+ "source": [
456
+ "processor_with_lm.save_pretrained(\"xls-r-300m-sv\")"
457
+ ]
458
+ }
459
+ ],
460
+ "metadata": {
461
+ "kernelspec": {
462
+ "display_name": "Python 3",
463
+ "language": "python",
464
+ "name": "python3"
465
+ },
466
+ "language_info": {
467
+ "codemirror_mode": {
468
+ "name": "ipython",
469
+ "version": 3
470
+ },
471
+ "file_extension": ".py",
472
+ "mimetype": "text/x-python",
473
+ "name": "python",
474
+ "nbconvert_exporter": "python",
475
+ "pygments_lexer": "ipython3",
476
+ "version": "3.8.8"
477
+ }
478
+ },
479
+ "nbformat": 4,
480
+ "nbformat_minor": 5
481
+ }
.ipynb_checkpoints/preprocessor_config-checkpoint.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "do_normalize": true,
3
+ "feature_extractor_type": "Wav2Vec2FeatureExtractor",
4
+ "feature_size": 1,
5
+ "padding_side": "right",
6
+ "padding_value": 0,
7
+ "return_attention_mask": true,
8
+ "sampling_rate": 16000
9
+ }
.ipynb_checkpoints/tokenizer_config-checkpoint.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"unk_token": "[UNK]", "bos_token": "<s>", "eos_token": "</s>", "pad_token": "[PAD]", "do_lower_case": false, "word_delimiter_token": "|", "special_tokens_map_file": null, "tokenizer_file": null, "name_or_path": "./", "tokenizer_class": "Wav2Vec2CTCTokenizer", "processor_class": "Wav2Vec2ProcessorWithLM"}
5gram.arpa ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:07a5b3058d8cca7e1a61aa31b7ab0907fdb6ff7a104dfef12d8d470b2513c391
3
+ size 376008972
5gram_correct.arpa ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a8d79210ff27e6e122fa9af6411f860d85ca20ecac3d76bb4d716341b467e7a8
3
+ size 376008991
alphabet.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"labels": [" ", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "r", "s", "t", "u", "v", "w", "x", "y", "\u00e1", "\u00e9", "\u00ed", "\u00f3", "\u00fa", "\u2047", "", "<s>", "</s>"], "is_bpe": false}
language_model/5gram.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1238bb0e9f91c4250009bb8496c4732cad0da6f6a9fbaa945cb5782af4a4bbdc
3
+ size 173705975
language_model/attrs.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"alpha": 0.5, "beta": 1.5, "unk_score_offset": -10.0, "score_boundary": true}
language_model/unigrams.txt ADDED
The diff for this file is too large to render. See raw diff
 
n-gram.ipynb ADDED
@@ -0,0 +1,450 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "cells": [
3
+ {
4
+ "cell_type": "code",
5
+ "execution_count": 1,
6
+ "id": "e960dfd7",
7
+ "metadata": {},
8
+ "outputs": [],
9
+ "source": [
10
+ "import pandas as pd"
11
+ ]
12
+ },
13
+ {
14
+ "cell_type": "code",
15
+ "execution_count": 2,
16
+ "id": "7168a253",
17
+ "metadata": {},
18
+ "outputs": [],
19
+ "source": [
20
+ "target_lang=\"ga-IE\" # change to your target lang"
21
+ ]
22
+ },
23
+ {
24
+ "cell_type": "code",
25
+ "execution_count": 101,
26
+ "id": "e170befe",
27
+ "metadata": {},
28
+ "outputs": [
29
+ {
30
+ "name": "stderr",
31
+ "output_type": "stream",
32
+ "text": [
33
+ "Using custom data configuration ga-pl-lang1=ga,lang2=pl\n",
34
+ "Reusing dataset opus_dgt (/workspace/cache/hf/datasets/opus_dgt/ga-pl-lang1=ga,lang2=pl/0.0.0/a4db75cea3712eb5d4384f0539db82abf897c6b6da5e5e81693e8fd201efc346)\n"
35
+ ]
36
+ }
37
+ ],
38
+ "source": [
39
+ "from datasets import load_dataset\n",
40
+ "\n",
41
+ "# dataset = load_dataset(\"mozilla-foundation/common_voice_8_0\", \n",
42
+ "# \"ga-IE\", \n",
43
+ "# split=\"train\", \n",
44
+ "# use_auth_token = True)\n",
45
+ "\n",
46
+ "# dataset = load_dataset(\"opus_dgt\", lang1=\"ga\", lang2=\"pl\", split = 'train')"
47
+ ]
48
+ },
49
+ {
50
+ "cell_type": "code",
51
+ "execution_count": 3,
52
+ "id": "33973bd4",
53
+ "metadata": {},
54
+ "outputs": [],
55
+ "source": [
56
+ "# ga_txt = [i['ga'] for i in dataset['translation']]\n",
57
+ "# ga_txt = pd.Series(ga_txt)\n",
58
+ "\n",
59
+ "chars_to_ignore_regex = '[,?.!\\-\\;\\:\"“%‘”�—’…–]' # change to the ignored characters of your fine-tuned model\n",
60
+ "\n",
61
+ "import re\n",
62
+ "\n",
63
+ "def extract_text(batch):\n",
64
+ " text = batch[\"translation\"]\n",
65
+ " ga_text = text['ga']\n",
66
+ " batch[\"text\"] = re.sub(chars_to_ignore_regex, \"\", ga_text.lower())\n",
67
+ " return batch\n",
68
+ "\n",
69
+ "# dataset = dataset.map(extract_text, remove_columns=dataset.column_names)\n",
70
+ "\n",
71
+ "# dataset.push_to_hub(f\"{target_lang}_opus_dgt_train\", split=\"train\")"
72
+ ]
73
+ },
74
+ {
75
+ "cell_type": "markdown",
76
+ "id": "53e62728",
77
+ "metadata": {},
78
+ "source": [
79
+ "## N-gram KenLM"
80
+ ]
81
+ },
82
+ {
83
+ "cell_type": "code",
84
+ "execution_count": 4,
85
+ "id": "cb04cc9d",
86
+ "metadata": {},
87
+ "outputs": [
88
+ {
89
+ "data": {
90
+ "application/vnd.jupyter.widget-view+json": {
91
+ "model_id": "0c3dbd6368014788bff9249dd460d03e",
92
+ "version_major": 2,
93
+ "version_minor": 0
94
+ },
95
+ "text/plain": [
96
+ "Downloading: 0%| | 0.00/1.60k [00:00<?, ?B/s]"
97
+ ]
98
+ },
99
+ "metadata": {},
100
+ "output_type": "display_data"
101
+ },
102
+ {
103
+ "name": "stderr",
104
+ "output_type": "stream",
105
+ "text": [
106
+ "Using custom data configuration jcmc--ga-IE_opus_dgt_train-aa318da91f5f84f6\n"
107
+ ]
108
+ },
109
+ {
110
+ "name": "stdout",
111
+ "output_type": "stream",
112
+ "text": [
113
+ "Downloading and preparing dataset opus_dgt/ga-pl (download: 12.11 MiB, generated: 28.99 MiB, post-processed: Unknown size, total: 41.11 MiB) to /workspace/cache/hf/datasets/parquet/jcmc--ga-IE_opus_dgt_train-aa318da91f5f84f6/0.0.0/1638526fd0e8d960534e2155dc54fdff8dce73851f21f031d2fb9c2cf757c121...\n"
114
+ ]
115
+ },
116
+ {
117
+ "data": {
118
+ "application/vnd.jupyter.widget-view+json": {
119
+ "model_id": "42c92d51527a41fd91a38c13265c4ea6",
120
+ "version_major": 2,
121
+ "version_minor": 0
122
+ },
123
+ "text/plain": [
124
+ " 0%| | 0/1 [00:00<?, ?it/s]"
125
+ ]
126
+ },
127
+ "metadata": {},
128
+ "output_type": "display_data"
129
+ },
130
+ {
131
+ "data": {
132
+ "application/vnd.jupyter.widget-view+json": {
133
+ "model_id": "ae0badc4154f4fc586d3fc415d70c06a",
134
+ "version_major": 2,
135
+ "version_minor": 0
136
+ },
137
+ "text/plain": [
138
+ "Downloading: 0%| | 0.00/12.7M [00:00<?, ?B/s]"
139
+ ]
140
+ },
141
+ "metadata": {},
142
+ "output_type": "display_data"
143
+ },
144
+ {
145
+ "data": {
146
+ "application/vnd.jupyter.widget-view+json": {
147
+ "model_id": "f25b9f17355149df880331f926c76279",
148
+ "version_major": 2,
149
+ "version_minor": 0
150
+ },
151
+ "text/plain": [
152
+ " 0%| | 0/1 [00:00<?, ?it/s]"
153
+ ]
154
+ },
155
+ "metadata": {},
156
+ "output_type": "display_data"
157
+ },
158
+ {
159
+ "name": "stdout",
160
+ "output_type": "stream",
161
+ "text": [
162
+ "Dataset parquet downloaded and prepared to /workspace/cache/hf/datasets/parquet/jcmc--ga-IE_opus_dgt_train-aa318da91f5f84f6/0.0.0/1638526fd0e8d960534e2155dc54fdff8dce73851f21f031d2fb9c2cf757c121. Subsequent calls will reuse this data.\n"
163
+ ]
164
+ }
165
+ ],
166
+ "source": [
167
+ "from datasets import load_dataset\n",
168
+ "\n",
169
+ "dataset = load_dataset(\"jcmc/ga-IE_opus_dgt_train\", split=\"train\")\n",
170
+ "\n",
171
+ "with open(\"text.txt\", \"w\") as file:\n",
172
+ " file.write(\" \".join(dataset[\"text\"]))"
173
+ ]
174
+ },
175
+ {
176
+ "cell_type": "code",
177
+ "execution_count": 7,
178
+ "id": "06ce00d3",
179
+ "metadata": {},
180
+ "outputs": [
181
+ {
182
+ "name": "stdout",
183
+ "output_type": "stream",
184
+ "text": [
185
+ "=== 1/5 Counting and sorting n-grams ===\n",
186
+ "Reading /workspace/wav2vec-cv7-1b-ir/text.txt\n",
187
+ "----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
188
+ "****************************************************************************************************\n",
189
+ "Unigram tokens 4378228 types 70781\n",
190
+ "=== 2/5 Calculating and sorting adjusted counts ===\n",
191
+ "Chain sizes: 1:849372 2:14476327936 3:27143116800 4:43428982784 5:63333937152\n",
192
+ "Statistics:\n",
193
+ "1 70780 D1=0.684187 D2=1.0538 D3+=1.37643\n",
194
+ "2 652306 D1=0.766205 D2=1.12085 D3+=1.39031\n",
195
+ "3 1669326 D1=0.84217 D2=1.20654 D3+=1.39941\n",
196
+ "4 2514789 D1=0.896214 D2=1.29731 D3+=1.47431\n",
197
+ "5 3053088 D1=0.794858 D2=1.47897 D3+=1.5117\n",
198
+ "Memory estimate for binary LM:\n",
199
+ "type MB\n",
200
+ "probing 164 assuming -p 1.5\n",
201
+ "probing 192 assuming -r models -p 1.5\n",
202
+ "trie 77 without quantization\n",
203
+ "trie 42 assuming -q 8 -b 8 quantization \n",
204
+ "trie 69 assuming -a 22 array pointer compression\n",
205
+ "trie 34 assuming -a 22 -q 8 -b 8 array pointer compression and quantization\n",
206
+ "=== 3/5 Calculating and sorting initial probabilities ===\n",
207
+ "Chain sizes: 1:849360 2:10436896 3:33386520 4:60354936 5:85486464\n",
208
+ "----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
209
+ "####################################################################################################\n",
210
+ "=== 4/5 Calculating and writing order-interpolated probabilities ===\n",
211
+ "Chain sizes: 1:849360 2:10436896 3:33386520 4:60354936 5:85486464\n",
212
+ "----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
213
+ "####################################################################################################\n",
214
+ "=== 5/5 Writing ARPA model ===\n",
215
+ "----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
216
+ "****************************************************************************************************\n",
217
+ "Name:lmplz\tVmPeak:145104204 kB\tVmRSS:51852 kB\tRSSMax:25679996 kB\tuser:9.46174\tsys:23.4312\tCPU:32.893\treal:30.3848\n"
218
+ ]
219
+ }
220
+ ],
221
+ "source": [
222
+ "!../kenlm/build/bin/lmplz -o 5 <\"text.txt\" > \"5gram.arpa\""
223
+ ]
224
+ },
225
+ {
226
+ "cell_type": "code",
227
+ "execution_count": 8,
228
+ "id": "e076416d",
229
+ "metadata": {},
230
+ "outputs": [],
231
+ "source": [
232
+ "with open(\"5gram.arpa\", \"r\") as read_file, open(\"5gram_correct.arpa\", \"w\") as write_file:\n",
233
+ " has_added_eos = False\n",
234
+ " for line in read_file:\n",
235
+ " if not has_added_eos and \"ngram 1=\" in line:\n",
236
+ " count=line.strip().split(\"=\")[-1]\n",
237
+ " write_file.write(line.replace(f\"{count}\", f\"{int(count)+1}\"))\n",
238
+ " elif not has_added_eos and \"<s>\" in line:\n",
239
+ " write_file.write(line)\n",
240
+ " write_file.write(line.replace(\"<s>\", \"</s>\"))\n",
241
+ " has_added_eos = True\n",
242
+ " else:\n",
243
+ " write_file.write(line)"
244
+ ]
245
+ },
246
+ {
247
+ "cell_type": "code",
248
+ "execution_count": 9,
249
+ "id": "34ac1708",
250
+ "metadata": {},
251
+ "outputs": [
252
+ {
253
+ "name": "stdout",
254
+ "output_type": "stream",
255
+ "text": [
256
+ "\\data\\\n",
257
+ "ngram 1=70781\n",
258
+ "ngram 2=652306\n",
259
+ "ngram 3=1669326\n",
260
+ "ngram 4=2514789\n",
261
+ "ngram 5=3053088\n",
262
+ "\n",
263
+ "\\1-grams:\n",
264
+ "-5.8501472\t<unk>\t0\n",
265
+ "0\t<s>\t-0.11565505\n",
266
+ "0\t</s>\t-0.11565505\n",
267
+ "-5.4088216\tmiontuairisc\t-0.20133564\n",
268
+ "-4.6517477\tcheartaitheach\t-0.24842946\n",
269
+ "-2.1893916\tmaidir\t-1.7147961\n",
270
+ "-2.1071756\tle\t-0.7007309\n",
271
+ "-4.156014\tcoinbhinsiún\t-0.31064242\n",
272
+ "-1.8876181\tar\t-0.9045828\n",
273
+ "-4.62287\tdhlínse\t-0.24268326\n",
274
+ "-1.6051095\tagus\t-0.8729715\n",
275
+ "-4.1465816\taithint\t-0.21693327\n"
276
+ ]
277
+ }
278
+ ],
279
+ "source": [
280
+ "!head -20 5gram_correct.arpa"
281
+ ]
282
+ },
283
+ {
284
+ "cell_type": "code",
285
+ "execution_count": 10,
286
+ "id": "a096b154",
287
+ "metadata": {},
288
+ "outputs": [],
289
+ "source": [
290
+ "from transformers import AutoProcessor\n",
291
+ "\n",
292
+ "processor = AutoProcessor.from_pretrained(\"./\")"
293
+ ]
294
+ },
295
+ {
296
+ "cell_type": "code",
297
+ "execution_count": 11,
298
+ "id": "097ae051",
299
+ "metadata": {},
300
+ "outputs": [],
301
+ "source": [
302
+ "vocab_dict = processor.tokenizer.get_vocab()\n",
303
+ "sorted_vocab_dict = {k.lower(): v for k, v in sorted(vocab_dict.items(), key=lambda item: item[1])}"
304
+ ]
305
+ },
306
+ {
307
+ "cell_type": "code",
308
+ "execution_count": 12,
309
+ "id": "edeb35c3",
310
+ "metadata": {},
311
+ "outputs": [
312
+ {
313
+ "name": "stderr",
314
+ "output_type": "stream",
315
+ "text": [
316
+ "Found entries of length > 1 in alphabet. This is unusual unless style is BPE, but the alphabet was not recognized as BPE type. Is this correct?\n",
317
+ "Unigrams and labels don't seem to agree.\n"
318
+ ]
319
+ }
320
+ ],
321
+ "source": [
322
+ "from pyctcdecode import build_ctcdecoder\n",
323
+ "\n",
324
+ "decoder = build_ctcdecoder(\n",
325
+ " labels=list(sorted_vocab_dict.keys()),\n",
326
+ " kenlm_model_path=\"5gram_correct.arpa\",\n",
327
+ ")"
328
+ ]
329
+ },
330
+ {
331
+ "cell_type": "code",
332
+ "execution_count": 13,
333
+ "id": "3e8debd2",
334
+ "metadata": {},
335
+ "outputs": [],
336
+ "source": [
337
+ "from transformers import Wav2Vec2ProcessorWithLM\n",
338
+ "\n",
339
+ "processor_with_lm = Wav2Vec2ProcessorWithLM(\n",
340
+ " feature_extractor=processor.feature_extractor,\n",
341
+ " tokenizer=processor.tokenizer,\n",
342
+ " decoder=decoder\n",
343
+ ")"
344
+ ]
345
+ },
346
+ {
347
+ "cell_type": "code",
348
+ "execution_count": 15,
349
+ "id": "e8f3f674",
350
+ "metadata": {},
351
+ "outputs": [
352
+ {
353
+ "name": "stderr",
354
+ "output_type": "stream",
355
+ "text": [
356
+ "/workspace/wav2vec-cv7-1b-ir/./ is already a clone of https://huggingface.co/jcmc/wav2vec-cv7-1b-ir. Make sure you pull the latest changes with `repo.git_pull()`.\n"
357
+ ]
358
+ }
359
+ ],
360
+ "source": [
361
+ "from huggingface_hub import Repository\n",
362
+ "\n",
363
+ "repo = Repository(local_dir=\"./\", clone_from=\"jcmc/wav2vec-cv7-1b-ir\")"
364
+ ]
365
+ },
366
+ {
367
+ "cell_type": "code",
368
+ "execution_count": 16,
369
+ "id": "a260b7f2",
370
+ "metadata": {},
371
+ "outputs": [
372
+ {
373
+ "data": {
374
+ "text/plain": [
375
+ "'/workspace/wav2vec-cv7-1b-ir'"
376
+ ]
377
+ },
378
+ "execution_count": 16,
379
+ "metadata": {},
380
+ "output_type": "execute_result"
381
+ }
382
+ ],
383
+ "source": [
384
+ "pwd"
385
+ ]
386
+ },
387
+ {
388
+ "cell_type": "code",
389
+ "execution_count": 17,
390
+ "id": "b5958d5e",
391
+ "metadata": {},
392
+ "outputs": [],
393
+ "source": [
394
+ "processor_with_lm.save_pretrained(\"./\")"
395
+ ]
396
+ },
397
+ {
398
+ "cell_type": "code",
399
+ "execution_count": 19,
400
+ "id": "34798422",
401
+ "metadata": {},
402
+ "outputs": [
403
+ {
404
+ "name": "stdout",
405
+ "output_type": "stream",
406
+ "text": [
407
+ "Reading ./language_model/5gram_correct.arpa\n",
408
+ "----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
409
+ "****************************************************************************************************\n",
410
+ "SUCCESS\n"
411
+ ]
412
+ }
413
+ ],
414
+ "source": [
415
+ "!../kenlm/build/bin/build_binary ./language_model/5gram_correct.arpa ./language_model/5gram.bin"
416
+ ]
417
+ },
418
+ {
419
+ "cell_type": "code",
420
+ "execution_count": null,
421
+ "id": "8f2900a8",
422
+ "metadata": {},
423
+ "outputs": [],
424
+ "source": [
425
+ "repo.push_to_hub(commit_message=\"Upload lm-boosted decoder\")"
426
+ ]
427
+ }
428
+ ],
429
+ "metadata": {
430
+ "kernelspec": {
431
+ "display_name": "Python 3",
432
+ "language": "python",
433
+ "name": "python3"
434
+ },
435
+ "language_info": {
436
+ "codemirror_mode": {
437
+ "name": "ipython",
438
+ "version": 3
439
+ },
440
+ "file_extension": ".py",
441
+ "mimetype": "text/x-python",
442
+ "name": "python",
443
+ "nbconvert_exporter": "python",
444
+ "pygments_lexer": "ipython3",
445
+ "version": "3.8.8"
446
+ }
447
+ },
448
+ "nbformat": 4,
449
+ "nbformat_minor": 5
450
+ }
preprocessor_config.json CHANGED
@@ -4,6 +4,7 @@
4
  "feature_size": 1,
5
  "padding_side": "right",
6
  "padding_value": 0,
 
7
  "return_attention_mask": true,
8
  "sampling_rate": 16000
9
  }
 
4
  "feature_size": 1,
5
  "padding_side": "right",
6
  "padding_value": 0,
7
+ "processor_class": "Wav2Vec2ProcessorWithLM",
8
  "return_attention_mask": true,
9
  "sampling_rate": 16000
10
  }
special_tokens_map.json CHANGED
@@ -1 +1 @@
1
- {"bos_token": "<s>", "eos_token": "</s>", "unk_token": "[UNK]", "pad_token": "[PAD]", "additional_special_tokens": [{"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}]}
 
1
+ {"bos_token": "<s>", "eos_token": "</s>", "unk_token": "[UNK]", "pad_token": "[PAD]", "additional_special_tokens": [{"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}]}
text.txt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ab790db751ff5b9ddc7575cb8fe62d57c8a4b8fdda4950b2f9a17aad421bfffc
3
+ size 29862507
tokenizer_config.json CHANGED
@@ -1 +1 @@
1
- {"unk_token": "[UNK]", "bos_token": "<s>", "eos_token": "</s>", "pad_token": "[PAD]", "do_lower_case": false, "word_delimiter_token": "|", "special_tokens_map_file": null, "tokenizer_file": null, "name_or_path": "./", "tokenizer_class": "Wav2Vec2CTCTokenizer"}
 
1
+ {"unk_token": "[UNK]", "bos_token": "<s>", "eos_token": "</s>", "pad_token": "[PAD]", "do_lower_case": false, "word_delimiter_token": "|", "special_tokens_map_file": null, "tokenizer_file": null, "name_or_path": "./", "tokenizer_class": "Wav2Vec2CTCTokenizer", "processor_class": "Wav2Vec2ProcessorWithLM"}
wandb/offline-run-20220203_154548-23cvd7o7/files/output.log CHANGED
@@ -0,0 +1,144 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {'loss': 10.7305, 'learning_rate': 1.9200000000000003e-06, 'epoch': 3.12}
2
+ {'loss': 3.0098, 'learning_rate': 3.920000000000001e-06, 'epoch': 6.25}
3
+ {'loss': 2.9327, 'learning_rate': 5.92e-06, 'epoch': 9.37}
4
+ {'loss': 2.8216, 'learning_rate': 7.92e-06, 'epoch': 12.49}
5
+ {'loss': 2.3731, 'learning_rate': 9.920000000000002e-06, 'epoch': 15.62}
6
+ {'eval_loss': 1.5517226457595825, 'eval_wer': 0.9499121265377856, 'eval_runtime': 24.8327, 'eval_samples_per_second': 20.497, 'eval_steps_per_second': 1.289, 'epoch': 15.62}
7
+ {'loss': 1.9105, 'learning_rate': 1.1920000000000001e-05, 'epoch': 18.74}
8
+ {'loss': 1.714, 'learning_rate': 1.392e-05, 'epoch': 21.86}
9
+ {'loss': 1.5476, 'learning_rate': 1.5920000000000003e-05, 'epoch': 24.98}
10
+ {'loss': 1.4238, 'learning_rate': 1.792e-05, 'epoch': 28.12}
11
+ {'loss': 1.3312, 'learning_rate': 1.9920000000000002e-05, 'epoch': 31.25}
12
+ {'eval_loss': 0.8717297911643982, 'eval_wer': 0.6189220855301699, 'eval_runtime': 24.7966, 'eval_samples_per_second': 20.527, 'eval_steps_per_second': 1.29, 'epoch': 31.25}
13
+ {'loss': 1.2049, 'learning_rate': 1.912727272727273e-05, 'epoch': 34.37}
14
+ {'loss': 1.1346, 'learning_rate': 1.821818181818182e-05, 'epoch': 37.49}
15
+ {'loss': 1.0533, 'learning_rate': 1.730909090909091e-05, 'epoch': 40.62}
16
+ {'loss': 0.9638, 'learning_rate': 1.64e-05, 'epoch': 43.74}
17
+ {'loss': 0.9135, 'learning_rate': 1.549090909090909e-05, 'epoch': 46.86}
18
+ {'eval_loss': 0.8298946619033813, 'eval_wer': 0.5310486233157586, 'eval_runtime': 24.721, 'eval_samples_per_second': 20.59, 'eval_steps_per_second': 1.294, 'epoch': 46.86}
19
+ {'loss': 0.8568, 'learning_rate': 1.4581818181818184e-05, 'epoch': 49.98}
20
+ {'loss': 0.8141, 'learning_rate': 1.3672727272727273e-05, 'epoch': 53.12}
21
+ {'loss': 0.7526, 'learning_rate': 1.2763636363636365e-05, 'epoch': 56.25}
22
+ {'loss': 0.7177, 'learning_rate': 1.1854545454545457e-05, 'epoch': 59.37}
23
+ {'loss': 0.6719, 'learning_rate': 1.0945454545454545e-05, 'epoch': 62.49}
24
+ {'eval_loss': 0.8842366933822632, 'eval_wer': 0.5043936731107206, 'eval_runtime': 25.0435, 'eval_samples_per_second': 20.325, 'eval_steps_per_second': 1.278, 'epoch': 62.49}
25
+ {'loss': 0.6552, 'learning_rate': 1.0036363636363637e-05, 'epoch': 65.62}
26
+ {'loss': 0.6145, 'learning_rate': 9.127272727272727e-06, 'epoch': 68.74}
27
+ {'loss': 0.596, 'learning_rate': 8.21818181818182e-06, 'epoch': 71.86}
28
+ {'loss': 0.5719, 'learning_rate': 7.30909090909091e-06, 'epoch': 74.98}
29
+ {'loss': 0.5583, 'learning_rate': 6.4000000000000006e-06, 'epoch': 78.12}
30
+ {'eval_loss': 0.9093144536018372, 'eval_wer': 0.4800820152314001, 'eval_runtime': 24.6074, 'eval_samples_per_second': 20.685, 'eval_steps_per_second': 1.3, 'epoch': 78.12}
31
+ {'loss': 0.5417, 'learning_rate': 5.490909090909091e-06, 'epoch': 81.25}
32
+ {'loss': 0.5241, 'learning_rate': 4.581818181818183e-06, 'epoch': 84.37}
33
+ {'loss': 0.4901, 'learning_rate': 3.672727272727273e-06, 'epoch': 87.49}
34
+ {'loss': 0.4882, 'learning_rate': 2.763636363636364e-06, 'epoch': 90.62}
35
+ {'loss': 0.4728, 'learning_rate': 1.8545454545454546e-06, 'epoch': 93.74}
36
+ {'eval_loss': 0.9488239884376526, 'eval_wer': 0.48125366139425896, 'eval_runtime': 24.6884, 'eval_samples_per_second': 20.617, 'eval_steps_per_second': 1.296, 'epoch': 93.74}
37
+ {'loss': 0.4682, 'learning_rate': 9.454545454545455e-07, 'epoch': 96.86}
38
+ {'loss': 0.4634, 'learning_rate': 3.636363636363637e-08, 'epoch': 99.98}
39
+ {'train_runtime': 8387.3816, 'train_samples_per_second': 12.34, 'train_steps_per_second': 0.382, 'train_loss': 1.4163260304927825, 'epoch': 99.98}
40
+ ***** train metrics *****
41
+ epoch = 99.98
42
+ train_loss = 1.4163
43
+ train_runtime = 2:19:47.38
44
+ train_samples = 1035
45
+ train_samples_per_second = 12.34
46
+ train_steps_per_second = 0.382
47
+ 02/03/2022 18:01:03 - INFO - __main__ - *** Evaluate ***
48
+ ***** eval metrics *****
49
+ epoch = 99.98
50
+ eval_loss = 0.9562
51
+ eval_runtime = 0:00:24.83
52
+ eval_samples = 509
53
+ eval_samples_per_second = 20.497
54
+ eval_steps_per_second = 1.289
55
+ eval_wer = 0.4801
56
+ 02/03/2022 18:04:19 - WARNING - huggingface_hub.repository - Adding files tracked by Git LFS: ['wandb/offline-run-20220203_154548-23cvd7o7/run-23cvd7o7.wandb']. This may take a bit of time if the files are large.
57
+ 02/03/2022 18:05:13 - WARNING - huggingface_hub.repository - Several commits (2) will be pushed upstream.
58
+ 02/03/2022 18:05:13 - WARNING - huggingface_hub.repository - The progress bars may be unreliable.
59
+ 02/03/2022 18:07:27 - WARNING - huggingface_hub.repository - To https://huggingface.co/jcmc/wav2vec-cv7-1b-ir
60
+ f30c4d7..a0c1812 main -> main
61
+ 02/03/2022 18:07:33 - WARNING - huggingface_hub.repository - To https://huggingface.co/jcmc/wav2vec-cv7-1b-ir
62
+ a0c1812..e90ef2f main -> main
63
+ 16%|█████████████████████████ | 500/3200 [20:22<1:57:09, 2.60s/it]The following columns in the evaluation set don't have a corresponding argument in `Wav2Vec2ForCTC.forward` and have been ignored: input_length.
64
+ ***** Running Evaluation *****
65
+ Num examples = 509
66
+ Batch size = 16
67
+ 16%|█████████████████████████ | 500/3200 [20:47<1:57:09, 2.60s/it]Saving model checkpoint to ./checkpoint-500
68
+ Configuration saved in ./checkpoint-500/config.json
69
+ Model weights saved in ./checkpoint-500/pytorch_model.bin
70
+ Configuration saved in ./checkpoint-500/preprocessor_config.json
71
+ Configuration saved in ./preprocessor_config.json
72
+ 31%|█████████████████████████████████████████████████▋ | 1000/3200 [42:27<1:10:05, 1.91s/it]The following columns in the evaluation set don't have a corresponding argument in `Wav2Vec2ForCTC.forward` and have been ignored: input_length.
73
+ ***** Running Evaluation *****
74
+ Num examples = 509
75
+ Batch size = 16
76
+ 31%|█████████████████████████████████████████████████▋ | 1000/3200 [42:51<1:10:05, 1.91s/it]Saving model checkpoint to ./checkpoint-1000
77
+ Configuration saved in ./checkpoint-1000/config.json
78
+ Model weights saved in ./checkpoint-1000/pytorch_model.bin
79
+ Configuration saved in ./checkpoint-1000/preprocessor_config.json
80
+ 47%|█████████████████████████████████████████████████████████████████████████▌ | 1500/3200 [1:03:33<1:10:34, 2.49s/it]The following columns in the evaluation set don't have a corresponding argument in `Wav2Vec2ForCTC.forward` and have been ignored: input_length.
81
+ ***** Running Evaluation *****
82
+ Num examples = 509
83
+ Batch size = 16
84
+ 47%|█████████████████████████████████████████████████████████████████████████▌ | 1500/3200 [1:03:57<1:10:34, 2.49s/it]Saving model checkpoint to ./checkpoint-1500
85
+ Configuration saved in ./checkpoint-1500/config.json
86
+ Model weights saved in ./checkpoint-1500/pytorch_model.bin
87
+ Configuration saved in ./checkpoint-1500/preprocessor_config.json
88
+ 62%|███████████████████████████████████████████████████████████████████████████████████████████████████▍ | 2000/3200 [1:24:29<36:01, 1.80s/it]The following columns in the evaluation set don't have a corresponding argument in `Wav2Vec2ForCTC.forward` and have been ignored: input_length.
89
+ ***** Running Evaluation *****
90
+ Num examples = 509
91
+ Batch size = 16
92
+ 62%|███████████████████████████████████████████████████████████████████████████████████████████████████▍ | 2000/3200 [1:24:54<36:01, 1.80s/it]Saving model checkpoint to ./checkpoint-2000
93
+ Configuration saved in ./checkpoint-2000/config.json
94
+ Model weights saved in ./checkpoint-2000/pytorch_model.bin
95
+ Configuration saved in ./checkpoint-2000/preprocessor_config.json
96
+ Deleting older checkpoint [checkpoint-500] due to args.save_total_limit
97
+ 78%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 2500/3200 [1:45:29<31:58, 2.74s/it]The following columns in the evaluation set don't have a corresponding argument in `Wav2Vec2ForCTC.forward` and have been ignored: input_length.
98
+ ***** Running Evaluation *****
99
+ Num examples = 509
100
+ Batch size = 16
101
+ 78%|████████████████████████████████████████████████████████��███████████████████████████████████████████████████████████████████▏ | 2500/3200 [1:45:54<31:58, 2.74s/it]Saving model checkpoint to ./checkpoint-2500
102
+ Configuration saved in ./checkpoint-2500/config.json
103
+ Model weights saved in ./checkpoint-2500/pytorch_model.bin
104
+ Configuration saved in ./checkpoint-2500/preprocessor_config.json
105
+ Deleting older checkpoint [checkpoint-1000] due to args.save_total_limit
106
+ 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 3000/3200 [2:06:18<05:55, 1.78s/it]The following columns in the evaluation set don't have a corresponding argument in `Wav2Vec2ForCTC.forward` and have been ignored: input_length.
107
+ ***** Running Evaluation *****
108
+ Num examples = 509
109
+ Batch size = 16
110
+ 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 3000/3200 [2:06:42<05:55, 1.78s/it]Saving model checkpoint to ./checkpoint-3000
111
+ Configuration saved in ./checkpoint-3000/config.json
112
+ Model weights saved in ./checkpoint-3000/pytorch_model.bin
113
+ Configuration saved in ./checkpoint-3000/preprocessor_config.json
114
+ Deleting older checkpoint [checkpoint-1500] due to args.save_total_limit
115
+ 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3200/3200 [2:15:04<00:00, 1.84s/it]
116
+ Training completed. Do not forget to share your model on huggingface.co/models =)
117
+ 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3200/3200 [2:15:04<00:00, 2.53s/it]
118
+ Saving model checkpoint to ./
119
+ Configuration saved in ./config.json
120
+ Model weights saved in ./pytorch_model.bin
121
+ Configuration saved in ./preprocessor_config.json
122
+ The following columns in the evaluation set don't have a corresponding argument in `Wav2Vec2ForCTC.forward` and have been ignored: input_length.
123
+ ***** Running Evaluation *****
124
+ Num examples = 509
125
+ Batch size = 16
126
+ 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:24<00:00, 1.33it/s]
127
+ Saving model checkpoint to ./
128
+ Configuration saved in ./config.json
129
+ Model weights saved in ./pytorch_model.bin
130
+ Configuration saved in ./preprocessor_config.json
131
+ Adding files tracked by Git LFS: ['wandb/offline-run-20220203_154548-23cvd7o7/run-23cvd7o7.wandb']. This may take a bit of time if the files are large.
132
+ Several commits (2) will be pushed upstream.
133
+ The progress bars may be unreliable.
134
+ Upload file pytorch_model.bin: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋| 3.58G/3.59G [02:11<00:00, 30.5MB/s]To https://huggingface.co/jcmc/wav2vec-cv7-1b-ir
135
+ f30c4d7..a0c1812 main -> main20203_154548-23cvd7o7/run-23cvd7o7.wandb: 100%|█████████████████████████████████████████████████████████████████████████████████████| 39.6M/39.6M [00:18<00:00, 19.7MB/s]
136
+ Upload file runs/Feb03_15-40-29_job-829e2c87-5501-41ef-ad65-a05e9b64bfd7/events.out.tfevents.1643911287.job-829e2c87-5501-41ef-ad65-a05e9b64bfd7.27059.2: 100%|██████████████████| 358/358 [00:00<?, ?B/s]
137
+ Upload file pytorch_model.bin: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3.59G/3.59G [02:12<00:00, 29.1MB/s]
138
+ Upload file wandb/offline-run-20220203_154548-23cvd7o7/run-23cvd7o7.wandb: 100%|██████████████████████████████████████████████████████████████████████████████████████| 39.6M/39.6M [02:12<00:00, 314kB/s]
139
+ Upload file runs/Feb03_15-40-29_job-829e2c87-5501-41ef-ad65-a05e9b64bfd7/events.out.tfevents.1643911287.job-829e2c87-5501-41ef-ad65-a05e9b64bfd7.27059.2: 100%|██████████████████| 358/358 [02:12<?, ?B/s]
140
+ Upload file runs/Feb03_15-40-29_job-829e2c87-5501-41ef-ad65-a05e9b64bfd7/events.out.tfevents.1643902867.job-829e2c87-5501-41ef-ad65-a05e9b64bfd7.27059.0: 100%|███████| 11.7k/11.7k [02:12<00:00, 64.2B/s]
141
+ Dropping the following result as it does not have all the necessary fields:ents.out.tfevents.1643911287.job-829e2c87-5501-41ef-ad65-a05e9b64bfd7.27059.2: 100%|██████████████████| 358/358 [02:12<?, ?B/s]
142
+ {'dataset': {'name': 'MOZILLA-FOUNDATION/COMMON_VOICE_7_0 - GA-IE', 'type': 'common_voice', 'args': 'Config: ga-IE, Training split: train+validation, Eval split: test'}}
143
+ To https://huggingface.co/jcmc/wav2vec-cv7-1b-ir1-41ef-ad65-a05e9b64bfd7/events.out.tfevents.1643902867.job-829e2c87-5501-41ef-ad65-a05e9b64bfd7.27059.0: 100%|███████| 11.7k/11.7k [02:12<00:00, 64.2B/s]
144
+ a0c1812..e90ef2f main -> main
wandb/offline-run-20220203_154548-23cvd7o7/files/wandb-summary.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"_wandb": {"runtime": 8510}}
wandb/offline-run-20220203_154548-23cvd7o7/logs/debug-internal.log CHANGED
@@ -11,3 +11,71 @@
11
  2022-02-03 15:45:49,618 DEBUG HandlerThread:29156 [meta.py:_save_conda():79] save conda
12
  2022-02-03 15:45:50,243 DEBUG HandlerThread:29156 [meta.py:_save_conda():89] save conda done
13
  2022-02-03 15:45:50,243 DEBUG HandlerThread:29156 [meta.py:probe():252] probe done
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
11
  2022-02-03 15:45:49,618 DEBUG HandlerThread:29156 [meta.py:_save_conda():79] save conda
12
  2022-02-03 15:45:50,243 DEBUG HandlerThread:29156 [meta.py:_save_conda():89] save conda done
13
  2022-02-03 15:45:50,243 DEBUG HandlerThread:29156 [meta.py:probe():252] probe done
14
+ 2022-02-03 18:07:40,029 DEBUG SenderThread:29156 [sender.py:send():234] send: exit
15
+ 2022-02-03 18:07:40,030 DEBUG HandlerThread:29156 [handler.py:handle_request():130] handle_request: poll_exit
16
+ 2022-02-03 18:07:40,030 INFO SenderThread:29156 [sender.py:send_exit():366] handling exit code: 0
17
+ 2022-02-03 18:07:40,031 INFO SenderThread:29156 [sender.py:send_exit():368] handling runtime: 8510
18
+ 2022-02-03 18:07:40,032 INFO SenderThread:29156 [sender.py:_save_file():939] saving file wandb-summary.json with policy end
19
+ 2022-02-03 18:07:40,032 INFO SenderThread:29156 [sender.py:send_exit():374] send defer
20
+ 2022-02-03 18:07:40,032 DEBUG SenderThread:29156 [sender.py:send_request():248] send_request: poll_exit
21
+ 2022-02-03 18:07:40,033 DEBUG HandlerThread:29156 [handler.py:handle_request():130] handle_request: defer
22
+ 2022-02-03 18:07:40,033 INFO HandlerThread:29156 [handler.py:handle_request_defer():147] handle defer: 0
23
+ 2022-02-03 18:07:40,034 DEBUG SenderThread:29156 [sender.py:send_request():248] send_request: defer
24
+ 2022-02-03 18:07:40,034 INFO SenderThread:29156 [sender.py:send_request_defer():383] handle sender defer: 0
25
+ 2022-02-03 18:07:40,034 INFO SenderThread:29156 [sender.py:transition_state():387] send defer: 1
26
+ 2022-02-03 18:07:40,035 DEBUG HandlerThread:29156 [handler.py:handle_request():130] handle_request: defer
27
+ 2022-02-03 18:07:40,035 INFO HandlerThread:29156 [handler.py:handle_request_defer():147] handle defer: 1
28
+ 2022-02-03 18:07:40,106 DEBUG SenderThread:29156 [sender.py:send_request():248] send_request: defer
29
+ 2022-02-03 18:07:40,107 INFO SenderThread:29156 [sender.py:send_request_defer():383] handle sender defer: 1
30
+ 2022-02-03 18:07:40,107 INFO SenderThread:29156 [sender.py:transition_state():387] send defer: 2
31
+ 2022-02-03 18:07:40,108 DEBUG HandlerThread:29156 [handler.py:handle_request():130] handle_request: defer
32
+ 2022-02-03 18:07:40,108 INFO HandlerThread:29156 [handler.py:handle_request_defer():147] handle defer: 2
33
+ 2022-02-03 18:07:40,108 DEBUG SenderThread:29156 [sender.py:send_request():248] send_request: defer
34
+ 2022-02-03 18:07:40,108 INFO SenderThread:29156 [sender.py:send_request_defer():383] handle sender defer: 2
35
+ 2022-02-03 18:07:40,108 INFO SenderThread:29156 [sender.py:transition_state():387] send defer: 3
36
+ 2022-02-03 18:07:40,109 DEBUG HandlerThread:29156 [handler.py:handle_request():130] handle_request: defer
37
+ 2022-02-03 18:07:40,109 INFO HandlerThread:29156 [handler.py:handle_request_defer():147] handle defer: 3
38
+ 2022-02-03 18:07:40,164 DEBUG HandlerThread:29156 [handler.py:handle_request():130] handle_request: poll_exit
39
+ 2022-02-03 18:07:40,168 DEBUG SenderThread:29156 [sender.py:send_request():248] send_request: defer
40
+ 2022-02-03 18:07:40,168 INFO SenderThread:29156 [sender.py:send_request_defer():383] handle sender defer: 3
41
+ 2022-02-03 18:07:40,168 INFO SenderThread:29156 [sender.py:transition_state():387] send defer: 4
42
+ 2022-02-03 18:07:40,169 DEBUG SenderThread:29156 [sender.py:send_request():248] send_request: poll_exit
43
+ 2022-02-03 18:07:40,169 DEBUG HandlerThread:29156 [handler.py:handle_request():130] handle_request: defer
44
+ 2022-02-03 18:07:40,170 INFO HandlerThread:29156 [handler.py:handle_request_defer():147] handle defer: 4
45
+ 2022-02-03 18:07:40,170 DEBUG SenderThread:29156 [sender.py:send_request():248] send_request: defer
46
+ 2022-02-03 18:07:40,170 INFO SenderThread:29156 [sender.py:send_request_defer():383] handle sender defer: 4
47
+ 2022-02-03 18:07:40,170 INFO SenderThread:29156 [sender.py:transition_state():387] send defer: 5
48
+ 2022-02-03 18:07:40,171 DEBUG HandlerThread:29156 [handler.py:handle_request():130] handle_request: defer
49
+ 2022-02-03 18:07:40,171 INFO HandlerThread:29156 [handler.py:handle_request_defer():147] handle defer: 5
50
+ 2022-02-03 18:07:40,171 DEBUG SenderThread:29156 [sender.py:send_request():248] send_request: defer
51
+ 2022-02-03 18:07:40,171 INFO SenderThread:29156 [sender.py:send_request_defer():383] handle sender defer: 5
52
+ 2022-02-03 18:07:40,172 INFO SenderThread:29156 [sender.py:transition_state():387] send defer: 6
53
+ 2022-02-03 18:07:40,172 DEBUG HandlerThread:29156 [handler.py:handle_request():130] handle_request: defer
54
+ 2022-02-03 18:07:40,172 INFO HandlerThread:29156 [handler.py:handle_request_defer():147] handle defer: 6
55
+ 2022-02-03 18:07:40,172 DEBUG SenderThread:29156 [sender.py:send_request():248] send_request: defer
56
+ 2022-02-03 18:07:40,173 INFO SenderThread:29156 [sender.py:send_request_defer():383] handle sender defer: 6
57
+ 2022-02-03 18:07:40,173 INFO SenderThread:29156 [sender.py:transition_state():387] send defer: 7
58
+ 2022-02-03 18:07:40,173 DEBUG HandlerThread:29156 [handler.py:handle_request():130] handle_request: defer
59
+ 2022-02-03 18:07:40,173 INFO HandlerThread:29156 [handler.py:handle_request_defer():147] handle defer: 7
60
+ 2022-02-03 18:07:40,173 DEBUG SenderThread:29156 [sender.py:send_request():248] send_request: defer
61
+ 2022-02-03 18:07:40,174 INFO SenderThread:29156 [sender.py:send_request_defer():383] handle sender defer: 7
62
+ 2022-02-03 18:07:40,174 INFO SenderThread:29156 [sender.py:transition_state():387] send defer: 8
63
+ 2022-02-03 18:07:40,174 DEBUG HandlerThread:29156 [handler.py:handle_request():130] handle_request: defer
64
+ 2022-02-03 18:07:40,174 INFO HandlerThread:29156 [handler.py:handle_request_defer():147] handle defer: 8
65
+ 2022-02-03 18:07:40,175 DEBUG SenderThread:29156 [sender.py:send_request():248] send_request: defer
66
+ 2022-02-03 18:07:40,175 INFO SenderThread:29156 [sender.py:send_request_defer():383] handle sender defer: 8
67
+ 2022-02-03 18:07:40,175 INFO SenderThread:29156 [sender.py:transition_state():387] send defer: 9
68
+ 2022-02-03 18:07:40,176 DEBUG SenderThread:29156 [sender.py:send():234] send: final
69
+ 2022-02-03 18:07:40,176 DEBUG HandlerThread:29156 [handler.py:handle_request():130] handle_request: defer
70
+ 2022-02-03 18:07:40,176 INFO HandlerThread:29156 [handler.py:handle_request_defer():147] handle defer: 9
71
+ 2022-02-03 18:07:40,176 DEBUG SenderThread:29156 [sender.py:send_request():248] send_request: defer
72
+ 2022-02-03 18:07:40,177 INFO SenderThread:29156 [sender.py:send_request_defer():383] handle sender defer: 9
73
+ 2022-02-03 18:07:40,271 DEBUG HandlerThread:29156 [handler.py:handle_request():130] handle_request: poll_exit
74
+ 2022-02-03 18:07:40,272 DEBUG SenderThread:29156 [sender.py:send_request():248] send_request: poll_exit
75
+ 2022-02-03 18:07:40,274 DEBUG HandlerThread:29156 [handler.py:handle_request():130] handle_request: get_summary
76
+ 2022-02-03 18:07:40,380 DEBUG HandlerThread:29156 [handler.py:handle_request():130] handle_request: sampled_history
77
+ 2022-02-03 18:07:40,383 DEBUG HandlerThread:29156 [handler.py:handle_request():130] handle_request: shutdown
78
+ 2022-02-03 18:07:40,384 INFO HandlerThread:29156 [handler.py:finish():731] shutting down handler
79
+ 2022-02-03 18:07:41,180 INFO WriterThread:29156 [datastore.py:close():281] close: /workspace/wav2vec-cv7-1b-ir/wandb/offline-run-20220203_154548-23cvd7o7/run-23cvd7o7.wandb
80
+ 2022-02-03 18:07:41,273 INFO SenderThread:29156 [sender.py:finish():1070] shutting down sender
81
+ 2022-02-03 18:07:41,281 INFO MainThread:29156 [internal.py:handle_exit():77] Internal process exited
wandb/offline-run-20220203_154548-23cvd7o7/logs/debug.log CHANGED
@@ -20,3 +20,15 @@ config: {}
20
  2022-02-03 15:45:50,377 INFO MainThread:27059 [wandb_init.py:init():633] run started, returning control to user process
21
  2022-02-03 15:45:50,383 INFO MainThread:27059 [wandb_run.py:_config_callback():956] config_cb None None {'return_dict': True, 'output_hidden_states': False, 'output_attentions': False, 'torchscript': False, 'torch_dtype': 'float32', 'use_bfloat16': False, 'pruned_heads': {}, 'tie_word_embeddings': True, 'is_encoder_decoder': False, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'max_length': 20, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'chunk_size_feed_forward': 0, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'architectures': ['Wav2Vec2ForPreTraining'], 'finetuning_task': None, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'tokenizer_class': None, 'prefix': None, 'bos_token_id': 1, 'pad_token_id': 31, 'eos_token_id': 2, 'sep_token_id': None, 'decoder_start_token_id': None, 'task_specific_params': None, 'problem_type': None, '_name_or_path': 'facebook/wav2vec2-xls-r-1b', 'transformers_version': '4.16.0.dev0', 'feat_extract_dropout': 0.0, 'model_type': 'wav2vec2', 'num_feat_extract_layers': 7, 'hidden_size': 1280, 'feat_extract_norm': 'layer', 'feat_extract_activation': 'gelu', 'conv_dim': [512, 512, 512, 512, 512, 512, 512], 'conv_stride': [5, 2, 2, 2, 2, 2, 2], 'conv_kernel': [10, 3, 3, 3, 3, 2, 2], 'conv_bias': True, 'num_conv_pos_embeddings': 128, 'num_conv_pos_embedding_groups': 16, 'num_hidden_layers': 48, 'intermediate_size': 5120, 'hidden_act': 'gelu', 'num_attention_heads': 16, 'hidden_dropout': 0.0, 'attention_dropout': 0.0, 'activation_dropout': 0.1, 'feat_proj_dropout': 0.0, 'final_dropout': 0.0, 'layerdrop': 0.0, 'layer_norm_eps': 1e-05, 'initializer_range': 0.02, 'vocab_size': 34, 'do_stable_layer_norm': True, 'use_weighted_layer_sum': False, 'apply_spec_augment': True, 'mask_time_prob': 0.75, 'mask_time_length': 10, 'mask_time_min_masks': 2, 'mask_feature_prob': 0.4, 'mask_feature_length': 64, 'mask_feature_min_masks': 0, 'num_codevectors_per_group': 320, 'num_codevector_groups': 2, 'contrastive_logits_temperature': 0.1, 'feat_quantizer_dropout': 0.0, 'num_negatives': 100, 'codevector_dim': 1024, 'proj_codevector_dim': 1024, 'diversity_loss_weight': 0.1, 'ctc_loss_reduction': 'mean', 'ctc_zero_infinity': False, 'add_adapter': False, 'adapter_kernel_size': 3, 'adapter_stride': 2, 'num_adapter_layers': 3, 'output_hidden_size': 1280, 'classifier_proj_size': 256, 'tdnn_dim': [512, 512, 512, 512, 1500], 'tdnn_kernel': [5, 3, 3, 1, 1], 'tdnn_dilation': [1, 2, 3, 1, 1], 'xvector_output_dim': 512, 'output_dir': './', 'overwrite_output_dir': True, 'do_train': True, 'do_eval': True, 'do_predict': False, 'evaluation_strategy': 'steps', 'prediction_loss_only': False, 'per_device_train_batch_size': 16, 'per_device_eval_batch_size': 16, 'per_gpu_train_batch_size': 'None', 'per_gpu_eval_batch_size': 'None', 'gradient_accumulation_steps': 2, 'eval_accumulation_steps': 'None', 'learning_rate': 2e-05, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 100.0, 'max_steps': -1, 'lr_scheduler_type': 'linear', 'warmup_ratio': 0.0, 'warmup_steps': 1000, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Feb03_15-40-29_job-829e2c87-5501-41ef-ad65-a05e9b64bfd7', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 100, 'logging_nan_inf_filter': True, 'save_strategy': 'steps', 'save_steps': 500, 'save_total_limit': 3, 'save_on_each_node': False, 'no_cuda': False, 'seed': 42, 'bf16': False, 'fp16': True, 'fp16_opt_level': 'O1', 'half_precision_backend': 'amp', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': 'None', 'local_rank': -1, 'xpu_backend': 'None', 'tpu_num_cores': 'None', 'tpu_metrics_debug': False, 'debug': '[]', 'dataloader_drop_last': False, 'eval_steps': 500, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': './', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': 'None', 'load_best_model_at_end': False, 'metric_for_best_model': 'None', 'greater_is_better': 'None', 'ignore_data_skip': False, 'sharded_ddp': '[]', 'deepspeed': 'None', 'label_smoothing_factor': 0.0, 'optim': 'adamw_hf', 'adafactor': False, 'group_by_length': True, 'length_column_name': 'input_length', 'report_to': "['tensorboard', 'wandb']", 'ddp_find_unused_parameters': 'None', 'ddp_bucket_cap_mb': 'None', 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': 'None', 'hub_model_id': 'None', 'hub_strategy': 'every_save', 'hub_token': '<HUB_TOKEN>', 'gradient_checkpointing': True, 'fp16_backend': 'auto', 'push_to_hub_model_id': 'None', 'push_to_hub_organization': 'None', 'push_to_hub_token': '<PUSH_TO_HUB_TOKEN>', '_n_gpu': 1, 'mp_parameters': '', 'train_batch_size': 16, 'eval_batch_size': 16}
22
  2022-02-03 15:45:50,390 INFO MainThread:27059 [wandb_watch.py:watch():43] Watching
 
 
 
 
 
 
 
 
 
 
 
 
 
20
  2022-02-03 15:45:50,377 INFO MainThread:27059 [wandb_init.py:init():633] run started, returning control to user process
21
  2022-02-03 15:45:50,383 INFO MainThread:27059 [wandb_run.py:_config_callback():956] config_cb None None {'return_dict': True, 'output_hidden_states': False, 'output_attentions': False, 'torchscript': False, 'torch_dtype': 'float32', 'use_bfloat16': False, 'pruned_heads': {}, 'tie_word_embeddings': True, 'is_encoder_decoder': False, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'max_length': 20, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'chunk_size_feed_forward': 0, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'architectures': ['Wav2Vec2ForPreTraining'], 'finetuning_task': None, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'tokenizer_class': None, 'prefix': None, 'bos_token_id': 1, 'pad_token_id': 31, 'eos_token_id': 2, 'sep_token_id': None, 'decoder_start_token_id': None, 'task_specific_params': None, 'problem_type': None, '_name_or_path': 'facebook/wav2vec2-xls-r-1b', 'transformers_version': '4.16.0.dev0', 'feat_extract_dropout': 0.0, 'model_type': 'wav2vec2', 'num_feat_extract_layers': 7, 'hidden_size': 1280, 'feat_extract_norm': 'layer', 'feat_extract_activation': 'gelu', 'conv_dim': [512, 512, 512, 512, 512, 512, 512], 'conv_stride': [5, 2, 2, 2, 2, 2, 2], 'conv_kernel': [10, 3, 3, 3, 3, 2, 2], 'conv_bias': True, 'num_conv_pos_embeddings': 128, 'num_conv_pos_embedding_groups': 16, 'num_hidden_layers': 48, 'intermediate_size': 5120, 'hidden_act': 'gelu', 'num_attention_heads': 16, 'hidden_dropout': 0.0, 'attention_dropout': 0.0, 'activation_dropout': 0.1, 'feat_proj_dropout': 0.0, 'final_dropout': 0.0, 'layerdrop': 0.0, 'layer_norm_eps': 1e-05, 'initializer_range': 0.02, 'vocab_size': 34, 'do_stable_layer_norm': True, 'use_weighted_layer_sum': False, 'apply_spec_augment': True, 'mask_time_prob': 0.75, 'mask_time_length': 10, 'mask_time_min_masks': 2, 'mask_feature_prob': 0.4, 'mask_feature_length': 64, 'mask_feature_min_masks': 0, 'num_codevectors_per_group': 320, 'num_codevector_groups': 2, 'contrastive_logits_temperature': 0.1, 'feat_quantizer_dropout': 0.0, 'num_negatives': 100, 'codevector_dim': 1024, 'proj_codevector_dim': 1024, 'diversity_loss_weight': 0.1, 'ctc_loss_reduction': 'mean', 'ctc_zero_infinity': False, 'add_adapter': False, 'adapter_kernel_size': 3, 'adapter_stride': 2, 'num_adapter_layers': 3, 'output_hidden_size': 1280, 'classifier_proj_size': 256, 'tdnn_dim': [512, 512, 512, 512, 1500], 'tdnn_kernel': [5, 3, 3, 1, 1], 'tdnn_dilation': [1, 2, 3, 1, 1], 'xvector_output_dim': 512, 'output_dir': './', 'overwrite_output_dir': True, 'do_train': True, 'do_eval': True, 'do_predict': False, 'evaluation_strategy': 'steps', 'prediction_loss_only': False, 'per_device_train_batch_size': 16, 'per_device_eval_batch_size': 16, 'per_gpu_train_batch_size': 'None', 'per_gpu_eval_batch_size': 'None', 'gradient_accumulation_steps': 2, 'eval_accumulation_steps': 'None', 'learning_rate': 2e-05, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 100.0, 'max_steps': -1, 'lr_scheduler_type': 'linear', 'warmup_ratio': 0.0, 'warmup_steps': 1000, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Feb03_15-40-29_job-829e2c87-5501-41ef-ad65-a05e9b64bfd7', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 100, 'logging_nan_inf_filter': True, 'save_strategy': 'steps', 'save_steps': 500, 'save_total_limit': 3, 'save_on_each_node': False, 'no_cuda': False, 'seed': 42, 'bf16': False, 'fp16': True, 'fp16_opt_level': 'O1', 'half_precision_backend': 'amp', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': 'None', 'local_rank': -1, 'xpu_backend': 'None', 'tpu_num_cores': 'None', 'tpu_metrics_debug': False, 'debug': '[]', 'dataloader_drop_last': False, 'eval_steps': 500, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': './', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': 'None', 'load_best_model_at_end': False, 'metric_for_best_model': 'None', 'greater_is_better': 'None', 'ignore_data_skip': False, 'sharded_ddp': '[]', 'deepspeed': 'None', 'label_smoothing_factor': 0.0, 'optim': 'adamw_hf', 'adafactor': False, 'group_by_length': True, 'length_column_name': 'input_length', 'report_to': "['tensorboard', 'wandb']", 'ddp_find_unused_parameters': 'None', 'ddp_bucket_cap_mb': 'None', 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': 'None', 'hub_model_id': 'None', 'hub_strategy': 'every_save', 'hub_token': '<HUB_TOKEN>', 'gradient_checkpointing': True, 'fp16_backend': 'auto', 'push_to_hub_model_id': 'None', 'push_to_hub_organization': 'None', 'push_to_hub_token': '<PUSH_TO_HUB_TOKEN>', '_n_gpu': 1, 'mp_parameters': '', 'train_batch_size': 16, 'eval_batch_size': 16}
22
  2022-02-03 15:45:50,390 INFO MainThread:27059 [wandb_watch.py:watch():43] Watching
23
+ 2022-02-03 18:07:37,273 INFO MainThread:27059 [wandb_run.py:_atexit_cleanup():1780] got exitcode: 0
24
+ 2022-02-03 18:07:37,278 INFO MainThread:27059 [wandb_run.py:_restore():1752] restore
25
+ 2022-02-03 18:07:40,034 INFO MainThread:27059 [wandb_run.py:_wait_for_finish():1912] got exit ret:
26
+ 2022-02-03 18:07:40,170 INFO MainThread:27059 [wandb_run.py:_wait_for_finish():1912] got exit ret:
27
+ 2022-02-03 18:07:40,273 INFO MainThread:27059 [wandb_run.py:_wait_for_finish():1912] got exit ret: done: true
28
+ exit_result {
29
+ }
30
+ local_info {
31
+ }
32
+
33
+ 2022-02-03 18:07:41,456 INFO MainThread:27059 [wandb_run.py:_append_history():2130] rendering history
34
+ 2022-02-03 18:07:41,457 INFO MainThread:27059 [wandb_run.py:_append_summary():2085] rendering summary
wandb/offline-run-20220203_154548-23cvd7o7/run-23cvd7o7.wandb CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:9bb35590405d0ad8e22d52e1228c23ec6aebc1d540d9016e15aca2d4e4843aef
3
- size 41575854
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5efdfd94d603c58a50259a019d4d2f809277b08b62ac0ad0f508b6fe193e2755
3
+ size 42863793