pere commited on
Commit
1ccf900
1 Parent(s): 42b3087
Files changed (36) hide show
  1. .gitattributes +2 -0
  2. add_kenlm.py +34 -0
  3. added_tokens.json +1 -0
  4. alphabet.json +1 -0
  5. config.json +107 -0
  6. eval.py +151 -0
  7. grid.csv +101 -0
  8. grid.py +34 -0
  9. language_model/5gram.bin +3 -0
  10. language_model/attrs.json +1 -0
  11. language_model/unigrams.txt +3 -0
  12. log_NbAiLab_NPSC_16K_mp3_bokmaal_test_eval_results.txt +3 -0
  13. log_NbAiLab_NPSC_16K_mp3_bokmaal_test_eval_results_no_lang_model.txt +3 -0
  14. log_NbAiLab_NPSC_16K_mp3_bokmaal_test_predictions.txt +3 -0
  15. log_NbAiLab_NPSC_16K_mp3_bokmaal_test_predictions_no_lang_model.txt +3 -0
  16. log_NbAiLab_NPSC_16K_mp3_bokmaal_test_targets.txt +3 -0
  17. log_NbAiLab_NPSC_16K_mp3_bokmaal_test_targets_no_lang_model.txt +3 -0
  18. preprocessor_config.json +10 -0
  19. pytorch_model.bin +3 -0
  20. run.sh +40 -0
  21. runs/Feb06_21-00-20_job-c93f32d8-97c5-48e7-b5ec-c6c950f627ba/1644181327.7862043/events.out.tfevents.1644181327.job-c93f32d8-97c5-48e7-b5ec-c6c950f627ba.1233455.1 +3 -0
  22. runs/Feb06_21-00-20_job-c93f32d8-97c5-48e7-b5ec-c6c950f627ba/events.out.tfevents.1644181327.job-c93f32d8-97c5-48e7-b5ec-c6c950f627ba.1233455.0 +3 -0
  23. runs/Feb06_21-06-35_job-c93f32d8-97c5-48e7-b5ec-c6c950f627ba/1644181697.767686/events.out.tfevents.1644181697.job-c93f32d8-97c5-48e7-b5ec-c6c950f627ba.1235981.1 +3 -0
  24. runs/Feb06_21-06-35_job-c93f32d8-97c5-48e7-b5ec-c6c950f627ba/events.out.tfevents.1644181697.job-c93f32d8-97c5-48e7-b5ec-c6c950f627ba.1235981.0 +3 -0
  25. runs/Feb07_20-59-13_dante/1644264797.365929/events.out.tfevents.1644264797.dante.3536313.1 +3 -0
  26. runs/Feb07_20-59-13_dante/events.out.tfevents.1644264797.dante.3536313.0 +3 -0
  27. runs/Feb07_22-48-25_dante/1644270591.175515/events.out.tfevents.1644270591.dante.3603912.1 +3 -0
  28. runs/Feb07_22-48-25_dante/events.out.tfevents.1644270591.dante.3603912.0 +3 -0
  29. runs/Feb07_22-48-25_dante/events.out.tfevents.1644597285.dante.3603912.2 +3 -0
  30. runs/Feb11_17-40-48_dante/1644598421.1311183/events.out.tfevents.1644598421.dante.2344752.1 +3 -0
  31. runs/Feb11_17-40-48_dante/events.out.tfevents.1644598421.dante.2344752.0 +3 -0
  32. runs/Feb11_17-40-48_dante/events.out.tfevents.1644599097.dante.2344752.2 +3 -0
  33. special_tokens_map.json +1 -0
  34. tokenizer_config.json +1 -0
  35. training_args.bin +3 -0
  36. vocab.json +1 -0
.gitattributes CHANGED
@@ -25,3 +25,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
25
  *.zip filter=lfs diff=lfs merge=lfs -text
26
  *.zstandard filter=lfs diff=lfs merge=lfs -text
27
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
 
25
  *.zip filter=lfs diff=lfs merge=lfs -text
26
  *.zstandard filter=lfs diff=lfs merge=lfs -text
27
  *tfevents* filter=lfs diff=lfs merge=lfs -text
28
+ *.txt filter=lfs diff=lfs merge=lfs -text
29
+ *unigram*.* filter=lfs diff=lfs merge=lfs -text
add_kenlm.py ADDED
@@ -0,0 +1,34 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import argparse
2
+ from transformers import AutoProcessor
3
+ from transformers import Wav2Vec2ProcessorWithLM
4
+ from pyctcdecode import build_ctcdecoder
5
+
6
+
7
+ def main(args):
8
+ processor = AutoProcessor.from_pretrained(args.model_name_or_path)
9
+ vocab_dict = processor.tokenizer.get_vocab()
10
+ sorted_vocab_dict = {
11
+ k.lower(): v for k, v in sorted(vocab_dict.items(), key=lambda item: item[1])
12
+ }
13
+ decoder = build_ctcdecoder(
14
+ labels=list(sorted_vocab_dict.keys()),
15
+ kenlm_model_path=args.kenlm_model_path,
16
+ )
17
+ processor_with_lm = Wav2Vec2ProcessorWithLM(
18
+ feature_extractor=processor.feature_extractor,
19
+ tokenizer=processor.tokenizer,
20
+ decoder=decoder,
21
+ )
22
+ processor_with_lm.save_pretrained(args.model_name_or_path)
23
+ print(f"Run: ~/bin/build_binary language_model/*.arpa language_model/5gram.bin -T $(pwd) && rm language_model/*.arpa")
24
+
25
+ def parse_args():
26
+ parser = argparse.ArgumentParser()
27
+ parser.add_argument('--model_name_or_path', default="./", help='Model name or path. Defaults to ./')
28
+ parser.add_argument('--kenlm_model_path', required=True, help='Path to KenLM arpa file.')
29
+ args = parser.parse_args()
30
+ return args
31
+
32
+ if __name__ == "__main__":
33
+ args = parse_args()
34
+ main(args)
added_tokens.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"<s>": 32, "</s>": 33}
alphabet.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"labels": [" ", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z", "\u00e5", "\u00e6", "\u00f8", "\u2047", "", "<s>", "</s>"], "is_bpe": false}
config.json ADDED
@@ -0,0 +1,107 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "./",
3
+ "activation_dropout": 0.055,
4
+ "adapter_kernel_size": 3,
5
+ "adapter_stride": 2,
6
+ "add_adapter": false,
7
+ "apply_spec_augment": true,
8
+ "architectures": [
9
+ "Wav2Vec2ForCTC"
10
+ ],
11
+ "attention_dropout": 0.094,
12
+ "bos_token_id": 1,
13
+ "classifier_proj_size": 256,
14
+ "codevector_dim": 1024,
15
+ "contrastive_logits_temperature": 0.1,
16
+ "conv_bias": true,
17
+ "conv_dim": [
18
+ 512,
19
+ 512,
20
+ 512,
21
+ 512,
22
+ 512,
23
+ 512,
24
+ 512
25
+ ],
26
+ "conv_kernel": [
27
+ 10,
28
+ 3,
29
+ 3,
30
+ 3,
31
+ 3,
32
+ 2,
33
+ 2
34
+ ],
35
+ "conv_stride": [
36
+ 5,
37
+ 2,
38
+ 2,
39
+ 2,
40
+ 2,
41
+ 2,
42
+ 2
43
+ ],
44
+ "ctc_loss_reduction": "mean",
45
+ "ctc_zero_infinity": true,
46
+ "diversity_loss_weight": 0.1,
47
+ "do_stable_layer_norm": true,
48
+ "eos_token_id": 2,
49
+ "feat_extract_activation": "gelu",
50
+ "feat_extract_dropout": 0.0,
51
+ "feat_extract_norm": "layer",
52
+ "feat_proj_dropout": 0.04,
53
+ "feat_quantizer_dropout": 0.0,
54
+ "final_dropout": 0.0,
55
+ "hidden_act": "gelu",
56
+ "hidden_dropout": 0.047,
57
+ "hidden_size": 1280,
58
+ "initializer_range": 0.02,
59
+ "intermediate_size": 5120,
60
+ "layer_norm_eps": 1e-05,
61
+ "layerdrop": 0.041,
62
+ "mask_feature_length": 64,
63
+ "mask_feature_min_masks": 0,
64
+ "mask_feature_prob": 0.25,
65
+ "mask_time_length": 10,
66
+ "mask_time_min_masks": 2,
67
+ "mask_time_prob": 0.082,
68
+ "model_type": "wav2vec2",
69
+ "num_adapter_layers": 3,
70
+ "num_attention_heads": 16,
71
+ "num_codevector_groups": 2,
72
+ "num_codevectors_per_group": 320,
73
+ "num_conv_pos_embedding_groups": 16,
74
+ "num_conv_pos_embeddings": 128,
75
+ "num_feat_extract_layers": 7,
76
+ "num_hidden_layers": 48,
77
+ "num_negatives": 100,
78
+ "output_hidden_size": 1280,
79
+ "pad_token_id": 31,
80
+ "proj_codevector_dim": 1024,
81
+ "tdnn_dilation": [
82
+ 1,
83
+ 2,
84
+ 3,
85
+ 1,
86
+ 1
87
+ ],
88
+ "tdnn_dim": [
89
+ 512,
90
+ 512,
91
+ 512,
92
+ 512,
93
+ 1500
94
+ ],
95
+ "tdnn_kernel": [
96
+ 5,
97
+ 3,
98
+ 3,
99
+ 1,
100
+ 1
101
+ ],
102
+ "torch_dtype": "float32",
103
+ "transformers_version": "4.17.0.dev0",
104
+ "use_weighted_layer_sum": false,
105
+ "vocab_size": 34,
106
+ "xvector_output_dim": 512
107
+ }
eval.py ADDED
@@ -0,0 +1,151 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ import argparse
3
+ import re
4
+ from typing import Dict
5
+
6
+ import torch
7
+ from datasets import Audio, Dataset, load_dataset, load_metric
8
+
9
+ from transformers import AutoFeatureExtractor, pipeline
10
+
11
+
12
+ def log_results(result: Dataset, args: Dict[str, str]):
13
+ """DO NOT CHANGE. This function computes and logs the result metrics."""
14
+
15
+ log_outputs = args.log_outputs
16
+ dataset_id = "_".join(args.dataset.split("/") + [args.config, args.split])
17
+
18
+ # load metric
19
+ wer = load_metric("wer")
20
+ cer = load_metric("cer")
21
+
22
+ # compute metrics
23
+ wer_result = wer.compute(references=result["target"], predictions=result["prediction"])
24
+ cer_result = cer.compute(references=result["target"], predictions=result["prediction"])
25
+
26
+ # print & log results
27
+ result_str = f"WER: {wer_result}\n" f"CER: {cer_result}"
28
+ print(result_str)
29
+
30
+ with open(f"{dataset_id}_eval_results.txt", "w") as f:
31
+ f.write(result_str)
32
+
33
+ # log all results in text file. Possibly interesting for analysis
34
+ if log_outputs is not None:
35
+ pred_file = f"log_{dataset_id}_predictions.txt"
36
+ target_file = f"log_{dataset_id}_targets.txt"
37
+
38
+ with open(pred_file, "w") as p, open(target_file, "w") as t:
39
+
40
+ # mapping function to write output
41
+ def write_to_file(batch, i):
42
+ p.write(f"{i}" + "\n")
43
+ p.write(batch["prediction"] + "\n")
44
+ t.write(f"{i}" + "\n")
45
+ t.write(batch["target"] + "\n")
46
+
47
+ result.map(write_to_file, with_indices=True)
48
+
49
+
50
+ def normalize_text(text: str) -> str:
51
+ """DO ADAPT FOR YOUR USE CASE. this function normalizes the target text."""
52
+
53
+ chars_to_ignore_regex = '[\,\?\.\!\-\;\:\"\“\%\‘\”\�\'\–\_\\\+\#\/]' # noqa: W605 IMPORTANT: this should correspond to the chars that were ignored during training
54
+
55
+ text = re.sub(chars_to_ignore_regex, "", text.lower()) + " "
56
+ text = re.sub('[áàâ]', 'a', text)
57
+ text = re.sub('[ä]', 'æ', text)
58
+ text = re.sub('[éèëê]', 'e', text)
59
+ text = re.sub('[íìïî]', 'i', text)
60
+ text = re.sub('[óòöô]', 'o', text)
61
+ text = re.sub('[ö]', 'ø', text)
62
+ text = re.sub('[ç]', 'c', text)
63
+ text = re.sub('[úùüû]', 'u', text)
64
+ text = re.sub('\s', ' ', text)
65
+ text = re.sub('<ee>', 'eee', text)
66
+ text = re.sub('<qq>', 'qqq', text)
67
+ text = re.sub('<mm>', 'mmm', text)
68
+ text = re.sub('<inaudible>', 'xxx', text)
69
+ text = re.sub('[<>]', '', text)
70
+
71
+ # # In addition, we can normalize the target text, e.g. removing new lines characters etc...
72
+ # # note that order is important here!
73
+ # token_sequences_to_ignore = ["\n\n", "\n", " ", " "]
74
+
75
+ # for t in token_sequences_to_ignore:
76
+ # text = " ".join(text.split(t))
77
+
78
+ return text
79
+
80
+
81
+ def main(args):
82
+ # load dataset
83
+ dataset = load_dataset(args.dataset, args.config, split=args.split, use_auth_token=True)
84
+
85
+ # for testing: only process the first two examples as a test
86
+ # dataset = dataset.select(range(10))
87
+
88
+ # load processor
89
+ feature_extractor = AutoFeatureExtractor.from_pretrained(args.model_id)
90
+ sampling_rate = feature_extractor.sampling_rate
91
+
92
+ # resample audio
93
+ dataset = dataset.cast_column("audio", Audio(sampling_rate=sampling_rate))
94
+
95
+ # load eval pipeline
96
+ if args.device is None:
97
+ args.device = 0 if torch.cuda.is_available() else -1
98
+ asr = pipeline("automatic-speech-recognition", model=args.model_id, device=args.device)
99
+
100
+ # map function to decode audio
101
+ def map_to_pred(batch):
102
+ prediction = asr(
103
+ batch["audio"]["array"], chunk_length_s=args.chunk_length_s, stride_length_s=args.stride_length_s
104
+ )
105
+
106
+ batch["prediction"] = prediction["text"]
107
+ batch["target"] = normalize_text(batch["text"])
108
+ return batch
109
+
110
+ # run inference on all examples
111
+ result = dataset.map(map_to_pred, remove_columns=dataset.column_names)
112
+
113
+ # compute and log_results
114
+ # do not change function below
115
+ log_results(result, args)
116
+
117
+
118
+ if __name__ == "__main__":
119
+ parser = argparse.ArgumentParser()
120
+
121
+ parser.add_argument(
122
+ "--model_id", type=str, required=True, help="Model identifier. Should be loadable with 🤗 Transformers"
123
+ )
124
+ parser.add_argument(
125
+ "--dataset",
126
+ type=str,
127
+ required=True,
128
+ help="Dataset name to evaluate the `model_id`. Should be loadable with 🤗 Datasets",
129
+ )
130
+ parser.add_argument(
131
+ "--config", type=str, required=True, help="Config of the dataset. *E.g.* `'en'` for Common Voice"
132
+ )
133
+ parser.add_argument("--split", type=str, required=True, help="Split of the dataset. *E.g.* `'test'`")
134
+ parser.add_argument(
135
+ "--chunk_length_s", type=float, default=None, help="Chunk length in seconds. Defaults to 5 seconds."
136
+ )
137
+ parser.add_argument(
138
+ "--stride_length_s", type=float, default=None, help="Stride of the audio chunks. Defaults to 1 second."
139
+ )
140
+ parser.add_argument(
141
+ "--log_outputs", action="store_true", help="If defined, write outputs to log file for analysis."
142
+ )
143
+ parser.add_argument(
144
+ "--device",
145
+ type=int,
146
+ default=None,
147
+ help="The device to run the pipeline on. -1 for CPU (default), 0 for the first GPU and so on.",
148
+ )
149
+ args = parser.parse_args()
150
+
151
+ main(args)
grid.csv ADDED
@@ -0,0 +1,101 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ alpha,beta,wer,cer
2
+ 0.001,0.001,0.07689050968468251,0.026711121521565424
3
+ 0.001,0.01,0.07686164326883294,0.026699492822600875
4
+ 0.001,0.1,0.07693862037776517,0.026702815308019316
5
+ 0.001,0.25,0.07732350592242632,0.026749330103877506
6
+ 0.001,0.5,0.07791045637803458,0.026812457326827906
7
+ 0.001,0.75,0.07878607099213872,0.02695532419982092
8
+ 0.001,1,0.07969055202209244,0.02708490113114016
9
+ 0.001,1.5,0.08199986529005937,0.02737894109067229
10
+ 0.001,2,0.08457859843928911,0.027772655612757678
11
+ 0.001,3,0.09057319079738663,0.028741160112233557
12
+ 0.01,0.001,0.07580320802101476,0.026571577133990852
13
+ 0.01,0.01,0.07583207443686434,0.026571577133990852
14
+ 0.01,0.1,0.07596678437749574,0.026606463230884496
15
+ 0.01,0.25,0.07640940275385608,0.026649655541324244
16
+ 0.01,0.5,0.07688088754606599,0.02667623542467178
17
+ 0.01,0.75,0.07770839146708748,0.02679916738515414
18
+ 0.01,1,0.0786513610515073,0.026963630413367023
19
+ 0.01,1.5,0.08097029645809077,0.027274282799991362
20
+ 0.01,2,0.08339507538945606,0.027613176312672457
21
+ 0.01,3,0.08929344636138828,0.02853018228816248
22
+ 0.1,0.001,0.0702319897620445,0.025917047506557756
23
+ 0.1,0.01,0.07021274548481145,0.025907080050302428
24
+ 0.1,0.1,0.07033783328682633,0.025920369991976198
25
+ 0.1,0.25,0.07057838675223956,0.025960239816997504
26
+ 0.1,0.5,0.0711268486533817,0.026010077098274135
27
+ 0.1,0.75,0.07168493269314037,0.026078188049352198
28
+ 0.1,1,0.07225263887151559,0.026169556398359356
29
+ 0.1,1.5,0.07360936041644615,0.026289165873423274
30
+ 0.1,2,0.07566849808038334,0.0265516422214802
31
+ 0.1,3,0.08056616663619656,0.027264315343736033
32
+ 0.25,0.001,0.06741270314740154,0.025528316712600026
33
+ 0.25,0.01,0.06744156956325113,0.025539945411564575
34
+ 0.25,0.1,0.06741270314740154,0.025521671741763143
35
+ 0.25,0.25,0.06753779094941642,0.025523332984472364
36
+ 0.25,0.5,0.06765325661281477,0.025533300440727692
37
+ 0.25,0.75,0.06790343221684451,0.02557649275116744
38
+ 0.25,1,0.06817285209810732,0.025568186537621333
39
+ 0.25,1.5,0.06890413463296352,0.025651248673082386
40
+ 0.25,2,0.0696642835836693,0.025696102226231355
41
+ 0.25,3,0.07246432592107922,0.02611141290353662
42
+ 0.5,0.001,0.06591164952322304,0.025593105178259648
43
+ 0.5,0.01,0.06596938235492221,0.025601411391805756
44
+ 0.5,0.1,0.06597900449353873,0.02559808890638731
45
+ 0.5,0.25,0.06613295871140319,0.02557649275116744
46
+ 0.5,0.5,0.06615220298863625,0.025506720557380153
47
+ 0.5,0.75,0.06619069154310236,0.0254867856448695
48
+ 0.5,1,0.06632540148373378,0.025505059314670932
49
+ 0.5,1.5,0.0667006648897784,0.025520010499053922
50
+ 0.5,2,0.06719139395922137,0.025490108130287945
51
+ 0.5,3,0.06871169186063295,0.02567118358559304
52
+ 0.75,0.001,0.06652746639468088,0.025825679157550598
53
+ 0.75,0.01,0.06651784425606436,0.025825679157550598
54
+ 0.75,0.1,0.06646011142436518,0.02581737294400449
55
+ 0.75,0.25,0.0663350236223503,0.025792454303366175
56
+ 0.75,0.5,0.06650822211744782,0.025809066730458387
57
+ 0.75,0.75,0.06641200073128253,0.025799099274203062
58
+ 0.75,1,0.0663638900381999,0.025755906963763314
59
+ 0.75,1.5,0.06648897784021476,0.025729327080415774
60
+ 0.75,2,0.06680650841456022,0.025714375896032787
61
+ 0.75,3,0.0673645924543189,0.025641281216827058
62
+ 1,0.001,0.06927939803900815,0.02653502979438799
63
+ 1,0.01,0.06926015376177509,0.02653336855167877
64
+ 1,0.1,0.06918317665284286,0.026548319736061757
65
+ 1,0.25,0.06900997815774534,0.02645861262976382
66
+ 1,0.5,0.068730936137866,0.026368905523465883
67
+ 1,0.75,0.06865395902893377,0.026353954339082896
68
+ 1,1,0.06855773764276848,0.02630411705780626
69
+ 1,1.5,0.06833642845458832,0.02620776498067144
70
+ 1,2,0.06804776429609245,0.026071543078515315
71
+ 1,3,0.06796116504854369,0.025993464671181923
72
+ 1.5,0.001,0.07664033408065277,0.027842427806544966
73
+ 1.5,0.01,0.07664995621926929,0.02785073402009107
74
+ 1.5,0.1,0.07654411269448748,0.02783910532112652
75
+ 1.5,0.25,0.07627469281322467,0.027794251767977552
76
+ 1.5,0.5,0.07588980726856351,0.027689593477296628
77
+ 1.5,0.75,0.07547605530805276,0.027579951458488038
78
+ 1.5,1,0.07511041404062467,0.02752014672095608
79
+ 1.5,1.5,0.07450421930778335,0.027398876003182943
80
+ 1.5,2,0.07381142532739327,0.027354022450033974
81
+ 1.5,3,0.07257016944586103,0.027058321247792623
82
+ 2,0.001,0.08493461756810068,0.02921793676978
83
+ 2,0.01,0.08491537329086762,0.02921627552707078
84
+ 2,0.1,0.08472293051853705,0.029146503333283495
85
+ 2,0.25,0.08439577780557507,0.02908835983846076
86
+ 2,0.5,0.08399164798368085,0.029020248887382697
87
+ 2,0.75,0.08366449527071887,0.028968750363396842
88
+ 2,1,0.0831737662012759,0.02886741455813436
89
+ 2,1.5,0.08208646453760812,0.028692984073666147
90
+ 2,2,0.0808452086560759,0.028500279919396503
91
+ 2,3,0.07907473515063458,0.028187966290062944
92
+ 3,0.001,0.09346945452096182,0.03065657295596544
93
+ 3,0.01,0.0934598323823453,0.03065823419867466
94
+ 3,0.1,0.09336361099618001,0.030631654315327123
95
+ 3,0.25,0.09322890105554861,0.030629993072617902
96
+ 3,0.5,0.0928632597881205,0.03056188212153984
97
+ 3,0.75,0.09247837424345935,0.030490448685043334
98
+ 3,1,0.0922955536097453,0.030488787442334114
99
+ 3,1.5,0.09158351535212217,0.030339275598504217
100
+ 3,2,0.09083298854003291,0.030241262278660176
101
+ 3,3,0.08947626699510233,0.029978785930603248
grid.py ADDED
@@ -0,0 +1,34 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python
2
+ from argparse import ArgumentDefaultsHelpFormatter
3
+ from collections import namedtuple
4
+ from functools import partialmethod
5
+ import json
6
+
7
+ from tqdm import tqdm
8
+ from eval import main
9
+
10
+
11
+ # tqdm.__init__ = partialmethod(tqdm.__init__, disable=True)
12
+
13
+ Args = namedtuple("Args", "model_id dataset config split log_outputs chunk_length_s stride_length_s device")
14
+ args = Args("./", "NbAiLab/NPSC", "16K_mp3_bokmaal", "test", True, None, None, 0)
15
+ with open("grid.csv", "w") as grid:
16
+ grid.write("alpha,beta,wer,cer")
17
+ for alpha in [0.001, 0.01, 0.1, 0.25, 0.5, 0.75, 1, 1.5, 2, 3]:
18
+ for beta in [0.001, 0.01, 0.1, 0.25, 0.5, 0.75, 1, 1.5, 2, 3]:
19
+ with open("./language_model/attrs.json", "r") as attrs_file:
20
+ attrs = json.load(attrs_file)
21
+ attrs["alpha"] = alpha
22
+ attrs["beta"] = beta
23
+ with open("./language_model/attrs.json", "w") as attrs_file:
24
+ json.dump(attrs, attrs_file)
25
+ print(f"alpha = {alpha}, beta = {beta}")
26
+ main(args)
27
+ with open("NbAiLab_NPSC_16K_mp3_bokmaal_test_eval_results.txt") as results_file:
28
+ results = results_file.read().strip().split("\n")
29
+ wer = float(results[0][5:])
30
+ cer = float(results[1][5:])
31
+ grid.write(f"\n{alpha},{beta},{wer},{cer}")
32
+ print("--------------")
33
+
34
+
language_model/5gram.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7b41c24c63f2f0585bea83666369593f3b3e6d047f327a90f36ebca2c35ef0ff
3
+ size 4243671427
language_model/attrs.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"alpha": 0.5, "beta": 0.0, "unk_score_offset": -10.0, "score_boundary": true}
language_model/unigrams.txt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ac3e71ca49838ca355df6fdcb8d89344a5a9bf9e1a76587cdf5df1367c19b9a9
3
+ size 16759269
log_NbAiLab_NPSC_16K_mp3_bokmaal_test_eval_results.txt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:658588cf79099ab697bc9ff0c5d53fe956e3c9fdf385b4715129d4148be99495
3
+ size 49
log_NbAiLab_NPSC_16K_mp3_bokmaal_test_eval_results_no_lang_model.txt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fcb02a32f3393edc8aaa2ec5dcb5c2f4d8c7a1471d4a2d553dd85fed00579e54
3
+ size 50
log_NbAiLab_NPSC_16K_mp3_bokmaal_test_predictions.txt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7ef742d63f7ba1dffc7c27f02beff74b79df5b904e3ba56f4df7c0c41bb6719e
3
+ size 648798
log_NbAiLab_NPSC_16K_mp3_bokmaal_test_predictions_no_lang_model.txt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:087bb268470c79b3b5e102928cbce002bfcd6ec791db8b0c0600a775bcc7a90f
3
+ size 649987
log_NbAiLab_NPSC_16K_mp3_bokmaal_test_targets.txt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d7e27a71ec76c0ded0b0cf852b9d42c620e1c5863bcf9a06fcbbbbd924e9d1b9
3
+ size 654671
log_NbAiLab_NPSC_16K_mp3_bokmaal_test_targets_no_lang_model.txt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d7e27a71ec76c0ded0b0cf852b9d42c620e1c5863bcf9a06fcbbbbd924e9d1b9
3
+ size 654671
preprocessor_config.json ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "do_normalize": true,
3
+ "feature_extractor_type": "Wav2Vec2FeatureExtractor",
4
+ "feature_size": 1,
5
+ "padding_side": "right",
6
+ "padding_value": 0,
7
+ "processor_class": "Wav2Vec2ProcessorWithLM",
8
+ "return_attention_mask": true,
9
+ "sampling_rate": 16000
10
+ }
pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:01edd44641353c2f34c79eb346e6153816c4f35f49334bba6a1a108cb99f9ca5
3
+ size 3850486961
run.sh ADDED
@@ -0,0 +1,40 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ python run_speech_recognition_ctc.py \
2
+ --dataset_name="NbAiLab/NPSC" \
3
+ --model_name_or_path="facebook/wav2vec2-xls-r-1b" \
4
+ --dataset_config_name="16K_mp3_bokmaal" \
5
+ --output_dir="./" \
6
+ --overwrite_output_dir \
7
+ --num_train_epochs="40" \
8
+ --per_device_train_batch_size="12" \
9
+ --per_device_eval_batch_size="12" \
10
+ --gradient_accumulation_steps="2" \
11
+ --learning_rate="2e-5" \
12
+ --warmup_steps="2000" \
13
+ --length_column_name="input_length" \
14
+ --evaluation_strategy="steps" \
15
+ --text_column_name="text" \
16
+ --save_steps="500" \
17
+ --eval_steps="500" \
18
+ --logging_steps="100" \
19
+ --layerdrop="0.041" \
20
+ --attention_dropout="0.094" \
21
+ --activation_dropout="0.055" \
22
+ --hidden_dropout="0.047" \
23
+ --save_total_limit="3" \
24
+ --freeze_feature_encoder \
25
+ --feat_proj_dropout="0.04" \
26
+ --mask_time_prob="0.082" \
27
+ --mask_time_length="10" \
28
+ --mask_feature_prob="0.25" \
29
+ --mask_feature_length="64" \
30
+ --gradient_checkpointing \
31
+ --min_duration_in_seconds="0.5" \
32
+ --max_duration_in_seconds="30.0" \
33
+ --ctc_zero_infinity=True \
34
+ --use_auth_token \
35
+ --seed="42" \
36
+ --fp16 \
37
+ --group_by_length \
38
+ --do_train --do_eval \
39
+ --push_to_hub \
40
+ --preprocessing_num_workers="16"
runs/Feb06_21-00-20_job-c93f32d8-97c5-48e7-b5ec-c6c950f627ba/1644181327.7862043/events.out.tfevents.1644181327.job-c93f32d8-97c5-48e7-b5ec-c6c950f627ba.1233455.1 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e42ad114d53443aa03f4085977f0278259c263c0e55643cbaa71e38f85416ac1
3
+ size 4776
runs/Feb06_21-00-20_job-c93f32d8-97c5-48e7-b5ec-c6c950f627ba/events.out.tfevents.1644181327.job-c93f32d8-97c5-48e7-b5ec-c6c950f627ba.1233455.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e0ebdea7a8a0435aad29811dd38d7788869adc2fe05cfc79439e65e4294566da
3
+ size 4716
runs/Feb06_21-06-35_job-c93f32d8-97c5-48e7-b5ec-c6c950f627ba/1644181697.767686/events.out.tfevents.1644181697.job-c93f32d8-97c5-48e7-b5ec-c6c950f627ba.1235981.1 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ce7b1a25c6e7bbf6d2ed9adbdc66ff7df07b371ba42014692b7c814df923ebb2
3
+ size 4776
runs/Feb06_21-06-35_job-c93f32d8-97c5-48e7-b5ec-c6c950f627ba/events.out.tfevents.1644181697.job-c93f32d8-97c5-48e7-b5ec-c6c950f627ba.1235981.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fe271cf8605594bd4a0e1505b702d0fc514760349acaf2eb5e7c2e3b80cf52db
3
+ size 35597
runs/Feb07_20-59-13_dante/1644264797.365929/events.out.tfevents.1644264797.dante.3536313.1 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ffaa426d205ad6d41b30a030b281139364685c886039e88dcc284cb2a3a0f617
3
+ size 4741
runs/Feb07_20-59-13_dante/events.out.tfevents.1644264797.dante.3536313.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5df3d4e4263a791a1b8a910fc20582aab5f452f23785aabf3b0127d3241bef44
3
+ size 7325
runs/Feb07_22-48-25_dante/1644270591.175515/events.out.tfevents.1644270591.dante.3603912.1 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:43d94d5a4ce2f9a3c0470505f7982a3d6ed9f0edf871889542ad0492f815bb19
3
+ size 4741
runs/Feb07_22-48-25_dante/events.out.tfevents.1644270591.dante.3603912.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cf808b11fe3fbb5ab86b38a08a766a78cb23b58d81fc7c716fd0768ed64b102b
3
+ size 162164
runs/Feb07_22-48-25_dante/events.out.tfevents.1644597285.dante.3603912.2 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:02ca0e871dca33ce2c29f8fcff4c737976489a20f3935f2766148a74c94ca749
3
+ size 364
runs/Feb11_17-40-48_dante/1644598421.1311183/events.out.tfevents.1644598421.dante.2344752.1 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:25ed8c04ecf820de1e43849ef6d73fd1d2bfc065b4b8e0ea55d4e4e4acc4b54a
3
+ size 4741
runs/Feb11_17-40-48_dante/events.out.tfevents.1644598421.dante.2344752.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e08300d3291d9aad128e41174c495ea05018dd7ad3834d0dcfbb5303cb1fda39
3
+ size 5008
runs/Feb11_17-40-48_dante/events.out.tfevents.1644599097.dante.2344752.2 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c1729f61fe3364a93e9a9d1f66da5ee6cdf7d7f562da2b58868f2f427d68e93f
3
+ size 364
special_tokens_map.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"bos_token": "<s>", "eos_token": "</s>", "unk_token": "[UNK]", "pad_token": "[PAD]", "additional_special_tokens": [{"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}]}
tokenizer_config.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"unk_token": "[UNK]", "bos_token": "<s>", "eos_token": "</s>", "pad_token": "[PAD]", "do_lower_case": false, "word_delimiter_token": "|", "special_tokens_map_file": null, "tokenizer_file": null, "name_or_path": "./", "tokenizer_class": "Wav2Vec2CTCTokenizer", "processor_class": "Wav2Vec2ProcessorWithLM"}
training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:74c7011121d1be25821c63534830c66a1646d322c4dc925d27818c2ade5a1e67
3
+ size 2991
vocab.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"a": 1, "b": 2, "c": 3, "d": 4, "e": 5, "f": 6, "g": 7, "h": 8, "i": 9, "j": 10, "k": 11, "l": 12, "m": 13, "n": 14, "o": 15, "p": 16, "q": 17, "r": 18, "s": 19, "t": 20, "u": 21, "v": 22, "w": 23, "x": 24, "y": 25, "z": 26, "å": 27, "æ": 28, "ø": 29, "|": 0, "[UNK]": 30, "[PAD]": 31}