versae commited on
Commit
a4d07cc
1 Parent(s): 62b28aa

Update eval.py

Browse files
Files changed (1) hide show
  1. eval.py +125 -22
eval.py CHANGED
@@ -5,16 +5,26 @@ from typing import Dict
5
 
6
  import torch
7
  from datasets import Audio, Dataset, load_dataset, load_metric
 
 
8
 
9
- from transformers import AutoFeatureExtractor, pipeline
 
 
 
10
 
11
 
12
  def log_results(result: Dataset, args: Dict[str, str]):
13
  """DO NOT CHANGE. This function computes and logs the result metrics."""
14
 
15
  log_outputs = args.log_outputs
 
16
  model_id = args.model_id.replace("/", "_").replace(".", "")
17
- dataset_id = "_".join(args.dataset.split("/") + [model_id, args.config, args.split])
 
 
 
 
18
 
19
  # load metric
20
  wer = load_metric("wer")
@@ -30,6 +40,8 @@ def log_results(result: Dataset, args: Dict[str, str]):
30
 
31
  with open(f"{dataset_id}_eval_results.txt", "w") as f:
32
  f.write(result_str)
 
 
33
 
34
  # log all results in text file. Possibly interesting for analysis
35
  if log_outputs is not None:
@@ -37,7 +49,6 @@ def log_results(result: Dataset, args: Dict[str, str]):
37
  target_file = f"log_{dataset_id}_targets.txt"
38
 
39
  with open(pred_file, "w") as p, open(target_file, "w") as t:
40
-
41
  # mapping function to write output
42
  def write_to_file(batch, i):
43
  p.write(f"{i}" + "\n")
@@ -48,24 +59,80 @@ def log_results(result: Dataset, args: Dict[str, str]):
48
  result.map(write_to_file, with_indices=True)
49
 
50
 
51
- def normalize_text(text: str) -> str:
52
  """DO ADAPT FOR YOUR USE CASE. this function normalizes the target text."""
53
 
54
- chars_to_ignore_regex = '[\,\?\.\!\-\;\:\"\“\%\‘\”\�\'\–\_\\\+\#\/]' # noqa: W605 IMPORTANT: this should correspond to the chars that were ignored during training
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
55
 
56
- text = re.sub(chars_to_ignore_regex, "", text.lower()) + " "
57
- text = re.sub('[áàâ]', 'a', text)
58
- text = re.sub('[ä]', 'æ', text)
59
- text = re.sub('[éèëê]', 'e', text)
60
- text = re.sub('[íìïî]', 'i', text)
61
- text = re.sub('[óòöô]', 'o', text)
62
- text = re.sub('[ö]', 'ø', text)
63
- text = re.sub('[ç]', 'c', text)
64
- text = re.sub('[úùüû]', 'u', text)
65
- text = re.sub('\s', ' ', text)
66
- text = re.sub('<ee>', 'eee', text)
67
- text = re.sub('<qq>', 'qqq', text)
68
- text = re.sub('<mm>', 'mmm', text)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
69
  text = re.sub('<inaudible>', 'xxx', text)
70
  text = re.sub('[<>]', '', text)
71
 
@@ -76,13 +143,18 @@ def normalize_text(text: str) -> str:
76
  # for t in token_sequences_to_ignore:
77
  # text = " ".join(text.split(t))
78
 
79
- return text
80
 
81
 
82
  def main(args):
83
  # load dataset
84
  dataset = load_dataset(args.dataset, args.config, split=args.split, use_auth_token=True)
85
-
 
 
 
 
 
86
  # for testing: only process the first two examples as a test
87
  # dataset = dataset.select(range(10))
88
 
@@ -96,7 +168,29 @@ def main(args):
96
  # load eval pipeline
97
  if args.device is None:
98
  args.device = 0 if torch.cuda.is_available() else -1
99
- asr = pipeline("automatic-speech-recognition", model=args.model_id, device=args.device)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
100
 
101
  # map function to decode audio
102
  def map_to_pred(batch):
@@ -105,7 +199,7 @@ def main(args):
105
  )
106
 
107
  batch["prediction"] = prediction["text"]
108
- batch["target"] = normalize_text(batch["text"])
109
  return batch
110
 
111
  # run inference on all examples
@@ -131,7 +225,13 @@ if __name__ == "__main__":
131
  parser.add_argument(
132
  "--config", type=str, required=True, help="Config of the dataset. *E.g.* `'en'` for Common Voice"
133
  )
 
 
 
134
  parser.add_argument("--split", type=str, required=True, help="Split of the dataset. *E.g.* `'test'`")
 
 
 
135
  parser.add_argument(
136
  "--chunk_length_s", type=float, default=None, help="Chunk length in seconds. Defaults to 5 seconds."
137
  )
@@ -147,6 +247,9 @@ if __name__ == "__main__":
147
  default=None,
148
  help="The device to run the pipeline on. -1 for CPU (default), 0 for the first GPU and so on.",
149
  )
 
 
 
150
  args = parser.parse_args()
151
 
152
  main(args)
 
5
 
6
  import torch
7
  from datasets import Audio, Dataset, load_dataset, load_metric
8
+ from num2words import num2words as n2w
9
+ from slugify import slugify
10
 
11
+ from transformers import AutoFeatureExtractor, AutoModelForCTC, pipeline, Wav2Vec2Processor, Wav2Vec2ProcessorWithLM, Wav2Vec2FeatureExtractor
12
+ # from pyctcdecode import BeamSearchDecoderCTC
13
+
14
+ from cardinal_numbers import convert_nums
15
 
16
 
17
  def log_results(result: Dataset, args: Dict[str, str]):
18
  """DO NOT CHANGE. This function computes and logs the result metrics."""
19
 
20
  log_outputs = args.log_outputs
21
+ lm = "withLM" if args.use_lm else "noLM"
22
  model_id = args.model_id.replace("/", "_").replace(".", "")
23
+ if args.filter:
24
+ extra_args = [args.config, slugify(args.filter), args.split, lm]
25
+ else:
26
+ extra_args = [args.config, args.split, lm]
27
+ dataset_id = "_".join([model_id] + args.dataset.split("/") + extra_args)
28
 
29
  # load metric
30
  wer = load_metric("wer")
 
40
 
41
  with open(f"{dataset_id}_eval_results.txt", "w") as f:
42
  f.write(result_str)
43
+ with open(f"{dataset_id}_eval_results.tsv", "w") as f:
44
+ f.write("\t".join([args.model_id, args.dataset, args.config, args.filter, args.split, str(lm), str(wer_result), str(cer_result)]))
45
 
46
  # log all results in text file. Possibly interesting for analysis
47
  if log_outputs is not None:
 
49
  target_file = f"log_{dataset_id}_targets.txt"
50
 
51
  with open(pred_file, "w") as p, open(target_file, "w") as t:
 
52
  # mapping function to write output
53
  def write_to_file(batch, i):
54
  p.write(f"{i}" + "\n")
 
59
  result.map(write_to_file, with_indices=True)
60
 
61
 
62
+ def normalize_text(original_text: str, dataset: str) -> str:
63
  """DO ADAPT FOR YOUR USE CASE. this function normalizes the target text."""
64
 
65
+ text = original_text.lower()
66
+ if dataset.lower().endswith("fleurs"):
67
+ replacements = (
68
+ (r"\be\.kr", "etter kristus fødsel"),
69
+ (r"\bf\.kr", "før kristi fødsel"),
70
+ (r"\bca[.]?\b", "circa"),
71
+ (r"(\d)\s*km/t", r"\1 kilometer i timen"),
72
+ (r"(\d)\s*km", r"\1 kilometer"),
73
+ (r"(\d)\s*cm", r"\1 centimeter"),
74
+ (r"(\d)\s*mm", r"\1 millimeter"),
75
+ (r"kl\.", "klokka"),
76
+ (r"f\.eks", "for eksempel"),
77
+ )
78
+ for abrev, expasion in replacements:
79
+ text = re.sub(abrev, expasion, text)
80
+ text = re.sub(r'(\d+)[-–](\d+)', r'\1 til \2', text) # 1-89, 70-90
81
+ text = re.sub(r'(\d{2}):00', r'\1', text) # 21:00
82
+ text = re.sub(r"(\d{2}):0(\d{1})", r"\1 null \2", text) # 17:03
83
+ text = re.sub(r"(\d{1,2}):(\d{1,2})", r"\1 \2", text) # 17:23 (time), 4:3 (aspect ratios)
84
+ text = re.sub(r"(1[1-9])00", r"\1 hundre", text) # 1800, 1900
85
+ text = re.sub(r"(1[1-9])0([1-9])", r"\1 null \2 ", text) # 1901, 1909
86
+ text = re.sub(r"(1[1-9])([1-9]\d)", r"\1 \2 ", text) # 1911, 1987
87
+ text = re.sub(r"(20)0([1-9])", r"\1 null \2 ", text) # 2009
88
+ text = re.sub(r"(20)(\d{2})", r"\1 \2 ", text) # 2009
89
+ text = re.sub(r"(\d{1,3})[.](\d{1,2})", r"\1 dot \2 ", text) # 802.11n, 2.5ghz (in English)
90
+ text = re.sub(r"(\d{1,2})[ .](\d{3})", r"\1\2", text) # 10 000, 32.000
91
+ text = re.sub(r'(\w+)-(\w+)', r'\1 \2', text) # n-standard
92
+ # text = re.compile(r"-?0?[1-9][\d.]*").sub(lambda x: n2w(x.group(0), lang="no"), text.replace(".", ""))
93
+ text = re.compile(r"-?0?[1-9][\d.]*").sub(lambda x: convert_nums(int(x.group(0)), nn=True), text.replace(".", ""))
94
+
95
 
96
+ chars_to_ignore_regex = '[\,\?\.\!\-\;\:\"\“\%\‘\”\�\'\–\_\\\+\#\/]' # noqa: W605 IMPORTANT: this should correspond to the chars that were ignored during training
97
+ text = re.sub(chars_to_ignore_regex, "", text) + " "
98
+
99
+ if dataset.lower().endswith("nst"):
100
+ text = text.lower()
101
+ text = text.replace("(...vær stille under dette opptaket...)", "")
102
+ text = re.sub('[áàâ]', 'a', text)
103
+ text = re.sub('[ä]', 'æ', text)
104
+ text = re.sub('[éèëê]', 'e', text)
105
+ text = re.sub('[íìïî]', 'i', text)
106
+ text = re.sub('[óòöô]', 'o', text)
107
+ text = re.sub('[ö]', 'ø', text)
108
+ text = re.sub('[ç]', 'c', text)
109
+ text = re.sub('[úùüû]', 'u', text)
110
+ # text = re.sub('\\(?=(Punktum|Komma|Utropstegn|Spørsmålstegn))', ' ', text)
111
+ text = re.sub('\s+', ' ', text)
112
+ elif dataset.lower().endswith("npsc"):
113
+ text = re.sub('[áàâ]', 'a', text)
114
+ text = re.sub('[ä]', 'æ', text)
115
+ text = re.sub('[éèëê]', 'e', text)
116
+ text = re.sub('[íìïî]', 'i', text)
117
+ text = re.sub('[óòöô]', 'o', text)
118
+ text = re.sub('[ö]', 'ø', text)
119
+ text = re.sub('[ç]', 'c', text)
120
+ text = re.sub('[úùüû]', 'u', text)
121
+ text = re.sub('\s+', ' ', text)
122
+ elif dataset.lower().endswith("fleurs"):
123
+ text = re.sub('[áàâ]', 'a', text)
124
+ text = re.sub('[ä]', 'æ', text)
125
+ text = re.sub('[éèëê]', 'e', text)
126
+ text = re.sub('[íìïî]', 'i', text)
127
+ text = re.sub('[óòöô]', 'o', text)
128
+ text = re.sub('[ö]', 'ø', text)
129
+ text = re.sub('[ç]', 'c', text)
130
+ text = re.sub('[úùüû]', 'u', text)
131
+ text = re.sub('[«»]', '', text)
132
+ text = re.sub('\s+', ' ', text)
133
+ text = re.sub('<e+h?>', 'eee', text)
134
+ text = re.sub('<m+>', 'mmm', text)
135
+ text = re.sub('<q+>', 'qqq', text)
136
  text = re.sub('<inaudible>', 'xxx', text)
137
  text = re.sub('[<>]', '', text)
138
 
 
143
  # for t in token_sequences_to_ignore:
144
  # text = " ".join(text.split(t))
145
 
146
+ return text.strip()
147
 
148
 
149
  def main(args):
150
  # load dataset
151
  dataset = load_dataset(args.dataset, args.config, split=args.split, use_auth_token=True)
152
+ if args.filter:
153
+ attribute, value = list(map(str.strip, args.filter.split(":")))
154
+ dataset = dataset.filter(
155
+ lambda x: x[attribute] == value,
156
+ desc=f"Filtering on {args.filter}",
157
+ )
158
  # for testing: only process the first two examples as a test
159
  # dataset = dataset.select(range(10))
160
 
 
168
  # load eval pipeline
169
  if args.device is None:
170
  args.device = 0 if torch.cuda.is_available() else -1
171
+ # asr = pipeline("automatic-speech-recognition", model=args.model_id, device=args.device)
172
+
173
+ model_instance = AutoModelForCTC.from_pretrained(args.model_id)
174
+ if args.use_lm:
175
+ processor = Wav2Vec2ProcessorWithLM.from_pretrained(args.model_id)
176
+ decoder = processor.decoder
177
+ else:
178
+ processor = Wav2Vec2Processor.from_pretrained(args.model_id)
179
+ decoder = None
180
+ asr = pipeline(
181
+ "automatic-speech-recognition",
182
+ model=model_instance,
183
+ tokenizer=processor.tokenizer,
184
+ feature_extractor=processor.feature_extractor,
185
+ decoder=decoder,
186
+ device=args.device
187
+ )
188
+
189
+ # feature_extractor_dict, _ = Wav2Vec2FeatureExtractor.get_feature_extractor_dict(args.model_id)
190
+ # feature_extractor_dict["processor_class"] = "Wav2Vec2Processor" if not args.use_lm else "Wav2Vec2ProcessorWithLM"
191
+ # feature_extractor = Wav2Vec2FeatureExtractor.from_dict(feature_extractor_dict)
192
+
193
+ # asr = pipeline("automatic-speech-recognition", model=args.model_id, feature_extractor=feature_extractor, device=args.device, decoder=BeamSearchDecoderCTC.load_from_dir("./"))
194
 
195
  # map function to decode audio
196
  def map_to_pred(batch):
 
199
  )
200
 
201
  batch["prediction"] = prediction["text"]
202
+ batch["target"] = normalize_text(batch[args.text_column], args.dataset)
203
  return batch
204
 
205
  # run inference on all examples
 
225
  parser.add_argument(
226
  "--config", type=str, required=True, help="Config of the dataset. *E.g.* `'en'` for Common Voice"
227
  )
228
+ parser.add_argument(
229
+ "--filter", type=str, default="", help="Simple filter on attributes. *E.g.* `region_of_youth:Troms` would pnly keep those samplesfor which the condition is met"
230
+ )
231
  parser.add_argument("--split", type=str, required=True, help="Split of the dataset. *E.g.* `'test'`")
232
+ parser.add_argument(
233
+ "--text_column", type=str, default="text", help="Column name containing the transcription."
234
+ )
235
  parser.add_argument(
236
  "--chunk_length_s", type=float, default=None, help="Chunk length in seconds. Defaults to 5 seconds."
237
  )
 
247
  default=None,
248
  help="The device to run the pipeline on. -1 for CPU (default), 0 for the first GPU and so on.",
249
  )
250
+ parser.add_argument(
251
+ "--use_lm", action="store_true", help="If defined, use included language model as the decoder."
252
+ )
253
  args = parser.parse_args()
254
 
255
  main(args)