add run_evaluation.py
Browse files- run_evaluation.py +3 -1
run_evaluation.py
CHANGED
|
@@ -22,7 +22,9 @@ class KenLM:
|
|
| 22 |
vocab_dict = tokenizer.get_vocab()
|
| 23 |
self.vocabulary = [x[0] for x in sorted(vocab_dict.items(), key=lambda x: x[1], reverse=False)]
|
| 24 |
self.vocabulary = self.vocabulary[:-2]
|
| 25 |
-
|
|
|
|
|
|
|
| 26 |
|
| 27 |
@staticmethod
|
| 28 |
def lm_postprocess(text):
|
|
|
|
| 22 |
vocab_dict = tokenizer.get_vocab()
|
| 23 |
self.vocabulary = [x[0] for x in sorted(vocab_dict.items(), key=lambda x: x[1], reverse=False)]
|
| 24 |
self.vocabulary = self.vocabulary[:-2]
|
| 25 |
+
with open(unigrams, "r") as f:
|
| 26 |
+
unigrams_text = f.read()
|
| 27 |
+
self.decoder = build_ctcdecoder(self.vocabulary, model_name, unigrams=unigrams_text)
|
| 28 |
|
| 29 |
@staticmethod
|
| 30 |
def lm_postprocess(text):
|