distinct

distinct.py

@@ -115,7 +115,7 @@ class distinct(evaluate.Measurement):
     def _download_and_prepare(self, dl_manager):
         """Optional: download external resources useful to compute the scores"""
 
-    def _compute(self, predictions, dataForVocabCal=None, vocab_size=None, tokenizer="white_space", mode="Expectation-Adjusted-Distinct"):
+    def _compute(self, predictions, dataForVocabCal=None, vocab_size=None, tokenizer=Tokenizer13a(), mode="Expectation-Adjusted-Distinct"):
         from nltk.util import ngrams
         from nltk.tokenize import WhitespaceTokenizer