distinct

tokenizer_13a.py

@@ -67,6 +67,7 @@ class TokenizerRegexp(BaseTokenizer):
         # no leading or trailing spaces, single space within words
         # return ' '.join(line.split())
         # This line is changed with regards to the original tokenizer (seen above) to return individual words
+        print(line)
         return line.split()
 
 
@@ -96,6 +97,7 @@ class Tokenizer13a(BaseTokenizer):
             line = line.replace("&", "&")
             line = line.replace("&lt;", "<")
             line = line.replace("&gt;", ">")
+        print(line)
 
         return self._post_tokenizer(f" {line} ")