distinct

tokenizer_13a.py

@@ -68,6 +68,7 @@ class TokenizerRegexp(BaseTokenizer):
         # return ' '.join(line.split())
         # This line is changed with regards to the original tokenizer (seen above) to return individual words
         print(line)
+        print("1:", line.split())
         return line.split()