distinct
Browse files- tokenizer_13a.py +1 -1
tokenizer_13a.py
CHANGED
@@ -101,4 +101,4 @@ class Tokenizer13a(BaseTokenizer):
|
|
101 |
|
102 |
@lru_cache(maxsize=2**16)
|
103 |
def tokenize(self, line):
|
104 |
-
self.__call__(
|
|
|
101 |
|
102 |
@lru_cache(maxsize=2**16)
|
103 |
def tokenize(self, line):
|
104 |
+
self.__call__(line)
|