update tokenizer
Browse files- tokenizer.py +0 -1
tokenizer.py
CHANGED
|
@@ -357,7 +357,6 @@ class LanTokenizerSFT(BaseTokenizer):
|
|
| 357 |
print(f"Invalid PGN: {text}")
|
| 358 |
lan_tokens = []
|
| 359 |
for word in text.split():
|
| 360 |
-
print(word)
|
| 361 |
lan_tokens.extend(self._lan_move_to_tokens(word))
|
| 362 |
tokens = [self._bos] + lan_tokens + [self._eos]
|
| 363 |
|
|
|
|
| 357 |
print(f"Invalid PGN: {text}")
|
| 358 |
lan_tokens = []
|
| 359 |
for word in text.split():
|
|
|
|
| 360 |
lan_tokens.extend(self._lan_move_to_tokens(word))
|
| 361 |
tokens = [self._bos] + lan_tokens + [self._eos]
|
| 362 |
|