austindavis
commited on
Commit
•
7873f14
1
Parent(s):
e685e45
Update agents/uci_tokenizers.py
Browse files- agents/uci_tokenizers.py +7 -1
agents/uci_tokenizers.py
CHANGED
@@ -116,7 +116,6 @@ class UciTokenizer(PreTrainedTokenizerFast):
|
|
116 |
return " ".join(processed_tokens)
|
117 |
|
118 |
raise ValueError(f"Unknown input type to decode() for argument 'token_ids'. Received: {type(token_ids)} ")
|
119 |
-
|
120 |
|
121 |
self._decode = _decode
|
122 |
|
@@ -159,6 +158,13 @@ class UciTileTokenizer(UciTokenizer):
|
|
159 |
return self.id2square
|
160 |
|
161 |
def __init__(self, **kwargs):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
162 |
super().__init__(
|
163 |
self.stoi,
|
164 |
self.itos,
|
|
|
116 |
return " ".join(processed_tokens)
|
117 |
|
118 |
raise ValueError(f"Unknown input type to decode() for argument 'token_ids'. Received: {type(token_ids)} ")
|
|
|
119 |
|
120 |
self._decode = _decode
|
121 |
|
|
|
158 |
return self.id2square
|
159 |
|
160 |
def __init__(self, **kwargs):
|
161 |
+
# Remove conflicting arguments from kwargs if they exist
|
162 |
+
kwargs.pop("pad_token", None)
|
163 |
+
kwargs.pop("unk_token", None)
|
164 |
+
kwargs.pop("bos_token", None)
|
165 |
+
kwargs.pop("eos_token", None)
|
166 |
+
kwargs.pop("clean_up_tokenization_spaces", None)
|
167 |
+
kwargs.pop("name_or_path", None)
|
168 |
super().__init__(
|
169 |
self.stoi,
|
170 |
self.itos,
|