austindavis commited on
Commit
7873f14
1 Parent(s): e685e45

Update agents/uci_tokenizers.py

Browse files
Files changed (1) hide show
  1. agents/uci_tokenizers.py +7 -1
agents/uci_tokenizers.py CHANGED
@@ -116,7 +116,6 @@ class UciTokenizer(PreTrainedTokenizerFast):
116
  return " ".join(processed_tokens)
117
 
118
  raise ValueError(f"Unknown input type to decode() for argument 'token_ids'. Received: {type(token_ids)} ")
119
-
120
 
121
  self._decode = _decode
122
 
@@ -159,6 +158,13 @@ class UciTileTokenizer(UciTokenizer):
159
  return self.id2square
160
 
161
  def __init__(self, **kwargs):
 
 
 
 
 
 
 
162
  super().__init__(
163
  self.stoi,
164
  self.itos,
 
116
  return " ".join(processed_tokens)
117
 
118
  raise ValueError(f"Unknown input type to decode() for argument 'token_ids'. Received: {type(token_ids)} ")
 
119
 
120
  self._decode = _decode
121
 
 
158
  return self.id2square
159
 
160
  def __init__(self, **kwargs):
161
+ # Remove conflicting arguments from kwargs if they exist
162
+ kwargs.pop("pad_token", None)
163
+ kwargs.pop("unk_token", None)
164
+ kwargs.pop("bos_token", None)
165
+ kwargs.pop("eos_token", None)
166
+ kwargs.pop("clean_up_tokenization_spaces", None)
167
+ kwargs.pop("name_or_path", None)
168
  super().__init__(
169
  self.stoi,
170
  self.itos,