jon-tow commited on
Commit
21ee10d
1 Parent(s): 810b45c

fix(tokenizer): expose `errors`

Browse files
Files changed (1) hide show
  1. tokenization_arcade100k.py +2 -0
tokenization_arcade100k.py CHANGED
@@ -111,6 +111,8 @@ class Arcade100kTokenizer(PreTrainedTokenizer):
111
  **kwargs,
112
  ):
113
  super().__init__(errors=errors, **kwargs)
 
 
114
  self._tiktoken_config = _arcade100k(vocab_file)
115
  self.tokenizer = tiktoken.Encoding(**self._tiktoken_config)
116
 
 
111
  **kwargs,
112
  ):
113
  super().__init__(errors=errors, **kwargs)
114
+ self.errors = errors
115
+
116
  self._tiktoken_config = _arcade100k(vocab_file)
117
  self.tokenizer = tiktoken.Encoding(**self._tiktoken_config)
118