duzx16 commited on
Commit
4e38bef
1 Parent(s): 8eb45c8

Fix clean_up_tokenization_spaces

Browse files
Files changed (1) hide show
  1. tokenization_chatglm.py +2 -2
tokenization_chatglm.py CHANGED
@@ -65,8 +65,8 @@ class ChatGLMTokenizer(PreTrainedTokenizer):
65
 
66
  model_input_names = ["input_ids", "attention_mask", "position_ids"]
67
 
68
- def __init__(self, vocab_file, padding_side="left", **kwargs):
69
- super().__init__(padding_side=padding_side, clean_up_tokenization_spaces=False, **kwargs)
70
  self.name = "GLMTokenizer"
71
 
72
  self.vocab_file = vocab_file
 
65
 
66
  model_input_names = ["input_ids", "attention_mask", "position_ids"]
67
 
68
+ def __init__(self, vocab_file, padding_side="left", clean_up_tokenization_spaces=False, **kwargs):
69
+ super().__init__(padding_side=padding_side, clean_up_tokenization_spaces=clean_up_tokenization_spaces, **kwargs)
70
  self.name = "GLMTokenizer"
71
 
72
  self.vocab_file = vocab_file