zRzRzRzRzRzRzR commited on
Commit
37f2196
1 Parent(s): b098244

fix for apply_chat_template

Browse files
Files changed (1) hide show
  1. tokenization_chatglm.py +1 -1
tokenization_chatglm.py CHANGED
@@ -32,7 +32,7 @@ class SPTokenizer:
32
  self.special_tokens[token] = self.n_words
33
  self.index_special_tokens[self.n_words] = token
34
  self.n_words += 1
35
- self.role_special_token_expression = "|".join([re.escape(token) for token in role_special_tokens])
36
 
37
  def tokenize(self, s: str, encode_special_tokens=False):
38
  if encode_special_tokens:
 
32
  self.special_tokens[token] = self.n_words
33
  self.index_special_tokens[self.n_words] = token
34
  self.n_words += 1
35
+ self.role_special_token_expression = "|".join([re.escape(token) for token in special_tokens]) # for apply_chat_template
36
 
37
  def tokenize(self, s: str, encode_special_tokens=False):
38
  if encode_special_tokens: