Gong Baitao commited on
Commit
31ce9ce
1 Parent(s): 1625156

fix normal token with <>

Browse files
Files changed (1) hide show
  1. tokenization_cpmbee.py +1 -0
tokenization_cpmbee.py CHANGED
@@ -236,6 +236,7 @@ class CpmBeeTokenizer(PreTrainedTokenizer):
236
  tail = sentence_split.pop(-1)
237
  sentence_split[-1] += tail
238
  sentence_split.append(c)
 
239
  elif c == ">":
240
  # end of special token
241
  sentence_split[-1] += c
 
236
  tail = sentence_split.pop(-1)
237
  sentence_split[-1] += tail
238
  sentence_split.append(c)
239
+ is_special_token = False
240
  elif c == ">":
241
  # end of special token
242
  sentence_split[-1] += c