Update tokenization_chatglm.py

#7
by felixdae - opened
Files changed (1) hide show
  1. tokenization_chatglm.py +2 -0
tokenization_chatglm.py CHANGED
@@ -72,6 +72,8 @@ class SPTokenizer:
72
  return self.index_special_tokens[index]
73
  if index in [self.eos_id, self.bos_id, self.pad_id] or index < 0:
74
  return ""
 
 
75
  return self.sp_model.IdToPiece(index)
76
 
77
 
 
72
  return self.index_special_tokens[index]
73
  if index in [self.eos_id, self.bos_id, self.pad_id] or index < 0:
74
  return ""
75
+ if index >= self.sp_model.vocab_size():
76
+ return ""
77
  return self.sp_model.IdToPiece(index)
78
 
79