zxdu20 commited on
Commit
36b7f2d
1 Parent(s): 63ce1ba

Add gmask token id

Browse files
Files changed (1) hide show
  1. tokenization_chatglm.py +6 -0
tokenization_chatglm.py CHANGED
@@ -200,6 +200,12 @@ class ChatGLMTokenizer(PreTrainedTokenizer):
200
 
201
  """ Initialisation """
202
 
 
 
 
 
 
 
203
  @property
204
  def eop_token_id(self) -> Optional[int]:
205
  """
 
200
 
201
  """ Initialisation """
202
 
203
+ @property
204
+ def gmask_token_id(self) -> Optional[int]:
205
+ if self.gmask_token is None:
206
+ return None
207
+ return self.convert_tokens_to_ids(self.gmask_token)
208
+
209
  @property
210
  def eop_token_id(self) -> Optional[int]:
211
  """