Update README.md

README.md

@@ -8,6 +8,8 @@ tags:
 - thudm
 ---
 # ChatGLM-6B
+**本仓库已经不再维护，请使用 [ChatGLM-6B-INT4](https://huggingface.co/THUDM/chatglm-6b-int4)**
+
 ## 介绍
 ChatGLM-6B 是一个开源的、支持中英双语问答的对话语言模型，基于 [General Language Model (GLM)](https://github.com/THUDM/GLM) 架构，具有 62 亿参数。结合模型量化技术，用户可以在消费级的显卡上进行本地部署（INT4 量化级别下最低只需 6GB 显存）。ChatGLM-6B 使用了和 [ChatGLM](https://chatglm.cn) 相同的技术，针对中文问答和对话进行了优化。经过约 1T 标识符的中英双语训练，辅以监督微调、反馈自助、人类反馈强化学习等技术的加持，62 亿参数的 ChatGLM-6B 已经能生成相当符合人类偏好的回答。
 

tokenization_chatglm.py

@@ -171,8 +171,8 @@ class ChatGLMTokenizer(PreTrainedTokenizer):
             do_lower_case=False,
             remove_space=False,
             bos_token='<sop>',
-            eos_token='</s>',
-            eop_token='<eop>',
+            eos_token='<eop>',
+            end_token='</s>',
             mask_token='[MASK]',
             gmask_token='[gMASK]',
             padding_side="left",
@@ -185,7 +185,7 @@ class ChatGLMTokenizer(PreTrainedTokenizer):
             padding_side=padding_side,
             bos_token=bos_token,
             eos_token=eos_token,
-            eop_token=eop_token,
+            end_token=end_token,
             mask_token=mask_token,
             gmask_token=gmask_token,
             num_image_tokens=num_image_tokens,
@@ -198,7 +198,7 @@ class ChatGLMTokenizer(PreTrainedTokenizer):
 
         self.bos_token = bos_token
         self.eos_token = eos_token
-        self.eop_token = eop_token
+        self.end_token = end_token
         self.mask_token = mask_token
         self.gmask_token = gmask_token
 
@@ -213,14 +213,14 @@ class ChatGLMTokenizer(PreTrainedTokenizer):
         return self.convert_tokens_to_ids(self.gmask_token)
 
     @property
-    def eop_token_id(self) -> Optional[int]:
+    def end_token_id(self) -> Optional[int]:
         """
-        `Optional[int]`: Id of the end of sentence token in the vocabulary. Returns `None` if the token has not been
+        `Optional[int]`: Id of the end of context token in the vocabulary. Returns `None` if the token has not been
         set.
         """
-        if self.eop_token is None:
+        if self.end_token is None:
             return None
-        return self.convert_tokens_to_ids(self.eop_token)
+        return self.convert_tokens_to_ids(self.end_token)
 
     @property
     def vocab_size(self):
@@ -324,18 +324,18 @@ class ChatGLMTokenizer(PreTrainedTokenizer):
         """
         mask_ids = self.sp_tokenizer[self.mask_token]
         gmask_ids = self.sp_tokenizer[self.gmask_token]
-        eop_id = self.sp_tokenizer[self.eop_token]
+        eos_id = self.sp_tokenizer[self.eos_token]
         if mask_ids not in token_ids_0 and gmask_ids not in token_ids_0:
             token_ids_0 += [gmask_ids]
 
         if token_ids_0[-1] != mask_ids and token_ids_0[-1] != gmask_ids:
-            token_ids_0 += [self.sp_tokenizer[self.eos_token]]
+            token_ids_0 += [self.sp_tokenizer[self.end_token]]
 
         token_ids_0 += [self.sp_tokenizer[self.bos_token]]
 
         if token_ids_1 is not None:
-            if not token_ids_1 or token_ids_1[-1] != eop_id:
-                token_ids_1 += [eop_id]
+            if not token_ids_1 or token_ids_1[-1] != eos_id:
+                token_ids_1 += [eos_id]
             token_ids_0 += token_ids_1
 
         return token_ids_0

tokenizer_config.json

@@ -1,8 +1,8 @@
 {
   "name_or_path": "THUDM/chatglm-6b",
   "bos_token": "<sop>",
-  "eop_token": "<eop>",
-  "eos_token": "</s>",
+  "eos_token": "<eop>",
+  "end_token": "</s>",
   "gmask_token": "[gMASK]",
   "mask_token": "[MASK]",
   "pad_token": "<pad>",